C++ DE_FOR_EACH*/DE_FOR_OBSERVERS/DE_NOTIFY* macro statements are not functions; re-bless DOOM-rosetta (Part of #5630)

gitgalaxy/standards/language_standards/languages/cpp.py

@@ -321,6 +321,9 @@
             # 5. THE "NOT A FUNCTION" SHIELD
             # Prevents control flow (if, while) and primitive types from being captured as function names.
             r"(?!(?:if|for|while|switch|return|catch|else|elif|sizeof|alignas|new|delete|ARGS\d+|NOARGS|int|float|double|char|void|long|short|unsigned|signed|bool|INTEGER|LOGICAL|real|__attribute__|__declspec|__asm__)\b)"
+            # #5630: a macro-statement loop or observer dispatch is a statement, not a definition: doomsday's
+            # `DE_FOR_EACH_CONST(T, i, c)\n{`, `DE_FOR_OBSERVERS(i, a)`, `DE_NOTIFY_PUBLIC(Swap, i) {`, `DE_NOTIFY_VAR(`.
+            r"(?!DE_(?:FOR_(?:EACH|OBSERVERS)|NOTIFY)(?![" + ID_CONTINUE + r"]*[a-z]))"
             # 6. THE IDENTIFIER CAPTURE (FUNCTION IDENTIFIER - GROUP 1)
             # [IRON WALL]: Ensures the actual function/operator name isn't hijacked by a macro definition.
             # OUT-OF-LINE OPERATOR FIX (epic #813/#821): the operator alternatives had no

tests/extraction/languages/test_cpp_macro_statement_loop_5630.py

@@ -0,0 +1,43 @@
+"""
+#5630: a C++ macro-statement loop or observer dispatch is a statement inside its function, never a function.
+
+doomsday's `DE_FOR_EACH*(T, i, c)`, `DE_FOR_OBSERVERS(i, audience)` and `DE_NOTIFY*(Name, i)` take a body (a brace
+block or one statement) like `for`. They were read as functions named for the macro, and the enclosing function lost
+their body's calls. Driven through `StructuralExtractor.splice()`, the path a scan takes.
+"""
+
+import pytest
+
+from gitgalaxy.core.detector import StructuralExtractor
+from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS
+
+
+def _functions(code: str) -> dict[str, dict]:
+    found = StructuralExtractor("cpp", LANGUAGE_DEFINITIONS).splice(code, "")["functions"]
+    return {f["name"]: f for f in found}
+
+
+@pytest.mark.parametrize(
+    "stmt",
+    [
+        "DE_FOR_EACH_CONST(Map::Elements, i, dict.elements())\n    {\n        hit(i);\n    }",
+        "DE_FOR_EACH(Lines, i, lines) { hit(*i); }",
+        "DE_FOR_EACH_CONST_REVERSE(Mapping, i, mapping)\n        hit(i);",
+        "DE_FOR_EACH_REVERSE(Mapping, i, mapping)\n    {\n        hit(i);\n    }",
+        "DE_FOR_OBSERVERS(i, wd->self().audience())\n    {\n        hit(i);\n    }",
+        "DE_NOTIFY_PUBLIC(Swap, i) { hit(i); }",
+        "DE_NOTIFY_PUBLIC(Resize, i)\n    {\n        hit(i);\n    }",
+        "DE_NOTIFY_PUBLIC_VAR(Change, i)\n    {\n        hit(i);\n    }",
+        "DE_NOTIFY(Change, i) { hit(i); }",
+        "DE_NOTIFY_VAR(Change, i)\n        hit(i);",
+    ],
+)
+def test_a_macro_statement_is_not_a_function_and_its_body_stays_in_the_caller(stmt):
+    functions = _functions(f"void run(Dict &d)\n{{\n    {stmt}\n}}\n")
+    assert list(functions) == ["run"]
+    assert "hit" in functions["run"]["calls_out_to"]
+
+
+def test_a_function_named_like_a_macro_prefix_is_still_a_function():
+    functions = _functions("void DE_NOTIFYfoo(int x)\n{\n    go(x);\n}\nvoid DE_FOR_EACHx(int x)\n{\n    go(x);\n}\n")
+    assert {"DE_NOTIFYfoo", "DE_FOR_EACHx"} <= set(functions)

tests/tools/doom_speedrun/callgraph_baseline.json

@@ -24,7 +24,7 @@
   },
   "quorum": 0.75
  },
- "engine_sha": "e4063a391",
+ "engine_sha": "a2b6c416c",
  "ports": {
   "chocolate-doom": {
    "agree": 12132,
@@ -130,31 +130,31 @@
    "unmatched_functions": 1
   },
   "doomsday": {
-   "agree": 69035,
-   "engine_calls": 71541,
+   "agree": 68874,
+   "engine_calls": 71351,
    "engine_only_by_kind": {
-    "absent": 256,
+    "absent": 231,
     "comment/string": 3,
-    "function-pointer": 239,
+    "function-pointer": 223,
     "macro": 7,
-    "type": 2001
+    "type": 2013
    },
-   "functions": 25733,
+   "functions": 25449,
    "misses_by_shape": {
-    "after-operator": 39,
-    "assignment-rhs": 25,
-    "condition/return": 10,
+    "after-operator": 14,
+    "assignment-rhs": 6,
+    "condition/return": 7,
     "ctor-init": 5,
-    "nested-argument": 48,
-    "other": 74,
-    "statement": 103,
-    "subscript": 21,
-    "unparsed": 37
+    "nested-argument": 10,
+    "other": 53,
+    "statement": 14,
+    "subscript": 19,
+    "unparsed": 16
    },
-   "oracle_calls": 69397,
-   "precision": 0.965,
-   "recall": 0.9948,
-   "text_fallback_functions": 825,
+   "oracle_calls": 69018,
+   "precision": 0.9653,
+   "recall": 0.9979,
+   "text_fallback_functions": 754,
    "unmatched_functions": 1
   },
   "dsda-doom": {

tests/tools/doom_speedrun/callgraph_oracle.py

@@ -355,17 +355,22 @@ def class_names(repo):
     return bases_of
 
 
+# doomsday's macro statements that take a body, like `for` (`DE_FOR_EACH_CONST(T, i, c) { ... }`, `DE_NOTIFY_PUBLIC(N, i) ...`):
+# tree-sitter reads one as a nested definition, the engine as a statement of its function (#5630).
+MACRO_STATEMENT = re.compile(r"DE_(?:FOR_(?:EACH|OBSERVERS)|NOTIFY)[A-Z_]*")
+
+
 def _is_real_nested_function(node):
     """A nested ``function_definition`` that is a function (a GNU C nested function), not the ``else if (...) {`` that
-    ``#ifdef`` splitting makes tree-sitter read as one: its type or declarator name is a keyword."""
+    ``#ifdef`` splitting makes tree-sitter read as one (its type or declarator name is a keyword) nor a macro statement."""
     kind = node.child_by_field_name("type")
     decl = node.child_by_field_name("declarator")
     while decl is not None and decl.type in ("pointer_declarator", "reference_declarator"):
         # `Hash &hashFor(int)` / `Item *find()`: the function_declarator sits under the `&` / `*` (a DE_PIMPL method
         # returning one was read as no function, so its calls were credited to the class, 1085 doomsday misses, #6266)
         decl = decl.child_by_field_name("declarator") or next(iter(decl.named_children), None)
     name = callee_node(decl.child_by_field_name("declarator")) if decl is not None else None
-    if name is None or decode_bytes(name.text) in KEYWORDS:
+    if name is None or decode_bytes(name.text) in KEYWORDS or MACRO_STATEMENT.fullmatch(decode_bytes(name.text)):
         return False
     return kind is None or decode_bytes(kind.text) not in KEYWORDS
 

tests/tools/test_callgraph_oracle.py

@@ -463,3 +463,18 @@ def test_a_macro_class_owns_only_its_member_initializer_calls():
 def test_a_macro_statement_with_a_brace_body_keeps_every_call():
     src = "DE_FOR_EACH(i, list)\n{\n    stmt();\n    other(1);\n}\n"
     assert set(co.TextBody(src.encode(), 1).calls) == {"stmt", "other"}
+
+
+@pytest.mark.parametrize(
+    "stmt",
+    [
+        "DE_FOR_EACH_CONST(Map::Elements, i, dict.elements())\n    {\n        inner(i);\n    }",
+        "DE_NOTIFY_PUBLIC(Swap, i) { inner(i); }",
+        "DE_FOR_OBSERVERS(i, audience())\n        inner(i);",
+    ],
+)
+def test_a_macro_statement_loop_is_part_of_its_function_not_a_nested_one(stmt):
+    """The engine reads `DE_FOR_EACH*` / `DE_NOTIFY*` as a statement of the enclosing function; so does the oracle."""
+    body = body_of(f"void run(Dict &d)\n{{\n    {stmt}\n}}\n", "cpp")
+    assert "inner" in body.calls
+    assert body.nested == []