codmap 0.0.3__py3-none-any.whl

codemap/extract/dataflow.py

@@ -0,0 +1,97 @@
+"""Dataflow-by-string-key pass — column/key flow (DESIGN §7, M12, gap-doc F6).
+
+The behavioral layer follows *symbols*; but bquant threads its data through
+string-keyed DataFrame columns — ``df['macd_hist'] = …`` here, ``macd_data['macd_hist']``
+there — and the call-graph sees none of it (dogfood F6: querying ``macd_hist``
+returned nothing, worse than grep). This pass makes those keys first-class:
+
+- a ``column`` node per distinct string key (``column:macd_hist``);
+- a ``writes`` edge  (function → column) for ``x['key'] = …`` (Store subscript) and
+  for a dict-literal producer ``{'key': value}`` (how bquant returns computed columns);
+- a ``reads``  edge  (function → column) for ``x['key']`` used as a value (Load).
+
+**Honesty.** This is *string-keyed access* flow, an **over-set** of DataFrame
+columns: ``config['path']`` and ``d['k']`` land here too — statically we cannot
+tell a frame from a dict without type inference. Querying a *specific* key
+(``macd_hist``) is precise regardless; treat the node set as "keys", not "columns".
+Embedded sample-data modules (literal datasets) are skipped — they are data, not code.
+"""
+
+from __future__ import annotations
+
+import ast
+
+from codemap.extract.behavior import _index_modules, _named_functions, _node_id
+from codemap.extract.gsource import module_file
+from codemap.model import Edge, Node
+
+_COLUMN_PREFIX = "column:"
+
+
+def add_dataflow(graph, griffe_root, target_pkg: str) -> None:
+    """Add column nodes + reads/writes edges for string-keyed subscripts.
+
+    Each column node records ``extras.subscripted`` (M14/F15): True iff the key was
+    ever accessed as ``x['k']`` somewhere, not only used as a dict-literal payload
+    key. Each edge records ``extras.access`` (``subscript`` | ``dict-literal``). The
+    B1 dogfood found 71% of keys were dict-literal-only (result dicts, config,
+    rcParams) — the flag lets aggregates surface the real column-like set.
+    """
+    modules = _index_modules(griffe_root)
+    subscripted: dict[str, bool] = {}  # key -> ever accessed as a subscript
+    edges: set[tuple[str, str, str, str]] = set()  # (type, func_id, col_id, access)
+    for modpath in sorted(modules):
+        if "samples.embedded" in modpath:
+            continue  # embedded datasets are data, not code
+        mod = modules[modpath]
+        fp = module_file(mod)  # None for a namespace dir (R1-C21)
+        if fp is None:
+            continue
+        try:
+            tree = ast.parse(fp.read_text(encoding="utf-8"))
+        except (OSError, SyntaxError):
+            continue
+        for fnode, class_stack in _named_functions(tree):
+            node_id = _node_id(modpath, class_stack, fnode.name)
+            if node_id not in graph.nodes:
+                continue
+            for key, is_write, is_subscript in _own_key_uses(fnode):
+                col_id = _COLUMN_PREFIX + key
+                subscripted[key] = subscripted.get(key, False) or is_subscript
+                access = "subscript" if is_subscript else "dict-literal"
+                edges.add(("writes" if is_write else "reads", node_id, col_id, access))
+
+    for key in sorted(subscripted):
+        col_id = _COLUMN_PREFIX + key
+        if col_id not in graph.nodes:
+            graph.add_node(Node(id=col_id, kind="column",
+                                extras={"key": key, "root": "core",
+                                        "subscripted": subscripted[key]}))
+    for etype, src, col_id, access in sorted(edges):
+        graph.add_edge(Edge(etype, src, col_id,
+                            extras={"resolution": "string-key", "access": access}))
+
+
+def _own_key_uses(fnode):
+    """Yield (key, is_write, is_subscript) for string-keyed use in a function body.
+
+    Skips nested defs/classes (their own scope). ``is_write`` is set for a Store
+    subscript (``x['k'] = …`` / ``x['k'] += …``) and for a dict-literal key
+    (``{'k': value}`` — a producer of that key). A Load subscript is a read.
+    ``is_subscript`` distinguishes container access (``x['k']``) from a dict-literal
+    payload key (``{'k': …}``) — the F15 precision discriminator.
+    """
+    def visit(node):
+        for child in ast.iter_child_nodes(node):
+            if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                continue
+            if isinstance(child, ast.Subscript) and isinstance(child.slice, ast.Constant) \
+                    and isinstance(child.slice.value, str):
+                yield child.slice.value, isinstance(child.ctx, ast.Store), True
+            elif isinstance(child, ast.Dict):
+                for k in child.keys:
+                    if isinstance(k, ast.Constant) and isinstance(k.value, str):
+                        yield k.value, True, False  # dict-literal key = producer (write)
+            yield from visit(child)
+
+    yield from visit(fnode)

codemap/extract/dispatch.py

@@ -0,0 +1,248 @@
+"""Registry-aware call bridging — reconnect the dispatch seams (DESIGN §7, M7).
+
+The behavioral pass (M4/M5) leaves calls that go through a factory/registry
+*unresolved*: bquant wires plugins by string key, not import —
+``self.swing_strategy = create_swing_strategy(name)`` then
+``self.swing_strategy.calculate_global(data)``. So the call chain from
+``analyze_zones`` dies exactly at the plugin seams — the swing/shape/… strategies
+and the zone detector, which is *where the real work happens* (dogfood F5).
+
+But codemap already holds the dispatch table: M1.5 recorded every
+``@Registry.register('key')`` binding in ``extras.registry`` ({key → class}). This
+pass uses it to bridge the seams with **candidate** edges — an honest
+over-approximation ("dispatches to one of {zigzag, find_peaks, …}") flagged
+``resolution="registry-candidate"``; a literal key resolves to a single class
+(``resolution="registry"``). Concrete strategies do NOT inherit their Protocol,
+so families are grouped by the *registrar*, not by inheritance.
+
+Bridged forms:
+- ``self.attr = create_X(...)`` in a class → ``self.attr.method(...)`` elsewhere in
+  that class → edges to ``{impl}.method`` for every impl of family X;
+- a direct factory/getter call → edges to the family's implementation classes
+  (exact class when the key argument is a string literal).
+"""
+
+from __future__ import annotations
+
+import ast
+
+from codemap.extract.behavior import _index_modules, _named_functions, _node_id, _own_calls
+from codemap.extract.gsource import module_file
+from codemap.model import Edge
+
+_DISPATCH_VERBS = ("get", "create", "make", "build", "new")
+
+
+def add_dispatch(graph, griffe_root, target_pkg: str) -> None:
+    """Bridge factory/registry dispatch seams using the M1.5 registry table."""
+    families = _build_families(graph)
+    if not families:
+        return
+    recognizer = _Recognizer(families)
+
+    modules = _index_modules(griffe_root)
+    edges: dict[tuple[str, str, str], str] = {}  # (src, tgt, kind) -> resolution (best)
+    for modpath in sorted(modules):
+        mod = modules[modpath]
+        fp = module_file(mod)  # None for a namespace dir (R1-C21)
+        if fp is None:
+            continue
+        try:
+            tree = ast.parse(fp.read_text(encoding="utf-8"))
+        except (OSError, SyntaxError):
+            continue
+        funcs = _named_functions(tree)
+        # pass A: bind class attributes to families (self.attr = create_X(...)).
+        attr_family: dict[str, dict[str, tuple]] = {}
+        for fnode, class_stack in funcs:
+            if not class_stack:
+                continue
+            class_id = ".".join([modpath, *class_stack])
+            for attr, fam in _attr_bindings(fnode, recognizer):
+                attr_family.setdefault(class_id, {})[attr] = fam
+        # pass B: bridge dispatch call-sites to family members.
+        for fnode, class_stack in funcs:
+            node_id = _node_id(modpath, class_stack, fnode.name)
+            if node_id not in graph.nodes:
+                continue
+            class_id = ".".join([modpath, *class_stack]) if class_stack else ""
+            binds = attr_family.get(class_id, {})
+            for call in _own_calls(fnode):
+                for tgt, resolution in _bridge_call(call, recognizer, binds, families, graph):
+                    key = (node_id, tgt, "calls")
+                    # prefer the exact "registry" resolution over "registry-candidate".
+                    if key not in edges or resolution == "registry":
+                        edges[key] = resolution
+
+    for (src, tgt, _kind), resolution in edges.items():
+        graph.add_edge(Edge("calls", src, tgt, extras={"resolution": resolution}))
+
+
+def add_family_links(graph) -> None:
+    """Link a registry family's members to the Protocol they satisfy (F4, M9).
+
+    Concrete strategies use *structural* typing — they never inherit their
+    Protocol — so the dogfood (F4) found the family invisible to ``query`` and to
+    the class diagram even though every datum is in the graph. This synthesises an
+    ``implements`` edge from each registered impl to its Protocol, matched
+    data-driven: the family token (``swing``) against the Protocol name
+    (``SwingCalculationStrategy``); for a token-less registrar the registry class
+    name drives it (``ZoneDetectionRegistry`` → ``ZoneDetectionStrategy``). No
+    hardcoded package names — a foreign convention just yields no link.
+    """
+    families = _build_families(graph)
+    if not families:
+        return
+    protocols = _protocols(graph)  # {name_lower: protocol_id}
+    for fid, fam in sorted(families.items()):
+        proto_id = _match_protocol(fid, protocols)
+        if proto_id is None:
+            continue
+        for class_id in sorted(set(fam["members"].values())):
+            if class_id in graph.nodes:
+                graph.add_edge(Edge("implements", class_id, proto_id,
+                                    extras={"via": "registry"}))
+
+
+def _protocols(graph) -> dict[str, str]:
+    """Protocol classes (inherit ``typing.Protocol``) keyed by lowercased name."""
+    out: dict[str, str] = {}
+    for e in graph.edges:
+        if e.type == "inherits" and e.target == "typing.Protocol":
+            out[e.source.rsplit(".", 1)[-1].lower()] = e.source
+    return out
+
+
+def _match_protocol(fid: tuple, protocols: dict[str, str]) -> str | None:
+    """Best Protocol for a family: token (or registry-class stem) ⊂ Protocol name."""
+    reg_class, token = fid
+    tokens = [token] if token else []
+    stem = reg_class[:-len("Registry")] if reg_class.endswith("Registry") else reg_class
+    if stem:
+        tokens.append(stem.lower())
+    for tok in tokens:
+        if not tok:
+            continue
+        matches = sorted((name, pid) for name, pid in protocols.items() if tok in name)
+        if matches:
+            return matches[0][1]  # deterministic: shortest/alphabetical name
+    return None
+
+
+# -- family table (from extras.registry) -------------------------------------
+
+def _build_families(graph) -> dict:
+    """family_id (reg_class, token) -> {'members': {key: class_id}, 'token': str}."""
+    families: dict = {}
+    for node in graph.nodes.values():
+        reg = node.extras.get("registry")
+        if not reg:
+            continue
+        fid = _family_of(reg.get("decorator", ""))
+        fam = families.setdefault(fid, {"members": {}, "token": fid[1]})
+        fam["members"][reg.get("key")] = node.id
+    return families
+
+
+def _family_of(decorator_path: str) -> tuple[str, str]:
+    """(registry_class, family_token) from a register decorator path.
+
+    ``StrategyRegistry.register_swing_strategy`` -> ("StrategyRegistry", "swing")
+    ``ZoneDetectionRegistry.register``           -> ("ZoneDetectionRegistry", "")
+    """
+    parts = decorator_path.split(".")
+    reg_class = parts[-2] if len(parts) >= 2 else ""
+    method = parts[-1]
+    token = method
+    for pre in ("register_", "register"):
+        if token.startswith(pre):
+            token = token[len(pre):]
+            break
+    token = token.replace("_strategy", "").replace("strategy", "").strip("_")
+    return reg_class, token
+
+
+class _Recognizer:
+    """Decide which family (if any) a call-site dispatches."""
+
+    def __init__(self, families: dict):
+        self.by_regclass: dict[str, list] = {}
+        self.token_families: list = []  # (token, fid)
+        for fid in families:
+            reg_class, token = fid
+            self.by_regclass.setdefault(reg_class, []).append(fid)
+            if token:
+                self.token_families.append((token, fid))
+
+    def family_of_call(self, call) -> tuple | None:
+        f = call.func
+        # <RegistryClass>.get_x(...) / .get(...) / .create(...)
+        if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name):
+            recv, meth = f.value.id, f.attr
+            if recv in self.by_regclass and _is_dispatch_verb(meth):
+                for fid in self.by_regclass[recv]:
+                    token = fid[1]
+                    if token == "" or token in meth:
+                        return fid
+        # create_x(...) module factory — matched by family token in the name.
+        name = f.id if isinstance(f, ast.Name) else None
+        if name and _is_dispatch_verb(name):
+            for token, fid in self.token_families:
+                if token and token in name:
+                    return fid
+        return None
+
+
+def _is_dispatch_verb(name: str) -> bool:
+    return name.split("_", 1)[0] in _DISPATCH_VERBS
+
+
+# -- ast bridging -------------------------------------------------------------
+
+def _attr_bindings(fnode, recognizer):
+    """Yield (attr, family) for ``self.attr = <dispatch call>`` in a method body."""
+    for node in ast.walk(fnode):
+        if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call):
+            continue
+        fam = recognizer.family_of_call(node.value)
+        if fam is None:
+            continue
+        for tgt in node.targets:
+            if isinstance(tgt, ast.Attribute) and isinstance(tgt.value, ast.Name) \
+                    and tgt.value.id == "self":
+                yield tgt.attr, fam
+
+
+def _bridge_call(call, recognizer, binds, families, graph):
+    """Yield (target_id, resolution) candidate edges for one call-site."""
+    f = call.func
+    # self.attr.method(...) where self.attr is bound to a family -> impl.method
+    if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Attribute) \
+            and isinstance(f.value.value, ast.Name) and f.value.value.id == "self":
+        fam = binds.get(f.value.attr)
+        if fam is not None:
+            method = f.attr
+            for class_id in families[fam]["members"].values():
+                target = f"{class_id}.{method}"
+                if target in graph.nodes:
+                    yield target, "registry-candidate"
+            return
+
+    # direct factory/getter call -> implementation class(es)
+    fam = recognizer.family_of_call(call)
+    if fam is not None:
+        members = families[fam]["members"]
+        key = _literal_key(call)
+        if key is not None and key in members:
+            yield members[key], "registry"  # exact — literal key
+        else:
+            for class_id in members.values():
+                yield class_id, "registry-candidate"
+
+
+def _literal_key(call):
+    """First positional arg if it's a string literal, else None."""
+    if call.args and isinstance(call.args[0], ast.Constant) \
+            and isinstance(call.args[0].value, str):
+        return call.args[0].value
+    return None