coltrace 0.1.0.dev4__py3-none-any.whl

coltrace/journey.py

@@ -0,0 +1,578 @@
+"""Usage-first column lifetimes, kept separate from value ancestry.
+
+An input propagated through a callback summary is not a literal column read.
+Similarly, preserving a column in a filtered frame does not mean that the
+filter predicate read that column. These distinctions belong in the public
+query, rather than being left for a visual renderer to guess.
+"""
+
+from __future__ import annotations
+
+import ast
+from collections import defaultdict, deque
+from dataclasses import asdict
+from typing import Any
+
+from .history import _fallback_events, lifetime_versions
+from .lineage_models import Column, LineageReport
+
+_PASS_KINDS = {"pass", "copy", "filter", "dropna", "projection"}
+_WRITE_KINDS = {"derive", "write", "assign", "apply", "overwrite", "rename", "source"}
+_BOUNDARY_KINDS = {"boundary", "opaque", "opaque_call", "unresolved"}
+_BOOKKEEPING = {"argument", "apply input", "callback input", "string transform"}
+
+
+def _relations(report: LineageReport) -> list[dict[str, Any]]:
+    reads: dict[str, list[str]] = defaultdict(list)
+    writes: dict[str, list[str]] = defaultdict(list)
+    for edge in report.edges:
+        (reads[edge.target] if edge.kind == "read" else writes[edge.source]).append(
+            edge.source if edge.kind == "read" else edge.target
+        )
+    return [
+        {
+            "source": source,
+            "target": target,
+            "kind": operation.kind if operation.kind in {"pass", "rename"} else "derive",
+            "evidence": operation.evidence,
+        }
+        for operation in report.operations
+        for source in reads[operation.id]
+        for target in writes[operation.id]
+    ]
+
+
+def _binding(event: dict[str, Any]) -> str:
+    try:
+        tree = ast.parse(event["expression"])
+    except (SyntaxError, ValueError, RecursionError):
+        return ""
+    if len(tree.body) == 1:
+        statement = tree.body[0]
+        if isinstance(statement, ast.Assign) and isinstance(statement.targets[0], ast.Name):
+            return statement.targets[0].id
+        if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Name):
+            return statement.value.id
+    return ""
+
+
+def _projection_names(event: dict[str, Any]) -> set[str] | None:
+    """Read literal projection syntax; never inspect user data or execute code."""
+    try:
+        tree = ast.parse(event["expression"])
+    except (SyntaxError, ValueError, RecursionError):
+        return None
+    expression: ast.AST | None = None
+    if len(tree.body) == 1 and isinstance(tree.body[0], ast.Expr):
+        expression = tree.body[0].value
+    if isinstance(expression, ast.Subscript) and isinstance(expression.slice, ast.List):
+        labels = expression.slice.elts
+        if all(isinstance(item, ast.Constant) and isinstance(item.value, str) for item in labels):
+            return {
+                item.value
+                for item in labels
+                if isinstance(item, ast.Constant) and isinstance(item.value, str)
+            }
+    return None
+
+
+def _stage(event: dict[str, Any], relationship: str, explanation: str) -> dict[str, Any]:
+    category = {
+        "origin": "source",
+        "application": "transform",
+        "direct_write": "transform",
+        "derivation": "derivation",
+        "frame_pass": "frame",
+        "boundary": "boundary",
+    }.get(relationship, "use")
+    return {
+        **event,
+        "category": category,
+        "relationship": relationship,
+        "explanation": explanation,
+        "event_ids": [event["id"]],
+    }
+
+
+def _contains(outer: dict[str, Any], inner: dict[str, Any]) -> bool:
+    """Check code spans so a different expression on the same line cannot match."""
+    return bool(
+        outer["path"] == inner["path"]
+        and (outer["line"], outer["column"]) <= (inner["line"], inner["column"])
+        and (
+            outer["end_line"] or outer["line"],
+            outer["end_column"] if outer["end_column"] is not None else 10**9,
+        )
+        >= (
+            inner["end_line"] or inner["line"],
+            inner["end_column"] if inner["end_column"] is not None else inner["column"],
+        )
+    )
+
+
+def _compact(steps: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Combine recorder bookkeeping without merging independent value actions."""
+    result: list[dict[str, Any]] = []
+    previous_key: tuple[str, str, int, str, str] | None = None
+    for step in steps:
+        # A direct application/string method already describes its receiver's
+        # read. Keep standalone reads where no richer action exists.
+        if (
+            step["relationship"] == "direct_read"
+            and step["kind"] == "read"
+            and any(
+                (
+                    other["relationship"] in {"application", "direct_write", "derivation"}
+                    or (other["relationship"] == "direct_read" and other["kind"] == "consume")
+                )
+                and other["location"]["path"] == step["location"]["path"]
+                and other["location"]["line"] == step["location"]["line"]
+                and other["context"] == step["context"]
+                and set(other["inputs"]).intersection(step["inputs"])
+                for other in steps
+            )
+        ):
+            continue
+        relation = step["relationship"]
+        key = (
+            step["context"],
+            step["location"]["path"],
+            step["location"]["line"],
+            relation,
+            step["kind"] if relation == "frame_pass" else "",
+        )
+        merge = relation in {"export", "boundary", "frame_pass", "projection_exclusion"}
+        # A call and its later result binding may share one source line while
+        # a whole function body lies between them. Contracting non-adjacent
+        # events can turn that acyclic causal path into a graph cycle.
+        previous = result[-1] if merge and previous_key == key else None
+        if previous is None:
+            result.append(step)
+            previous_key = key if merge else None
+        else:
+            for field in ("inputs", "outputs", "dataframe_ids", "event_ids"):
+                previous[field] = list(dict.fromkeys((*previous[field], *step[field])))
+            if len(step["expression"]) > len(previous["expression"]):
+                previous["label"], previous["expression"] = step["label"], step["expression"]
+    return result
+
+
+def build_journey(
+    report: LineageReport,
+    selected_id: str,
+    included_ids: set[str] | None = None,
+    *,
+    evidence: str = "all",
+    relations: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    """Explain actual recorded uses and destinations of one logical column.
+
+    Families join explicit unchanged passes/renames and versions of one frame's
+    label. They never join unrelated same-name columns. Filtered graph relations
+    are supplied by ``LineageGraph`` so evidence/depth restrictions remain in
+    force. A lifetime can include a constant overwrite without inventing a
+    value-dependency edge between its old and new values.
+    """
+    columns = {column.id: column for column in report.columns}
+    if selected_id not in columns:
+        raise ValueError(f"Unknown column ID: {selected_id}")
+    selected = columns[selected_id]
+    allowed = set(columns if included_ids is None else included_ids)
+    allowed.add(selected_id)
+    allowed.update(
+        identifier
+        for identifier in lifetime_versions(report, selected)
+        if evidence != "confirmed" or columns[identifier].evidence == "confirmed"
+    )
+    allowed.intersection_update(columns)
+    relations = _relations(report) if relations is None else relations
+    relations = [
+        relation
+        for relation in relations
+        if relation["source"] in allowed
+        and relation["target"] in allowed
+        and (evidence != "confirmed" or relation.get("evidence") == "confirmed")
+    ]
+    parent = {identifier: identifier for identifier in allowed}
+
+    def find(identifier: str) -> str:
+        while parent[identifier] != identifier:
+            parent[identifier] = parent[parent[identifier]]
+            identifier = parent[identifier]
+        return identifier
+
+    def union(first: str, second: str) -> None:
+        parent[find(second)] = find(first)
+
+    labels: dict[tuple[str, str], str] = {}
+    for identifier in sorted(allowed):
+        column = columns[identifier]
+        key = (column.dataframe_id, column.name)
+        if key in labels:
+            union(labels[key], identifier)
+        labels[key] = identifier
+    for relation in relations:
+        source, target = relation["source"], relation["target"]
+        if relation["kind"] == "rename" or (
+            relation["kind"] == "pass" and columns[source].name == columns[target].name
+        ):
+            union(source, target)
+    families: dict[str, set[str]] = defaultdict(set)
+    for identifier in allowed:
+        families[find(identifier)].add(identifier)
+    selected_family = find(selected_id)
+    members = families[selected_family]
+    forward: dict[str, set[str]] = defaultdict(set)
+    for relation in relations:
+        source, target = find(relation["source"]), find(relation["target"])
+        if source != target:
+            forward[source].add(target)
+    distances = {selected_family: 0}
+    queue = deque([selected_family])
+    while queue:
+        source = queue.popleft()
+        for target in sorted(forward[source]):
+            if target not in distances:
+                distances[target] = distances[source] + 1
+                queue.append(target)
+    events = (
+        [asdict(event) for event in report.history] if report.history else _fallback_events(report)
+    )
+    events = [
+        event for event in events if evidence != "confirmed" or event["evidence"] == "confirmed"
+    ]
+    events.sort(key=lambda event: (event.get("sequence") is None, event.get("sequence") or 0))
+    frames = {frame.id: frame for frame in report.dataframes}
+
+    def frame_details(family: set[str]) -> list[dict[str, Any]]:
+        details = []
+        for identifier in sorted({columns[item].dataframe_id for item in family}):
+            bindings = [
+                {
+                    "name": name,
+                    "scope": event["scope"],
+                    "context": event["context"],
+                    "location": event["location"],
+                }
+                for event in events
+                if event["kind"] == "bind"
+                and identifier in event["dataframe_ids"]
+                and (name := _binding(event))
+            ]
+            details.append(
+                {
+                    **(asdict(frames[identifier]) if identifier in frames else {"id": identifier}),
+                    "bindings": bindings,
+                    "column_ids": sorted(
+                        item for item in family if columns[item].dataframe_id == identifier
+                    ),
+                    "uses": [
+                        {
+                            key: event[key]
+                            for key in ("kind", "scope", "context", "location", "expression")
+                        }
+                        for event in events
+                        if not event["kind"].startswith("callback_")
+                        and identifier in event["dataframe_ids"]
+                        and family.intersection((*event["inputs"], *event["outputs"]))
+                    ],
+                }
+            )
+        return details
+
+    def active_names(family: set[str], frame_ids: set[str], at: dict[str, Any]) -> set[str]:
+        """A former rename alias may no longer label a column in this frame."""
+        active: dict[str, set[str]] = defaultdict(set)
+        for event in events:
+            if event["id"] == at["id"]:
+                break
+            for identifier in event["outputs"]:
+                if identifier not in columns:
+                    continue
+                column = columns[identifier]
+                if identifier in family:
+                    active[column.dataframe_id].add(column.name)
+                else:
+                    active[column.dataframe_id].discard(column.name)
+            if event["kind"] == "rename":
+                outputs = [columns[item] for item in event["outputs"] if item in columns]
+                for identifier in event["inputs"]:
+                    if identifier in family:
+                        before = columns[identifier]
+                        if any(
+                            after.dataframe_id == before.dataframe_id and after.name != before.name
+                            for after in outputs
+                        ):
+                            active[before.dataframe_id].discard(before.name)
+        return set().union(*(active[identifier] for identifier in frame_ids))
+
+    def family_steps(family: set[str], *, derivations: bool) -> list[dict[str, Any]]:
+        frame_ids = {columns[item].dataframe_id for item in family}
+        creators = [event for event in events if family.intersection(event["outputs"])]
+        start = min((event.get("sequence") or 0 for event in creators), default=0)
+        cutoffs: dict[str, int] = {}
+        result = []
+        for event in events:
+            kind = event["kind"]
+            if kind.startswith("callback_"):
+                continue
+            inputs = family.intersection(event["inputs"])
+            outputs = family.intersection(event["outputs"])
+            related_frames = frame_ids.intersection(event["dataframe_ids"])
+            sequence = event.get("sequence") or 0
+            beyond = any(frame in cutoffs and sequence > cutoffs[frame] for frame in related_frames)
+            if sequence < start:
+                continue
+            if kind == "sink" and related_frames:
+                if inputs and not beyond:
+                    result.append(
+                        _stage(event, "export", "The exported DataFrame contains this column.")
+                    )
+                elif beyond:
+                    stage = _stage(
+                        event,
+                        "export",
+                        "The same DataFrame is exported after an unresolved operation; "
+                        "survival of this column is unverified.",
+                    )
+                    stage.update(
+                        continuity="unresolved",
+                        evidence="unresolved",
+                        completeness="unresolved",
+                        inputs=[],
+                        outputs=[],
+                        parents=[],
+                    )
+                    result.append(stage)
+                continue
+            if kind in _BOUNDARY_KINDS and related_frames:
+                if not beyond:
+                    result.append(
+                        _stage(
+                            event,
+                            "boundary",
+                            "An unresolved operation receives or may mutate a containing "
+                            "DataFrame. Its internal column reads and subsequent survival "
+                            "are unknown.",
+                        )
+                    )
+                    for frame in related_frames:
+                        cutoffs.setdefault(frame, sequence)
+                continue
+            if beyond:
+                continue
+            if outputs:
+                if kind in _PASS_KINDS:
+                    result.append(
+                        _stage(
+                            event,
+                            "frame_pass",
+                            "This pass records preserved column values in the resulting "
+                            "DataFrame; predicate reads are listed separately.",
+                        )
+                    )
+                elif kind in {"external", "parameter"}:
+                    result.append(
+                        _stage(
+                            event,
+                            "origin",
+                            "First recorded observation; the definition in the external "
+                            "source is unknown.",
+                        )
+                    )
+                else:
+                    result.append(
+                        _stage(
+                            event,
+                            "direct_write",
+                            "Creates, overwrites or explicitly renames this column.",
+                        )
+                    )
+            elif kind == "projection" and related_frames:
+                labels = _projection_names(event)
+                names = active_names(family, related_frames, event)
+                if labels is not None and names and not names.intersection(labels):
+                    result.append(
+                        _stage(
+                            event,
+                            "projection_exclusion",
+                            "This projection excludes the column from the resulting DataFrame.",
+                        )
+                    )
+            elif inputs:
+                if kind in _WRITE_KINDS and derivations and event["outputs"]:
+                    if any(
+                        item in allowed and find(item) in distances and distances[find(item)] == 1
+                        for item in event["outputs"]
+                    ):
+                        result.append(
+                            _stage(
+                                event,
+                                "derivation",
+                                "Creates a new column directly derived from this column.",
+                            )
+                        )
+                elif kind == "apply":
+                    result.append(
+                        _stage(
+                            event,
+                            "application",
+                            "Applies the callback directly to this column; internal "
+                            "preparation and control are available in creation context.",
+                        )
+                    )
+                elif kind == "string_transform":
+                    result.append(
+                        _stage(
+                            event,
+                            "application",
+                            "Applies this Series string method directly to the column. "
+                            "A temporary result need not overwrite the stored column.",
+                        )
+                    )
+                elif kind == "read":
+                    result.append(
+                        _stage(event, "direct_read", "Explicit column access in the source code.")
+                    )
+                elif kind == "filter" and not event["outputs"]:
+                    result.append(
+                        _stage(
+                            event,
+                            "direct_read",
+                            "Reads this column to select rows; retained column values "
+                            "are preserved.",
+                        )
+                    )
+                elif kind in {"consume", "call", "return"}:
+                    if event["label"] not in _BOOKKEEPING:
+                        explicit_read = any(
+                            other["kind"] == "read"
+                            and other["context"] == event["context"]
+                            and inputs.intersection(other["inputs"])
+                            and _contains(event["location"], other["location"])
+                            for other in events
+                        )
+                        result.append(
+                            _stage(
+                                event,
+                                "direct_read" if explicit_read else "frame_use",
+                                "Directly consumes an explicitly accessed column; the call "
+                                "may observe values or column metadata."
+                                if explicit_read
+                                else "Uses or passes a containing DataFrame or propagated "
+                                "value; no explicit column access is established at this event.",
+                            )
+                        )
+            elif related_frames and kind in {"bind", "call", "return"}:
+                result.append(
+                    _stage(
+                        event, "frame_pass", "Binds or passes a DataFrame containing the column."
+                    )
+                )
+        return _compact(result)
+
+    def fate(steps: list[dict[str, Any]]) -> dict[str, Any]:
+        terminal = [
+            step
+            for step in steps
+            if step["relationship"] in {"export", "boundary", "projection_exclusion"}
+        ]
+        last = terminal[-1] if terminal else None
+        if last is not None and last["relationship"] == "export":
+            status = "unverified_export" if last.get("continuity") == "unresolved" else "exported"
+            label = (
+                "Exported column survival is unverified"
+                if status == "unverified_export"
+                else "Included in a recorded DataFrame export"
+            )
+        elif last is not None and last["relationship"] == "projection_exclusion":
+            status, label = (
+                "excluded",
+                "Excluded from the resulting DataFrame by an explicit projection",
+            )
+        elif last is not None:
+            status, label = (
+                "boundary",
+                "Tracking reaches an unresolved operation",
+            )
+        elif steps:
+            last, status, label = (
+                steps[-1],
+                "last_observed",
+                "Last recorded use; no later destination was established",
+            )
+        else:
+            return {
+                "status": "last_observed",
+                "label": "No eligible recorded use under these query filters",
+                "location": asdict(selected.location),
+            }
+        return {
+            "status": status,
+            "label": label,
+            "location": last["location"],
+            "event_id": last["id"],
+            "expression": last["expression"],
+            "evidence": last["evidence"],
+            "dataframe_ids": last["dataframe_ids"],
+            "scope": last["scope"],
+            "context": last["context"],
+            "scope_note": "Outcome of this recorded DataFrame branch; earlier "
+            "exports remain listed in destinations.",
+        }
+
+    steps = family_steps(members, derivations=True)
+    derived = []
+    for family_id, distance in sorted(distances.items(), key=lambda item: (item[1], item[0])):
+        if family_id == selected_family:
+            continue
+        family = families[family_id]
+        definitions = [
+            event
+            for event in events
+            if family.intersection(event["outputs"]) and event["kind"] not in _PASS_KINDS
+        ]
+        representative: Column = columns[min(family)]
+        if definitions:
+            representative = columns[
+                next(item for item in definitions[0]["outputs"] if item in family)
+            ]
+        uses = family_steps(family, derivations=False)
+        derived.append(
+            {
+                "id": representative.id,
+                "name": representative.name,
+                "column_ids": sorted(family),
+                "distance": distance,
+                "relationship": "direct" if distance == 1 else "transitive",
+                "definition": definitions[0] if definitions else None,
+                "upstream": [
+                    asdict(columns[item])
+                    for item in (definitions[0]["inputs"] if definitions else ())
+                    if item in columns
+                ],
+                "fate": fate(uses),
+                "dataframes": frame_details(family),
+                "steps": uses,
+            }
+        )
+    return {
+        "selected_column_id": selected_id,
+        "selected_name": selected.name,
+        "column_ids": sorted(members),
+        "aliases": sorted({columns[item].name for item in members}),
+        "steps": steps,
+        "derived_columns": derived,
+        "dataframes": frame_details(members),
+        "destinations": [step for step in steps if step["relationship"] == "export"],
+        "boundaries": [step for step in steps if step["relationship"] == "boundary"],
+        "fate": fate(steps),
+        "counts": {
+            "direct_reads": sum(step["relationship"] == "direct_read" for step in steps),
+            "applications": sum(step["relationship"] == "application" for step in steps),
+            "writes": sum(step["relationship"] == "direct_write" for step in steps),
+            "derived_columns": len(derived),
+            "dataframes": len(frame_details(members)),
+        },
+        "recorded": bool(report.history),
+    }

coltrace/lineage_models.py

@@ -0,0 +1,112 @@
+"""Public, JSON-serializable lineage facts. IDs are opaque and scan-local."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+from .models import Diagnostic, FileInfo, FunctionInfo, Location
+
+Confidence = Literal["confirmed", "inferred", "unresolved"]
+Completeness = Literal["complete", "partial", "unresolved"]
+
+
+@dataclass(frozen=True)
+class Frame:
+    id: str
+    origin: str
+    location: Location
+    module: str
+    scope: str
+    context: str
+    evidence: Confidence = "confirmed"
+
+
+@dataclass(frozen=True)
+class Column:
+    id: str
+    dataframe_id: str
+    name: str
+    version: int
+    location: Location
+    module: str
+    scope: str
+    context: str
+    origin: str = "write"
+    evidence: Confidence = "confirmed"
+    completeness: Completeness = "complete"
+
+
+@dataclass(frozen=True)
+class Operation:
+    id: str
+    kind: str
+    location: Location
+    module: str
+    scope: str
+    context: str
+    evidence: Confidence = "confirmed"
+    completeness: Completeness = "complete"
+
+
+@dataclass(frozen=True)
+class Edge:
+    source: str
+    target: str
+    kind: Literal["read", "write"]
+    evidence: Confidence = "confirmed"
+
+
+@dataclass(frozen=True)
+class Consumption:
+    column_id: str
+    location: Location
+    module: str
+    scope: str
+    context: str
+    kind: str = "read"
+
+
+@dataclass(frozen=True)
+class HistoryEvent:
+    """A recorded analysis step; ordering describes analysis, not a runtime log."""
+
+    id: str
+    sequence: int
+    kind: str
+    label: str
+    location: Location
+    module: str
+    scope: str
+    context: str
+    dataframe_ids: tuple[str, ...] = ()
+    inputs: tuple[str, ...] = ()
+    outputs: tuple[str, ...] = ()
+    expression: str = ""
+    evidence: Confidence = "confirmed"
+    completeness: Completeness = "complete"
+    parents: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class LineageReport:
+    files: tuple[FileInfo, ...] = ()
+    functions: tuple[FunctionInfo, ...] = ()
+    dataframes: tuple[Frame, ...] = ()
+    columns: tuple[Column, ...] = ()
+    operations: tuple[Operation, ...] = ()
+    edges: tuple[Edge, ...] = ()
+    consumptions: tuple[Consumption, ...] = ()
+    diagnostics: tuple[Diagnostic, ...] = ()
+    exclude: tuple[str, ...] = ()
+    history: tuple[HistoryEvent, ...] = ()
+    schema_version: str = "0.3"
+    report_kind: str = "column_lineage"
+    tool_version: str = "0.1.0.dev4"
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def to_json(self, *, indent: int | None = 2) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, indent=indent)