astscribe 0.8.2__py3-none-any.whl

astscribe/__init__.py

@@ -0,0 +1,37 @@
+from .api import analyze, explain
+from .dependency import (
+    CellDependencyEdge,
+    CellDependencyNode,
+    NotebookDependencyGraph,
+    SymbolRedefinition,
+)
+from .diagnostics import NotebookDiagnostic, NotebookDiagnostics
+from .impact import CellImpactSummary, ImpactHop, ImpactPath, NotebookImpactReport
+from .methodology import MethodologyReport, MethodologySection
+from .notebook import NotebookAnalyzer, SkippedCell
+from .pipeline import ExperimentPipeline, PipelineStage
+from .sir import AnalysisResult
+
+__version__ = "0.8.2"
+
+__all__ = [
+    "AnalysisResult",
+    "CellDependencyEdge",
+    "CellDependencyNode",
+    "CellImpactSummary",
+    "ExperimentPipeline",
+    "ImpactHop",
+    "ImpactPath",
+    "MethodologyReport",
+    "MethodologySection",
+    "NotebookAnalyzer",
+    "NotebookDependencyGraph",
+    "NotebookDiagnostic",
+    "NotebookDiagnostics",
+    "NotebookImpactReport",
+    "PipelineStage",
+    "SkippedCell",
+    "SymbolRedefinition",
+    "analyze",
+    "explain",
+]

astscribe/__main__.py

@@ -0,0 +1,3 @@
+from astscribe.cli import main
+
+raise SystemExit(main())

astscribe/api.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from astscribe.parser import (
+    ImportTable,
+    ParsedSource,
+    SymbolTable,
+    build_import_table,
+    build_symbol_table,
+    parse_source,
+)
+from astscribe.patterns import detect_inference, detect_training_step
+from astscribe.semantics import DEFAULT_REGISTRY
+from astscribe.sir import AnalysisResult
+
+
+def _analyze_parsed(
+    parsed: ParsedSource,
+    imports: ImportTable,
+    symbols: SymbolTable,
+) -> AnalysisResult:
+    result = AnalysisResult(source=parsed.source)
+    for framework, analyzers in DEFAULT_REGISTRY.analyzers().items():
+        framework_detected = False
+        for analyzer in analyzers:
+            semantic = analyzer(parsed, imports, symbols)
+            if semantic.operations or semantic.claims:
+                framework_detected = True
+                result.operations.extend(semantic.operations)
+                result.claims.extend(semantic.claims)
+        if framework_detected:
+            result.frameworks.append(framework)
+
+    result.training_step = detect_training_step(result.operations)
+    result.inference = detect_inference(result.operations)
+    return result
+
+
+def analyze(source: str) -> AnalysisResult:
+    parsed = parse_source(source)
+    imports = build_import_table(parsed.tree)
+    symbols = build_symbol_table(parsed.tree, imports, source=source)
+    return _analyze_parsed(parsed, imports, symbols)
+
+
+def explain(
+    source: str,
+    style: str = "scientific",
+    *,
+    return_result: bool = False,
+) -> str | AnalysisResult:
+    result = analyze(source)
+    return result if return_result else result.render(style)

astscribe/cli.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from astscribe import NotebookAnalyzer, __version__, explain
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="astscribe",
+        description="Explain Python ML code and inspect Jupyter notebooks without executing them.",
+    )
+    parser.add_argument(
+        "path",
+        type=Path,
+        help="Python source file, Jupyter notebook, or '-' to read Python source from stdin",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument(
+        "--style",
+        choices=("scientific", "educational", "concise"),
+        default="scientific",
+        help="rendering style for Python source (default: scientific)",
+    )
+    parser.add_argument(
+        "--report",
+        choices=("methodology", "pipeline", "techniques", "dependencies", "diagnostics", "impact"),
+        default="methodology",
+        help="notebook report to render (default: methodology)",
+    )
+    parser.add_argument("--cell", type=int, help="notebook cell for an impact report")
+    parser.add_argument(
+        "--evidence",
+        action="store_true",
+        help="include source evidence in methodology reports",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="fail if a notebook code cell is not valid Python instead of skipping it",
+    )
+    return parser
+
+
+def _render_notebook(args: argparse.Namespace) -> str:
+    notebook = NotebookAnalyzer.from_ipynb(
+        args.path,
+        skip_invalid_python=not args.strict,
+    )
+    if args.report == "methodology":
+        return notebook.render_methodology(include_evidence=args.evidence)
+    if args.report == "pipeline":
+        return notebook.render_pipeline()
+    if args.report == "techniques":
+        return notebook.render_techniques()
+    if args.report == "dependencies":
+        return notebook.render_dependency_graph()
+    if args.report == "diagnostics":
+        return notebook.render_diagnostics()
+    if args.cell is None:
+        raise ValueError("--cell is required when --report impact is selected")
+    return notebook.render_impact(args.cell)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = _parser()
+    args = parser.parse_args(argv)
+
+    try:
+        if args.path.suffix.lower() == ".ipynb":
+            output = _render_notebook(args)
+        else:
+            source = sys.stdin.read() if str(args.path) == "-" else args.path.read_text(encoding="utf-8")
+            output = str(explain(source, style=args.style))
+    except (OSError, ValueError, SyntaxError) as exc:
+        parser.error(str(exc))
+
+    sys.stdout.write(output)
+    if output and not output.endswith("\n"):
+        sys.stdout.write("\n")
+    return 0

astscribe/dependency.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+import ast
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+EventKind = Literal["read", "write", "delete"]
+_BUILTIN_NAMES = frozenset(dir(__import__("builtins"))) | {
+    "__name__",
+    "__file__",
+    "__package__",
+}
+
+
+@dataclass(frozen=True)
+class SymbolEvent:
+    kind: EventKind
+    symbol: str
+    line: int | None
+    column: int | None
+
+
+@dataclass(frozen=True)
+class CellDependencyNode:
+    cell: int
+    defines: tuple[str, ...]
+    reads: tuple[str, ...]
+    unresolved_reads: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class CellDependencyEdge:
+    producer_cell: int
+    consumer_cell: int
+    symbols: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class SymbolRedefinition:
+    symbol: str
+    previous_cell: int
+    new_cell: int
+
+
+@dataclass(frozen=True)
+class NotebookDependencyGraph:
+    nodes: tuple[CellDependencyNode, ...]
+    edges: tuple[CellDependencyEdge, ...]
+    redefinitions: tuple[SymbolRedefinition, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def parents(self, cell: int) -> tuple[int, ...]:
+        return tuple(
+            sorted(
+                {
+                    edge.producer_cell
+                    for edge in self.edges
+                    if edge.consumer_cell == cell
+                }
+            )
+        )
+
+    def children(self, cell: int) -> tuple[int, ...]:
+        return tuple(
+            sorted(
+                {
+                    edge.consumer_cell
+                    for edge in self.edges
+                    if edge.producer_cell == cell
+                }
+            )
+        )
+
+    def render(self) -> str:
+        if not self.nodes:
+            return "No analyzable notebook cells are available."
+        if not self.edges:
+            return "No cross-cell symbol dependencies were detected."
+
+        lines = ["# Cell dependency graph", ""]
+        for edge in self.edges:
+            symbols = ", ".join(edge.symbols)
+            lines.append(
+                f"Cell {edge.producer_cell} -> Cell {edge.consumer_cell} [{symbols}]"
+            )
+        return "\n".join(lines)
+
+    def to_dot(self) -> str:
+        lines = ["digraph ASTScribeNotebook {", "  rankdir=LR;"]
+        for node in self.nodes:
+            lines.append(f'  c{node.cell} [label="Cell {node.cell}"];')
+        for edge in self.edges:
+            symbols = ", ".join(edge.symbols).replace('"', '\\"')
+            lines.append(
+                f'  c{edge.producer_cell} -> c{edge.consumer_cell} [label="{symbols}"];'
+            )
+        lines.append("}")
+        return "\n".join(lines)
+
+
+class _EventCollector(ast.NodeVisitor):
+    def __init__(self) -> None:
+        self.events: list[SymbolEvent] = []
+        self._locals: list[set[str]] = []
+
+    def _emit(self, kind: EventKind, symbol: str, node: ast.AST) -> None:
+        self.events.append(
+            SymbolEvent(
+                kind=kind,
+                symbol=symbol,
+                line=getattr(node, "lineno", None),
+                column=getattr(node, "col_offset", None),
+            )
+        )
+
+    def _is_local(self, name: str) -> bool:
+        return any(name in scope for scope in reversed(self._locals))
+
+    def _write_target(self, node: ast.AST) -> None:
+        if isinstance(node, ast.Name):
+            if not self._locals:
+                self._emit("write", node.id, node)
+            else:
+                self._locals[-1].add(node.id)
+            return
+        if isinstance(node, ast.Tuple | ast.List):
+            for element in node.elts:
+                self._write_target(element)
+            return
+        if isinstance(node, ast.Starred):
+            self._write_target(node.value)
+            return
+        # Attribute/subscript assignment reads the base/index expressions but
+        # does not create a new notebook-global symbol.
+        self.visit(node)
+
+    def _delete_target(self, node: ast.AST) -> None:
+        if isinstance(node, ast.Name):
+            if not self._is_local(node.id):
+                self._emit("delete", node.id, node)
+            return
+        if isinstance(node, ast.Tuple | ast.List):
+            for element in node.elts:
+                self._delete_target(element)
+            return
+        self.visit(node)
+
+    def _visit_function_definition(
+        self,
+        node: ast.FunctionDef | ast.AsyncFunctionDef,
+    ) -> None:
+        for decorator in node.decorator_list:
+            self.visit(decorator)
+        for default in (*node.args.defaults, *node.args.kw_defaults):
+            if default is not None:
+                self.visit(default)
+        self._emit("write", node.name, node)
+
+    def _visit_for(self, node: ast.For | ast.AsyncFor) -> None:
+        self.visit(node.iter)
+        self._write_target(node.target)
+        for statement in node.body:
+            self.visit(statement)
+        for statement in node.orelse:
+            self.visit(statement)
+
+    def _visit_with(self, node: ast.With | ast.AsyncWith) -> None:
+        for item in node.items:
+            self.visit(item.context_expr)
+            if item.optional_vars is not None:
+                self._write_target(item.optional_vars)
+        for statement in node.body:
+            self.visit(statement)
+
+    def visit_Name(self, node: ast.Name) -> None:
+        if isinstance(node.ctx, ast.Load) and not self._is_local(node.id):
+            self._emit("read", node.id, node)
+        elif isinstance(node.ctx, ast.Store):
+            self._write_target(node)
+
+    def visit_Assign(self, node: ast.Assign) -> None:
+        self.visit(node.value)
+        for target in node.targets:
+            self._write_target(target)
+
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
+        if node.value is not None:
+            self.visit(node.value)
+        self._write_target(node.target)
+
+    def visit_AugAssign(self, node: ast.AugAssign) -> None:
+        if isinstance(node.target, ast.Name) and not self._is_local(node.target.id):
+            self._emit("read", node.target.id, node.target)
+        else:
+            self.visit(node.target)
+        self.visit(node.value)
+        self._write_target(node.target)
+
+    def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
+        self.visit(node.value)
+        self._write_target(node.target)
+
+    def visit_Delete(self, node: ast.Delete) -> None:
+        for target in node.targets:
+            self._delete_target(target)
+
+    def visit_Import(self, node: ast.Import) -> None:
+        for alias in node.names:
+            name = alias.asname or alias.name.split(".", 1)[0]
+            self._emit("write", name, node)
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+        for alias in node.names:
+            if alias.name == "*":
+                continue
+            self._emit("write", alias.asname or alias.name, node)
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+        self._visit_function_definition(node)
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
+        self._visit_function_definition(node)
+
+    def visit_Lambda(self, node: ast.Lambda) -> None:
+        for default in (*node.args.defaults, *node.args.kw_defaults):
+            if default is not None:
+                self.visit(default)
+
+    def visit_ClassDef(self, node: ast.ClassDef) -> None:
+        for decorator in node.decorator_list:
+            self.visit(decorator)
+        for base in node.bases:
+            self.visit(base)
+        for keyword in node.keywords:
+            self.visit(keyword.value)
+        self._emit("write", node.name, node)
+
+    def visit_For(self, node: ast.For) -> None:
+        self._visit_for(node)
+
+    def visit_AsyncFor(self, node: ast.AsyncFor) -> None:
+        self._visit_for(node)
+
+    def visit_With(self, node: ast.With) -> None:
+        self._visit_with(node)
+
+    def visit_AsyncWith(self, node: ast.AsyncWith) -> None:
+        self._visit_with(node)
+
+    def _visit_comprehension(
+        self,
+        generators: list[ast.comprehension],
+        final_nodes: tuple[ast.AST, ...],
+    ) -> None:
+        self._locals.append(set())
+        try:
+            for generator in generators:
+                self.visit(generator.iter)
+                self._write_target(generator.target)
+                for condition in generator.ifs:
+                    self.visit(condition)
+            for node in final_nodes:
+                self.visit(node)
+        finally:
+            self._locals.pop()
+
+    def visit_ListComp(self, node: ast.ListComp) -> None:
+        self._visit_comprehension(node.generators, (node.elt,))
+
+    def visit_SetComp(self, node: ast.SetComp) -> None:
+        self._visit_comprehension(node.generators, (node.elt,))
+
+    def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None:
+        self._visit_comprehension(node.generators, (node.elt,))
+
+    def visit_DictComp(self, node: ast.DictComp) -> None:
+        self._visit_comprehension(node.generators, (node.key, node.value))
+
+
+def collect_symbol_events(source: str) -> tuple[SymbolEvent, ...]:
+    tree = ast.parse(source)
+    collector = _EventCollector()
+    collector.visit(tree)
+    return tuple(collector.events)
+
+
+def build_dependency_graph(
+    cells: tuple[tuple[int, str], ...],
+) -> NotebookDependencyGraph:
+    producer: dict[str, int] = {}
+    nodes: list[CellDependencyNode] = []
+    edge_symbols: dict[tuple[int, int], set[str]] = {}
+    redefinitions: list[SymbolRedefinition] = []
+
+    for cell, source in cells:
+        events = collect_symbol_events(source)
+        defines: set[str] = set()
+        reads: set[str] = set()
+        unresolved: set[str] = set()
+
+        for event in events:
+            if event.kind == "read":
+                reads.add(event.symbol)
+                source_cell = producer.get(event.symbol)
+                if source_cell is None:
+                    if event.symbol not in _BUILTIN_NAMES:
+                        unresolved.add(event.symbol)
+                elif source_cell != cell:
+                    edge_symbols.setdefault((source_cell, cell), set()).add(event.symbol)
+                continue
+
+            if event.kind == "delete":
+                producer.pop(event.symbol, None)
+                continue
+
+            defines.add(event.symbol)
+            previous_cell = producer.get(event.symbol)
+            if previous_cell is not None and previous_cell != cell:
+                redefinitions.append(
+                    SymbolRedefinition(
+                        symbol=event.symbol,
+                        previous_cell=previous_cell,
+                        new_cell=cell,
+                    )
+                )
+            producer[event.symbol] = cell
+
+        nodes.append(
+            CellDependencyNode(
+                cell=cell,
+                defines=tuple(sorted(defines)),
+                reads=tuple(sorted(reads)),
+                unresolved_reads=tuple(sorted(unresolved)),
+            )
+        )
+
+    edges = tuple(
+        CellDependencyEdge(
+            producer_cell=producer_cell,
+            consumer_cell=consumer_cell,
+            symbols=tuple(sorted(symbols)),
+        )
+        for (producer_cell, consumer_cell), symbols in sorted(edge_symbols.items())
+    )
+    return NotebookDependencyGraph(
+        nodes=tuple(nodes),
+        edges=edges,
+        redefinitions=tuple(redefinitions),
+    )

astscribe/diagnostics.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+from astscribe.dependency import NotebookDependencyGraph
+
+DiagnosticSeverity = Literal["info", "warning"]
+
+
+@dataclass(frozen=True)
+class NotebookDiagnostic:
+    code: str
+    severity: DiagnosticSeverity
+    message: str
+    cell: int
+    symbol: str
+    related_cell: int | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class NotebookDiagnostics:
+    items: tuple[NotebookDiagnostic, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"items": [item.to_dict() for item in self.items]}
+
+    def by_code(self, code: str) -> tuple[NotebookDiagnostic, ...]:
+        return tuple(item for item in self.items if item.code == code)
+
+    def render(self) -> str:
+        if not self.items:
+            return "No supported notebook dependency diagnostics were detected."
+
+        lines = ["# Notebook diagnostics", ""]
+        for item in self.items:
+            label = item.severity.upper()
+            lines.append(f"- [{label}] cell {item.cell}: {item.message} (`{item.code}`)")
+        return "\n".join(lines)
+
+
+def _definitions_by_symbol(graph: NotebookDependencyGraph) -> dict[str, tuple[int, ...]]:
+    cells: dict[str, list[int]] = {}
+    for node in graph.nodes:
+        for symbol in node.defines:
+            cells.setdefault(symbol, []).append(node.cell)
+    return {symbol: tuple(values) for symbol, values in cells.items()}
+
+
+def _next_definition(
+    definitions: dict[str, tuple[int, ...]],
+    symbol: str,
+    cell: int,
+) -> int | None:
+    return next(
+        (definition_cell for definition_cell in definitions.get(symbol, ()) if definition_cell > cell),
+        None,
+    )
+
+
+def _definition_consumed_before_redefinition(
+    graph: NotebookDependencyGraph,
+    symbol: str,
+    producer_cell: int,
+    redefinition_cell: int,
+) -> bool:
+    return any(
+        edge.producer_cell == producer_cell
+        and edge.consumer_cell <= redefinition_cell
+        and symbol in edge.symbols
+        for edge in graph.edges
+    )
+
+
+def build_notebook_diagnostics(graph: NotebookDependencyGraph) -> NotebookDiagnostics:
+    definitions = _definitions_by_symbol(graph)
+    diagnostics: list[NotebookDiagnostic] = []
+
+    for node in graph.nodes:
+        for symbol in node.unresolved_reads:
+            later_cell = _next_definition(definitions, symbol, node.cell)
+            if later_cell is not None:
+                diagnostics.append(
+                    NotebookDiagnostic(
+                        code="dependency.forward_reference",
+                        severity="warning",
+                        message=(
+                            f"`{symbol}` is read before its next supported definition in "
+                            f"cell {later_cell}; source-order analysis cannot resolve this read."
+                        ),
+                        cell=node.cell,
+                        symbol=symbol,
+                        related_cell=later_cell,
+                    )
+                )
+            else:
+                diagnostics.append(
+                    NotebookDiagnostic(
+                        code="dependency.unresolved_symbol",
+                        severity="warning",
+                        message=(
+                            f"`{symbol}` is read without a supported prior notebook definition; "
+                            "it may come from external or hidden kernel state."
+                        ),
+                        cell=node.cell,
+                        symbol=symbol,
+                    )
+                )
+
+    for redefinition in graph.redefinitions:
+        diagnostics.append(
+            NotebookDiagnostic(
+                code="dependency.symbol_redefinition",
+                severity="info",
+                message=(
+                    f"`{redefinition.symbol}` replaces the definition from cell "
+                    f"{redefinition.previous_cell}."
+                ),
+                cell=redefinition.new_cell,
+                symbol=redefinition.symbol,
+                related_cell=redefinition.previous_cell,
+            )
+        )
+
+        if not _definition_consumed_before_redefinition(
+            graph,
+            redefinition.symbol,
+            redefinition.previous_cell,
+            redefinition.new_cell,
+        ):
+            diagnostics.append(
+                NotebookDiagnostic(
+                    code="dependency.overwritten_before_cross_cell_use",
+                    severity="info",
+                    message=(
+                        f"The definition of `{redefinition.symbol}` in cell "
+                        f"{redefinition.previous_cell} is replaced in cell "
+                        f"{redefinition.new_cell} before any later analyzed cell consumes it."
+                    ),
+                    cell=redefinition.new_cell,
+                    symbol=redefinition.symbol,
+                    related_cell=redefinition.previous_cell,
+                )
+            )
+
+    diagnostics.sort(
+        key=lambda item: (
+            item.cell,
+            item.code,
+            item.symbol,
+            item.related_cell if item.related_cell is not None else -1,
+        )
+    )
+    return NotebookDiagnostics(items=tuple(diagnostics))