cellgraph 0.1.0__tar.gz → 0.3.0__tar.gz

{cellgraph-0.1.0 → cellgraph-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellgraph
-Version: 0.1.0
+Version: 0.3.0
 Summary: The cell graph of a notebook or script: annotations with a host-supplied vocabulary, analysis, the DAG, and the keys that say whether a cell must run.
 Author: Fangchen Li
 Author-email: Fangchen Li <fangchen.li@outlook.com>

{cellgraph-0.1.0 → cellgraph-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cellgraph"
-version = "0.1.0"
+version = "0.3.0"
 description = "The cell graph of a notebook or script: annotations with a host-supplied vocabulary, analysis, the DAG, and the keys that say whether a cell must run."
 readme = "README.md"
 requires-python = ">=3.12"

{cellgraph-0.1.0 → cellgraph-0.3.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "cellgraph"
-version = "0.1.0"
+version = "0.3.0"
 description = "The cell graph of a notebook or script: annotations with a host-supplied vocabulary, analysis, the DAG, and the keys that say whether a cell must run."
 readme = "README.md"
 requires-python = ">=3.12"

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/__init__.py

@@ -40,13 +40,16 @@ from cellgraph.grammar import GRAMMAR, Directive, Grammar
 from cellgraph.keys import (
     PARTS_SCHEMA,
     HostKeyed,
+    Item,
     Keyed,
     OutputRef,
     Provenance,
     Published,
     Resolved,
+    Resolver,
     Run,
     Unavailable,
+    VariantRead,
     canonical_json,
     compute_provenance_hash,
     derive_subkey,
@@ -103,6 +106,7 @@ __all__ = [
     "GroupConfig",
     "HostKeyed",
     "Input",
+    "Item",
     "Keeps",
     "Keyed",
     "LanguageAnalyzer",
@@ -117,11 +121,13 @@ __all__ = [
     "PythonAnalyzer",
     "Reads",
     "Resolved",
+    "Resolver",
     "Run",
     "Seeds",
     "Severity",
     "SweepProducer",
     "Unavailable",
+    "VariantRead",
     "UnknownLanguageError",
     "ValidationContext",
     "VariantGroupResolution",

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/analysis.py

@@ -136,6 +136,8 @@ class Dag:
     inactive_cells: set[str] = field(default_factory=set)
     wired_inputs: dict[str, dict[str, set[str]]] = field(default_factory=dict)
     fanout_cells: dict[str, SweepProducer] = field(default_factory=dict)
+    # An ``@each`` cell: the name its items come from. Not a sweep: a reader sees one output.
+    each_cells: dict[str, str] = field(default_factory=dict)
     read_producers: dict[str, dict[str, str | SweepProducer]] = field(default_factory=dict)
 
     def wired_variables(self, consumer_id: str, producer_id: str) -> set[str]:
@@ -275,6 +277,8 @@ class CellAnalysis:
     annotations: dict[str, Any] = field(default_factory=dict)
     directives: list[Directive] = field(default_factory=list)
     effects: list[Effect] = field(default_factory=list)
+    # Per effect: the annotation key and line it came from; None for the host's hook.
+    origins: list[tuple[str | None, int | None]] = field(default_factory=list)
     defines: list[str] = field(default_factory=list)
     references: list[str] = field(default_factory=list)
     builtin_references: list[str] = field(default_factory=list)
@@ -282,6 +286,7 @@ class CellAnalysis:
     bound: list[str] = field(default_factory=list)
     after: list[str] = field(default_factory=list)
     seeds: dict[str, str] = field(default_factory=dict)
+    seed_origins: dict[str, tuple[str | None, int | None]] = field(default_factory=dict)
     member: Member | None = None
     fanout: FanOut | None = None
     kept: list[str] = field(default_factory=list)
@@ -292,12 +297,21 @@ class CellAnalysis:
     plannable: bool = True
     # The cells whose own errors keep this one from planning (itself included).
     blocked_by: list[str] = field(default_factory=list)
+    # Set by ``analyze`` for every cell, plannable or not.
     definition_key: str | None = None
     normalized: str | None = None
 
-    def error(self, code: str, message: str, *, annotation: str | None = None, line=None):
+    def error(
+        self,
+        code: str,
+        message: str,
+        *,
+        annotation: str | None = None,
+        line=None,
+        args: Mapping[str, Any] | None = None,
+    ):
         self.diagnostics.append(
-            Diagnostic(code, Severity.ERROR, message, self.id, annotation, line)
+            Diagnostic(code, Severity.ERROR, message, self.id, annotation, line, dict(args or {}))
         )
 
     def note(
@@ -308,8 +322,11 @@ class CellAnalysis:
         *,
         annotation: str | None = None,
         line=None,
+        args: Mapping[str, Any] | None = None,
     ):
-        self.diagnostics.append(Diagnostic(code, severity, message, self.id, annotation, line))
+        self.diagnostics.append(
+            Diagnostic(code, severity, message, self.id, annotation, line, dict(args or {}))
+        )
 
     @property
     def has_error(self) -> bool:
@@ -354,6 +371,14 @@ def _first_line(directives: list[Directive], key: str) -> int | None:
     return next((d.line for d in directives if d.key == key), None)
 
 
+def _naming_line(directives: list[Directive], key: str, token: str) -> int | None:
+    """The first *key* line with *token* among its values; else the first *key* line."""
+    for d in directives:
+        if d.key == key and token in (t.rstrip(",") for t in d.value.split()):
+            return d.line
+    return _first_line(directives, key)
+
+
 def _top_level_imports(source: str) -> set[str]:
     try:
         tree = ast.parse(source)
@@ -389,6 +414,7 @@ def _parse_phase(
                 f"`@{directive.key}` is not an annotation this vocabulary knows.{hint}",
                 annotation=directive.key,
                 line=directive.line,
+                args={"key": directive.key, "suggestion": close[0] if close else None},
             )
             continue
         by_owner.setdefault(owner.key, []).append(directive)
@@ -427,14 +453,19 @@ def _parse_phase(
                     f"({type(encoded).__name__}); the annotation needs an encoder",
                     annotation=annotation.key,
                     line=directives[0].line,
+                    args={"type": type(encoded).__name__},
                 )
             else:
                 ca.key_fields[annotation.key] = encoded
         if annotation.effects is not None:
-            ca.effects.extend(annotation.effects(value))
+            produced = list(annotation.effects(value))
+            ca.effects.extend(produced)
+            ca.origins.extend([(annotation.key, directives[0].line)] * len(produced))
 
     if vocabulary.cell_effects is not None:
-        ca.effects.extend(vocabulary.cell_effects(cell.meta, dict(ca.annotations)))
+        hooked = list(vocabulary.cell_effects(cell.meta, dict(ca.annotations)))
+        ca.effects.extend(hooked)
+        ca.origins.extend([(None, None)] * len(hooked))
     for effect in ca.effects:
         if not isinstance(effect, EFFECT_TYPES):
             raise TypeError(f"{effect!r} is not one of the ten effects")
@@ -472,11 +503,29 @@ def _names_phase(ca: CellAnalysis, analyzed: Analyzed) -> None:
         if isinstance(effect, Binds):
             for name in effect.names:
                 if name in bound:
-                    ca.error("bind_conflict", f"`{name}` is bound twice")
+                    ca.error("bind_conflict", f"`{name}` is bound twice", args={"name": name})
                 else:
                     bound.append(name)
     for name in sorted(set(bound) & set(read)):
-        ca.error("bind_read_conflict", f"`{name}` is both bound and read from upstream")
+        ca.error(
+            "bind_read_conflict",
+            f"`{name}` is both bound and read from upstream",
+            args={"name": name},
+        )
+
+    for effect in ca.effects:
+        if isinstance(effect, FanOut) and effect.over is not None:
+            # An instance's key leaves the source out, so the body must not read it whole.
+            if effect.over in analyzed.references:
+                ca.error(
+                    "each_reads_source",
+                    f"the cell reads `{effect.over}` itself, which every item's result would "
+                    "then depend on; read each item, or compute it in a cell before this one",
+                    annotation="each",
+                    args={"source": effect.over},
+                )
+            elif effect.over not in bound and effect.over not in references:
+                references.append(effect.over)
 
     ca.defines = defines
     ca.references = [name for name in references if name not in bound]
@@ -484,32 +533,59 @@ def _names_phase(ca: CellAnalysis, analyzed: Analyzed) -> None:
     ca.mutation_defines = list(analyzed.mutation_defines)
     ca.bound = bound
 
-    for effect in ca.effects:
+    for effect, (origin, origin_line) in zip(ca.effects, ca.origins, strict=True):
         if isinstance(effect, After):
             for cell_id in effect.cell_ids:
                 if cell_id not in ca.after:
                     ca.after.append(cell_id)
         elif isinstance(effect, Seeds):
             if effect.name in ca.seeds and ca.seeds[effect.name] != effect.cell_id:
-                ca.error("seed_conflict", f"`{effect.name}` is seeded from two cells")
+                ca.error(
+                    "seed_conflict",
+                    f"`{effect.name}` is seeded from two cells",
+                    annotation=origin,
+                    line=origin_line,
+                    args={"name": effect.name},
+                )
             else:
                 ca.seeds[effect.name] = effect.cell_id
+                ca.seed_origins[effect.name] = (origin, origin_line)
             if effect.name in bound:
-                ca.error("bind_read_conflict", f"`{effect.name}` is both bound and seeded")
+                ca.error(
+                    "bind_read_conflict",
+                    f"`{effect.name}` is both bound and seeded",
+                    annotation=origin,
+                    line=origin_line,
+                    args={"name": effect.name},
+                )
         elif isinstance(effect, Member):
             if ca.member is not None and ca.member != effect:
-                ca.error("member_conflict", "the cell is a member of two variants")
+                ca.error(
+                    "member_conflict",
+                    "the cell is a member of two variants",
+                    annotation=origin,
+                    line=origin_line,
+                )
             ca.member = effect
         elif isinstance(effect, FanOut):
             if ca.fanout is not None and ca.fanout != effect:
-                ca.error("fanout_conflict", "the cell fans out over two groups")
+                ca.error(
+                    "fanout_conflict",
+                    "the cell fans out two ways",
+                    annotation=origin,
+                    line=origin_line,
+                )
             ca.fanout = effect
         elif isinstance(effect, Keeps):
             ca.kept.extend(name for name in effect.names if name not in ca.kept)
         elif isinstance(effect, Input):
             if any((i.kind, i.name) == (effect.kind, effect.name) for i in ca.inputs):
                 ca.error(
-                    "input_conflict", f"two {effect.kind} inputs have the address {effect.name!r}"
+                    "input_conflict",
+                    f"two {effect.kind} inputs have the address {effect.name!r}",
+                    annotation=origin,
+                    line=origin_line,
+                    args={"kind": effect.kind, "name": effect.name},
                 )
             else:
                 ca.inputs.append(effect)
@@ -537,7 +613,12 @@ def _resolve_groups(
                 f"{cell.id[:8]} in group '{group}'"
             )
             for cid in (clash, cell.id):
-                by_id[cid].error("variant_collision", message, annotation="variant")
+                by_id[cid].error(
+                    "variant_collision",
+                    message,
+                    annotation="variant",
+                    args={"group": group, "name": name, "cells": [clash, cell.id]},
+                )
             continue
         if not members:
             group_order.append(group)
@@ -591,6 +672,13 @@ def _variant_notes(cells: list[CellAnalysis], dag: Dag, groups: GroupConfig) ->
                     f"defines {sorted(theirs)}",
                     annotation="variant",
                     line=line,
+                    args={
+                        "group": group,
+                        "name": cell.member.name,
+                        "defines": sorted(own),
+                        "sibling": sibling.member.name if sibling.member else None,
+                        "sibling_defines": sorted(theirs),
+                    },
                 )
                 break
         mode = groups.modes.get(group, "switch")
@@ -601,6 +689,7 @@ def _variant_notes(cells: list[CellAnalysis], dag: Dag, groups: GroupConfig) ->
                 f"group `{group}` has mode {mode!r}; it runs as a switch",
                 annotation="variant",
                 line=line,
+                args={"group": group, "mode": mode},
             )
         active = groups.active.get(group)
         if mode == "sweep" and active:
@@ -610,6 +699,7 @@ def _variant_notes(cells: list[CellAnalysis], dag: Dag, groups: GroupConfig) ->
                 f"group `{group}` is a sweep, so its active selection is ignored",
                 annotation="variant",
                 line=line,
+                args={"group": group, "active": active},
             )
         if mode != "sweep" and active:
             names = [name for _, name in members[group].members]
@@ -620,6 +710,7 @@ def _variant_notes(cells: list[CellAnalysis], dag: Dag, groups: GroupConfig) ->
                     f"group `{group}` selects `{active}`, which no cell provides",
                     annotation="variant",
                     line=line,
+                    args={"group": group, "active": active},
                 )
 
 
@@ -683,27 +774,35 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
 
         for name, seed_id in cell.seeds.items():
             seed = cell_by_id.get(seed_id)
+            origin, origin_line = cell.seed_origins.get(name, (None, None))
             if seed is None:
-                cell.error("seed_unknown_cell", f"`{name}` is seeded from {seed_id!r}, no cell")
+                reason, message = "unknown", f"`{name}` is seeded from {seed_id!r}, no cell"
             elif seed_id == cell.id:
-                cell.error("seed_unknown_cell", f"`{name}` is seeded from the cell itself")
+                reason, message = "self", f"`{name}` is seeded from the cell itself"
             elif seed_id in inactive:
-                cell.error(
-                    "seed_unknown_cell", f"`{name}` is seeded from {seed_id}, an inactive variant"
+                reason, message = (
+                    "inactive",
+                    f"`{name}` is seeded from {seed_id}, an inactive variant",
                 )
             elif name not in seed.defines:
-                cell.error(
-                    "seed_unknown_cell",
-                    f"`{name}` is seeded from {seed_id}, which does not define it",
-                )
+                reason = "undefined"
+                message = f"`{name}` is seeded from {seed_id}, which does not define it"
             else:
                 dag.read_producers.setdefault(cell.id, {})[name] = seed_id
                 _wire(dag, seed_id, cell.id, name)
+                continue
+            cell.error(
+                "seed_unknown_cell",
+                message,
+                annotation=origin,
+                line=origin_line,
+                args={"name": name, "cell_id": seed_id, "reason": reason},
+            )
 
         dag.consumed_variables[cell.id].update(v for v in cell.kept if v in cell.defines)
 
         for upstream_id in cell.after:
-            line = _first_line(cell.directives, "after")
+            line = _naming_line(cell.directives, "after", upstream_id)
             if upstream_id == cell.id:
                 cell.note(
                     "after_self",
@@ -711,6 +810,7 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
                     "`@after` names this cell",
                     annotation="after",
                     line=line,
+                    args={"cell_id": upstream_id},
                 )
                 continue
             if upstream_id not in cell_by_id:
@@ -719,6 +819,7 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
                     f"`@after {upstream_id}` names no cell",
                     annotation="after",
                     line=line,
+                    args={"cell_id": upstream_id},
                 )
                 continue
             if upstream_id not in active_ids:
@@ -727,6 +828,7 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
                     f"`@after {upstream_id}` names a variant that is not active",
                     annotation="after",
                     line=line,
+                    args={"cell_id": upstream_id},
                 )
                 continue
             dag.edges.append(DagEdge(from_cell_id=upstream_id, to_cell_id=cell.id, variable=""))
@@ -737,7 +839,10 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
 
         is_sweep_member = cell.id in sweep_member_group
         fanout_group: str | None = None
-        if cell.fanout is not None:
+        if cell.fanout is not None and cell.fanout.over is not None:
+            dag.each_cells[cell.id] = cell.fanout.over
+            _each_notes(cell, dag)
+        elif cell.fanout is not None:
             read_groups = dag.sweep_groups_read(cell.id)
             named = cell.fanout.group
             if named is not None:
@@ -781,11 +886,44 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
         message = "Cycle detected in DAG: " + (" → ".join(cycles[0]) if cycles else "unknown")
         for cid in active_cell_ids:
             if cid in on_cycle:
-                cell_by_id[cid].error("cycle", message)
+                cell_by_id[cid].error("cycle", message, args={"cycle": cycles[0] if cycles else []})
     dag.topological_order = _kahn(dag, active_cell_ids)
     return dag
 
 
+def _each_notes(cell: CellAnalysis, dag: Dag) -> None:
+    """What an ``@each`` cell cannot iterate, or be."""
+    assert cell.fanout is not None and cell.fanout.over is not None
+    source = cell.fanout.over
+    line = _first_line(cell.directives, "each")
+    if cell.member is not None:
+        cell.error(
+            "each_on_variant_member",
+            "a variant member cannot also run once per item",
+            annotation="each",
+            line=line,
+        )
+    if source in cell.bound:
+        return
+    producer = dag.read_producer(cell.id, source)
+    if isinstance(producer, SweepProducer):
+        cell.error(
+            "each_over_sweep",
+            f"`{source}` comes from a sweep, which has no one collection to iterate",
+            annotation="each",
+            line=line,
+            args={"source": source},
+        )
+    elif producer is None:
+        cell.error(
+            "each_unknown_source",
+            f"no cell before this one defines `{source}`, and the cell does not bind it",
+            annotation="each",
+            line=line,
+            args={"source": source},
+        )
+
+
 def _fanout_notes(
     cell: CellAnalysis, read_groups: set[str], sizes: Mapping[str, tuple[str, ...]]
 ) -> None:
@@ -810,6 +948,7 @@ def _fanout_notes(
                 f"`@per_variant {named}` names no sweep group the cell reads",
                 annotation="per_variant",
                 line=line,
+                args={"group": named},
             )
             return
         effective = named
@@ -830,6 +969,7 @@ def _fanout_notes(
                 f"`@per_variant` reads {', '.join(sorted(read_groups))}; name one",
                 annotation="per_variant",
                 line=line,
+                args={"groups": sorted(read_groups)},
             )
             return
         effective = next(iter(read_groups))
@@ -840,6 +980,7 @@ def _fanout_notes(
             f"group `{effective}` has one variant",
             annotation="per_variant",
             line=line,
+            args={"group": effective},
         )
 
 
@@ -861,6 +1002,7 @@ def _restrict(graph: Dag, keep: set[str]) -> Dag:
             if c in keep
         },
         fanout_cells={c: p for c, p in graph.fanout_cells.items() if c in keep},
+        each_cells={c: s for c, s in graph.each_cells.items() if c in keep},
         read_producers={c: dict(r) for c, r in graph.read_producers.items() if c in keep},
     )
     active = [cid for cid in ids if cid not in graph.inactive_cells]
@@ -909,6 +1051,20 @@ class Analysis:
 
         return execution_key(self, cell_id, **kwargs)
 
+    def variant_read(
+        self, cell_id: str, upstream_id: str, var: str, instance: str | None = None
+    ) -> Any:
+        """See ``cellgraph.keys.variant_read``."""
+        from cellgraph.keys import variant_read
+
+        return variant_read(self, cell_id, upstream_id, var, instance)
+
+    def shadowed_reads(self, cell_id: str) -> set[str]:
+        """See ``cellgraph.keys.shadowed_reads``."""
+        from cellgraph.keys import shadowed_reads
+
+        return shadowed_reads(self, cell_id)
+
     def plan(self, target: str, ready: Callable[[str], bool]) -> list[str]:
         """See ``cellgraph.plan.plan``."""
         from cellgraph.plan import plan
@@ -1008,6 +1164,5 @@ def analyze(
         digest=digest,
     )
     for ca in analyses:
-        if ca.plannable:
-            ca.normalized, ca.definition_key = definition_key(analysis, ca)
+        ca.normalized, ca.definition_key = definition_key(analysis, ca)
     return analysis

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/core.py

@@ -1,4 +1,5 @@
-"""The core vocabulary: ``@name``, ``@after``, ``@nocache``, ``@variant``, ``@per_variant``.
+"""The core vocabulary: ``@name``, ``@after``, ``@nocache``, ``@variant``, ``@per_variant``,
+``@each``.
 
 The parsing rules and diagnostic codes are Strata's (``annotations.py`` and
 ``annotation_validation.py``), which these were ported from.
@@ -6,7 +7,7 @@ The parsing rules and diagnostic codes are Strata's (``annotations.py`` and
 
 from __future__ import annotations
 
-from cellgraph.effects import After, FanOut, Member, NoCache
+from cellgraph.effects import After, Binds, FanOut, Member, NoCache
 from cellgraph.grammar import Directive
 from cellgraph.vocabulary import Annotation, AnnotationError, flag, mark_stacked, scalar
 
@@ -56,6 +57,36 @@ def _parse_per_variant(directives: list[Directive]) -> str | None:
     return tokens[0] if tokens else None
 
 
+def _parse_each(directives: list[Directive]) -> dict[str, str]:
+    """``# @each <name> in <source> [key=<column>] [glob=<pattern>]``; the last line wins."""
+    value: dict[str, str] = {}
+    for directive in directives:
+        tokens = directive.value.split()
+        value = {}
+        if len(tokens) >= 3 and tokens[1] == "in":
+            value = {"name": tokens[0], "source": tokens[2]}
+            for token in tokens[3:]:
+                option, _, setting = token.partition("=")
+                if option not in ("key", "glob") or not setting or option in value:
+                    value = {}
+                    break
+                value[option] = setting
+        if not value or not (value["name"].isidentifier() and value["source"].isidentifier()):
+            raise AnnotationError(
+                f"`@each {directive.value}`: expected `@each <name> in <source> "
+                "[key=<column>] [glob=<pattern>]`, with the name and source Python identifiers",
+                code="each_malformed",
+                line=directive.line,
+            )
+        if value["name"] == value["source"]:
+            raise AnnotationError(
+                f"`@each {directive.value}`: an item cannot have its source's name",
+                code="each_malformed",
+                line=directive.line,
+            )
+    return value
+
+
 def core_annotations() -> list[Annotation]:
     return [
         Annotation("name", parse=scalar(str), in_key=False),
@@ -84,4 +115,10 @@ def core_annotations() -> list[Annotation]:
             parse=_parse_per_variant,
             effects=lambda group: [FanOut(group)],
         ),
+        # What it iterates, and how an item is named, decide what each instance is.
+        Annotation(
+            "each",
+            parse=_parse_each,
+            effects=lambda each: [Binds(each["name"]), FanOut(over=each["source"])],
+        ),
     ]

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/diagnostics.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+from collections.abc import Mapping
+from dataclasses import dataclass, field
 from enum import StrEnum
+from typing import Any
 
 
 class Severity(StrEnum):
@@ -16,7 +18,11 @@ class Severity(StrEnum):
 
 @dataclass(frozen=True)
 class Diagnostic:
-    """One finding. ``annotation`` is the directive key it is about, if any."""
+    """One finding. ``annotation`` is the directive key it is about, if any.
+
+    ``args`` holds what the message was built from, such as the cell id an ``@after``
+    names, so a host can word the finding its own way.
+    """
 
     code: str
     severity: Severity
@@ -24,3 +30,4 @@ class Diagnostic:
     cell: str | None = None
     annotation: str | None = None
     line: int | None = None
+    args: Mapping[str, Any] = field(default_factory=dict, compare=False)

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/effects.py

@@ -88,9 +88,14 @@ class Member:
 
 @dataclass(frozen=True)
 class FanOut:
-    """Run once per member of ``group``, or of the single sweep group the cell reads."""
+    """Run once per member of ``group``, or of the single sweep group the cell reads.
+
+    With ``over``, once per item the host lists from that name instead: an upstream
+    variable, or a name the cell binds. The cell reads it through its items alone.
+    """
 
     group: str | None = None
+    over: str | None = None
 
 
 @dataclass(frozen=True)

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/keys.py

@@ -104,6 +104,18 @@ class Published:
     cell_ref: OutputRef | Unavailable | None = None
 
 
+@dataclass(frozen=True)
+class Item:
+    """One item of an ``@each`` cell: what the host calls it, and its current content.
+
+    The key is the instance's name, so it must be stable across runs; the digest is
+    what makes the instance run again.
+    """
+
+    key: str
+    digest: str
+
+
 @dataclass(frozen=True)
 class Resolved:
     """A resolver's answer for one input.
@@ -142,7 +154,11 @@ class Unavailable:
 
 @dataclass(frozen=True)
 class HostKeyed:
-    """The host computes this cell's key (``owns_key=False``)."""
+    """The host computes this cell's key (``owns_key=False``).
+
+    ``parts`` holds the upstream components the cell reads, as a key would, and no
+    inputs: those are the host's to key.
+    """
 
     parts: dict[str, Any] | None = None
 
@@ -174,15 +190,23 @@ def definition_key(analysis: Analysis, cell: CellAnalysis) -> tuple[str, str]:
 
 
 @dataclass(frozen=True)
-class _VariantRead:
+class VariantRead:
+    """What a cell reads of one variable a sweep makes: the variants, and how.
+
+    ``scalar``: a fan-out instance's own group, read as that variant's value; otherwise
+    the ``{variant: value}`` dict. ``fanout``: the variants are one fan-out cell's
+    instances rather than the members of a group. No variants: another variant's
+    member, which this cell does not read.
+    """
+
     variants: tuple[str, ...]
     scalar: bool
     fanout: bool
 
 
-def _variant_read(
-    analysis: Analysis, cell_id: str, upstream_id: str, var: str, instance: str | None
-) -> _VariantRead | None:
+def variant_read(
+    analysis: Analysis, cell_id: str, upstream_id: str, var: str, instance: str | None = None
+) -> VariantRead | None:
     """What *cell_id* (or its *instance*) reads of *var* from *upstream_id*, if a sweep makes it."""
     from cellgraph.analysis import SweepProducer
 
@@ -198,15 +222,33 @@ def _variant_read(
         names = (
             (instance,) if own and instance is not None else tuple(n for n, _ in producer.variants)
         )
-        return _VariantRead(variants=names, scalar=own, fanout=True)
+        return VariantRead(variants=names, scalar=own, fanout=True)
     member = next((name for name, cid in producer.variants if cid == upstream_id), None)
     if member is None:
         return None
     if own:
-        return _VariantRead(
+        return VariantRead(
             variants=(member,) if member == instance else (), scalar=True, fanout=False
         )
-    return _VariantRead(variants=(member,), scalar=False, fanout=False)
+    return VariantRead(variants=(member,), scalar=False, fanout=False)
+
+
+def shadowed_reads(analysis: Analysis, cell_id: str) -> set[str]:
+    """Names *cell_id* reads that an upstream defines but is not wired to give it.
+
+    Such a name is read from its wired producer only.
+    """
+    graph = analysis.graph
+    cell = analysis.cells[cell_id]
+    reads = set(cell.references) | set(cell.builtin_references)
+    shadowed: set[str] = set()
+    for upstream_id in graph.cell_upstream.get(cell_id, []):
+        upstream = analysis.cells.get(upstream_id)
+        if upstream is not None:
+            shadowed |= (reads & set(upstream.defines)) - graph.wired_variables(
+                cell_id, upstream_id
+            )
+    return shadowed
 
 
 class _Assembly:
@@ -247,6 +289,7 @@ def _upstream(
     cell_id: str,
     instance: str | None,
     published: Callable[[str], Published],
+    without: str | None = None,
 ) -> _Assembly:
     graph = analysis.graph
     cell = analysis.cells[cell_id]
@@ -256,14 +299,7 @@ def _upstream(
         return assembly
 
     reads = set(cell.references) | set(cell.builtin_references)
-    # A name several upstreams define is read from its wired producer only.
-    shadowed: set[str] = set()
-    for upstream_id in upstream_ids:
-        upstream = analysis.cells.get(upstream_id)
-        if upstream is not None:
-            shadowed |= (reads & set(upstream.defines)) - graph.wired_variables(
-                cell_id, upstream_id
-            )
+    shadowed = shadowed_reads(analysis, cell_id)
 
     sweep_buckets: dict[str, list[tuple[str, str]]] = {}
     for upstream_id in upstream_ids:
@@ -274,19 +310,19 @@ def _upstream(
         wired = graph.wired_variables(cell_id, upstream_id) if shadowed else set()
         snapshot = published(upstream_id)
 
-        items: list[tuple[str | None, Any]] = list(snapshot.outputs.items())
-        if not items and snapshot.cell_ref is not None:
-            items = [(None, snapshot.cell_ref)]
+        names = list(snapshot.outputs)
+        if not names and snapshot.cell_ref is not None:
+            value = assembly.component(snapshot.cell_ref, by_content, f"cell:{upstream_id}")
+            if value is not None:
+                assembly.add(f"cell:{upstream_id}", value)
 
-        for var, ref in items:
-            if var is None:
-                value = assembly.component(ref, by_content, f"cell:{upstream_id}")
-                if value is not None:
-                    assembly.add(f"cell:{upstream_id}", value)
-                continue
+        # An output is looked up only once it counts, so a host's mapping can be lazy.
+        for var in names:
             if by_content and var not in reads:
                 continue
-            read = _variant_read(analysis, cell_id, upstream_id, var, instance)
+            if var == without:
+                continue
+            read = variant_read(analysis, cell_id, upstream_id, var, instance)
             if read is not None and read.fanout:
                 parts = []
                 for name in read.variants:
@@ -304,7 +340,9 @@ def _upstream(
                 continue
             if read is not None and not read.variants:
                 continue  # another variant's member
-            value = assembly.component(ref, by_content, f"output:{upstream_id}.{var}")
+            value = assembly.component(
+                snapshot.outputs[var], by_content, f"output:{upstream_id}.{var}"
+            )
             if value is None:
                 continue
             if read is not None:
@@ -333,7 +371,7 @@ def execution_key(
     analysis: Analysis,
     cell_id: str,
     *,
-    instance: str | None = None,
+    instance: str | Item | None = None,
     published: Callable[[str], Published],
     resolve: Mapping[str, Resolver] | None = None,
     env_hash: str,
@@ -341,27 +379,51 @@ def execution_key(
     """The key that says whether *cell_id* (or one fan-out *instance*) must run.
 
     ``published(cell_id)`` is what an upstream published; ``resolve`` has one resolver
-    per input kind. ``instance=None`` on a fan-out cell is its cell-level key.
+    per input kind. ``instance=None`` on a fan-out cell is its cell-level key. An ``@each``
+    cell's instance is an :class:`Item`, keyed on that item in place of its source, so
+    changing one item leaves every other item's key alone.
     """
-    from cellgraph.analysis import NotPlannableError
+    from cellgraph.analysis import STRUCTURAL_CODES, NotPlannableError
 
     cell = analysis.cells[cell_id]
-    if not cell.plannable:
-        raise NotPlannableError(cell_id, analysis.blocking(cell_id))
-    if not analysis.analyzers[cell.language].owns_key:
-        return HostKeyed()
+    # A blocked cell is keyed too: a result cached under its key is still its result.
+    # One a cycle or a variant collision reaches has no sound upstreams to key it on.
+    structural = [
+        d
+        for blocker in cell.blocked_by
+        for d in analysis.cells[blocker].diagnostics
+        if d.code in STRUCTURAL_CODES
+    ]
+    if structural:
+        raise NotPlannableError(cell_id, structural)
+    owns_key = analysis.analyzers[cell.language].owns_key
     fanout = analysis.graph.fanout_cells.get(cell_id)
-    if instance is not None:
+    source = analysis.graph.each_cells.get(cell_id)
+    item = instance if isinstance(instance, Item) else None
+    if item is not None:
+        if source is None:
+            raise ValueError(f"cell {cell_id} has no items, so it has no item {item.key!r}")
+        instance = item.key
+    elif instance is not None:
+        if source is not None:
+            raise ValueError(f"cell {cell_id} runs per item; {instance!r} is not an Item")
         if fanout is None:
             raise ValueError(f"cell {cell_id} does not fan out, so it has no instance {instance!r}")
         if instance not in {name for name, _ in fanout.variants}:
             raise ValueError(f"cell {cell_id} has no instance {instance!r}")
 
     legacy = analysis.digest == "legacy-1"
-    assembly = _upstream(analysis, cell_id, instance, published)
+    if item is not None:
+        assembly = _upstream(analysis, cell_id, None, published, without=source)
+        assembly.add(f"each:{source}", f"each:{item.key}={item.digest}")
+    else:
+        assembly = _upstream(analysis, cell_id, instance, published)
     inputs: list[tuple[str, str, str]] = []
     resolvers = resolve or {}
-    for spec in cell.inputs:
+    # A host-keyed cell's inputs are the host's to key. An item stands in for its source.
+    for spec in cell.inputs if owns_key else []:
+        if item is not None and spec.name == source:
+            continue
         if spec.kind not in resolvers:
             raise ValueError(f"no resolver for {spec.kind!r} inputs (cell {cell_id})")
         result = resolvers[spec.kind](spec)
@@ -391,6 +453,8 @@ def execution_key(
         "upstream": sorted([address, value] for address, value in assembly.components),
         "inputs": sorted([kind, name, component] for kind, name, component in inputs),
     }
+    if not owns_key:
+        return HostKeyed(parts=parts)
     if assembly.unresolved:
         return Unavailable(
             reason="unresolved: " + ", ".join(assembly.unresolved),
@@ -401,7 +465,9 @@ def execution_key(
     components = [value for _, value in assembly.components] + [c for _, _, c in inputs]
     if legacy:
         digest = compute_provenance_hash(components, cell.definition_key, env_hash)
-        if instance is not None:
+        if item is not None:
+            digest = derive_subkey(digest, f"each={item.key}")
+        elif instance is not None:
             digest = derive_subkey(digest, f"variant={instance}")
     else:
         digest = _sha256(

{cellgraph-0.1.0 → cellgraph-0.3.0}/src/cellgraph/python.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import ast
 import builtins
+import functools
 import symtable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
@@ -641,11 +642,26 @@ def analyze_cell(source: str) -> CellAnalysis:
     )
 
 
+# A notebook is analyzed again on every edit with most cells unchanged, so the
+# per-source work is kept for the sources seen last.
+_MEMO_SIZE = 4096
+
+
+@functools.lru_cache(maxsize=_MEMO_SIZE)
+def _analyzed(source: str) -> CellAnalysis:
+    return analyze_cell(source)
+
+
 def normalize(source: str) -> str:
     """Canonical text for *source*: its AST round trip, so comments and layout drop out.
 
     Source that does not parse falls back to the line strip, which keeps comments.
     """
+    return _normalized(source)
+
+
+@functools.lru_cache(maxsize=_MEMO_SIZE)
+def _normalized(source: str) -> str:
     try:
         return ast.unparse(ast.parse(source))
     except SyntaxError:
@@ -659,7 +675,8 @@ class PythonAnalyzer:
     owns_key = True
 
     def analyze(self, cell: Cell) -> Analyzed:
-        result = analyze_cell(cell.source)
+        # Memoized: the lists are copied, so the caller may change its own.
+        result = _analyzed(cell.source)
         return Analyzed(
             defines=list(result.defines),
             references=list(result.references),