cellgraph 0.2.0__tar.gz → 0.3.0__tar.gz

{cellgraph-0.2.0 → cellgraph-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellgraph
-Version: 0.2.0
+Version: 0.3.0
 Summary: The cell graph of a notebook or script: annotations with a host-supplied vocabulary, analysis, the DAG, and the keys that say whether a cell must run.
 Author: Fangchen Li
 Author-email: Fangchen Li <fangchen.li@outlook.com>

{cellgraph-0.2.0 → cellgraph-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cellgraph"
-version = "0.2.0"
+version = "0.3.0"
 description = "The cell graph of a notebook or script: annotations with a host-supplied vocabulary, analysis, the DAG, and the keys that say whether a cell must run."
 readme = "README.md"
 requires-python = ">=3.12"

{cellgraph-0.2.0 → cellgraph-0.3.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "cellgraph"
-version = "0.2.0"
+version = "0.3.0"
 description = "The cell graph of a notebook or script: annotations with a host-supplied vocabulary, analysis, the DAG, and the keys that say whether a cell must run."
 readme = "README.md"
 requires-python = ">=3.12"

{cellgraph-0.2.0 → cellgraph-0.3.0}/src/cellgraph/__init__.py

@@ -40,6 +40,7 @@ from cellgraph.grammar import GRAMMAR, Directive, Grammar
 from cellgraph.keys import (
     PARTS_SCHEMA,
     HostKeyed,
+    Item,
     Keyed,
     OutputRef,
     Provenance,
@@ -105,6 +106,7 @@ __all__ = [
     "GroupConfig",
     "HostKeyed",
     "Input",
+    "Item",
     "Keeps",
     "Keyed",
     "LanguageAnalyzer",

{cellgraph-0.2.0 → cellgraph-0.3.0}/src/cellgraph/analysis.py

@@ -136,6 +136,8 @@ class Dag:
     inactive_cells: set[str] = field(default_factory=set)
     wired_inputs: dict[str, dict[str, set[str]]] = field(default_factory=dict)
     fanout_cells: dict[str, SweepProducer] = field(default_factory=dict)
+    # An ``@each`` cell: the name its items come from. Not a sweep: a reader sees one output.
+    each_cells: dict[str, str] = field(default_factory=dict)
     read_producers: dict[str, dict[str, str | SweepProducer]] = field(default_factory=dict)
 
     def wired_variables(self, consumer_id: str, producer_id: str) -> set[str]:
@@ -511,6 +513,20 @@ def _names_phase(ca: CellAnalysis, analyzed: Analyzed) -> None:
             args={"name": name},
         )
 
+    for effect in ca.effects:
+        if isinstance(effect, FanOut) and effect.over is not None:
+            # An instance's key leaves the source out, so the body must not read it whole.
+            if effect.over in analyzed.references:
+                ca.error(
+                    "each_reads_source",
+                    f"the cell reads `{effect.over}` itself, which every item's result would "
+                    "then depend on; read each item, or compute it in a cell before this one",
+                    annotation="each",
+                    args={"source": effect.over},
+                )
+            elif effect.over not in bound and effect.over not in references:
+                references.append(effect.over)
+
     ca.defines = defines
     ca.references = [name for name in references if name not in bound]
     ca.builtin_references = [name for name in analyzed.builtin_references if name not in bound]
@@ -555,7 +571,7 @@ def _names_phase(ca: CellAnalysis, analyzed: Analyzed) -> None:
             if ca.fanout is not None and ca.fanout != effect:
                 ca.error(
                     "fanout_conflict",
-                    "the cell fans out over two groups",
+                    "the cell fans out two ways",
                     annotation=origin,
                     line=origin_line,
                 )
@@ -823,7 +839,10 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
 
         is_sweep_member = cell.id in sweep_member_group
         fanout_group: str | None = None
-        if cell.fanout is not None:
+        if cell.fanout is not None and cell.fanout.over is not None:
+            dag.each_cells[cell.id] = cell.fanout.over
+            _each_notes(cell, dag)
+        elif cell.fanout is not None:
             read_groups = dag.sweep_groups_read(cell.id)
             named = cell.fanout.group
             if named is not None:
@@ -872,6 +891,39 @@ def _wiring_phase(cells: list[CellAnalysis], groups: GroupConfig) -> Dag:
     return dag
 
 
+def _each_notes(cell: CellAnalysis, dag: Dag) -> None:
+    """What an ``@each`` cell cannot iterate, or be."""
+    assert cell.fanout is not None and cell.fanout.over is not None
+    source = cell.fanout.over
+    line = _first_line(cell.directives, "each")
+    if cell.member is not None:
+        cell.error(
+            "each_on_variant_member",
+            "a variant member cannot also run once per item",
+            annotation="each",
+            line=line,
+        )
+    if source in cell.bound:
+        return
+    producer = dag.read_producer(cell.id, source)
+    if isinstance(producer, SweepProducer):
+        cell.error(
+            "each_over_sweep",
+            f"`{source}` comes from a sweep, which has no one collection to iterate",
+            annotation="each",
+            line=line,
+            args={"source": source},
+        )
+    elif producer is None:
+        cell.error(
+            "each_unknown_source",
+            f"no cell before this one defines `{source}`, and the cell does not bind it",
+            annotation="each",
+            line=line,
+            args={"source": source},
+        )
+
+
 def _fanout_notes(
     cell: CellAnalysis, read_groups: set[str], sizes: Mapping[str, tuple[str, ...]]
 ) -> None:
@@ -950,6 +1002,7 @@ def _restrict(graph: Dag, keep: set[str]) -> Dag:
             if c in keep
         },
         fanout_cells={c: p for c, p in graph.fanout_cells.items() if c in keep},
+        each_cells={c: s for c, s in graph.each_cells.items() if c in keep},
         read_producers={c: dict(r) for c, r in graph.read_producers.items() if c in keep},
     )
     active = [cid for cid in ids if cid not in graph.inactive_cells]

{cellgraph-0.2.0 → cellgraph-0.3.0}/src/cellgraph/core.py

@@ -1,4 +1,5 @@
-"""The core vocabulary: ``@name``, ``@after``, ``@nocache``, ``@variant``, ``@per_variant``.
+"""The core vocabulary: ``@name``, ``@after``, ``@nocache``, ``@variant``, ``@per_variant``,
+``@each``.
 
 The parsing rules and diagnostic codes are Strata's (``annotations.py`` and
 ``annotation_validation.py``), which these were ported from.
@@ -6,7 +7,7 @@ The parsing rules and diagnostic codes are Strata's (``annotations.py`` and
 
 from __future__ import annotations
 
-from cellgraph.effects import After, FanOut, Member, NoCache
+from cellgraph.effects import After, Binds, FanOut, Member, NoCache
 from cellgraph.grammar import Directive
 from cellgraph.vocabulary import Annotation, AnnotationError, flag, mark_stacked, scalar
 
@@ -56,6 +57,36 @@ def _parse_per_variant(directives: list[Directive]) -> str | None:
     return tokens[0] if tokens else None
 
 
+def _parse_each(directives: list[Directive]) -> dict[str, str]:
+    """``# @each <name> in <source> [key=<column>] [glob=<pattern>]``; the last line wins."""
+    value: dict[str, str] = {}
+    for directive in directives:
+        tokens = directive.value.split()
+        value = {}
+        if len(tokens) >= 3 and tokens[1] == "in":
+            value = {"name": tokens[0], "source": tokens[2]}
+            for token in tokens[3:]:
+                option, _, setting = token.partition("=")
+                if option not in ("key", "glob") or not setting or option in value:
+                    value = {}
+                    break
+                value[option] = setting
+        if not value or not (value["name"].isidentifier() and value["source"].isidentifier()):
+            raise AnnotationError(
+                f"`@each {directive.value}`: expected `@each <name> in <source> "
+                "[key=<column>] [glob=<pattern>]`, with the name and source Python identifiers",
+                code="each_malformed",
+                line=directive.line,
+            )
+        if value["name"] == value["source"]:
+            raise AnnotationError(
+                f"`@each {directive.value}`: an item cannot have its source's name",
+                code="each_malformed",
+                line=directive.line,
+            )
+    return value
+
+
 def core_annotations() -> list[Annotation]:
     return [
         Annotation("name", parse=scalar(str), in_key=False),
@@ -84,4 +115,10 @@ def core_annotations() -> list[Annotation]:
             parse=_parse_per_variant,
             effects=lambda group: [FanOut(group)],
         ),
+        # What it iterates, and how an item is named, decide what each instance is.
+        Annotation(
+            "each",
+            parse=_parse_each,
+            effects=lambda each: [Binds(each["name"]), FanOut(over=each["source"])],
+        ),
     ]

{cellgraph-0.2.0 → cellgraph-0.3.0}/src/cellgraph/effects.py

@@ -88,9 +88,14 @@ class Member:
 
 @dataclass(frozen=True)
 class FanOut:
-    """Run once per member of ``group``, or of the single sweep group the cell reads."""
+    """Run once per member of ``group``, or of the single sweep group the cell reads.
+
+    With ``over``, once per item the host lists from that name instead: an upstream
+    variable, or a name the cell binds. The cell reads it through its items alone.
+    """
 
     group: str | None = None
+    over: str | None = None
 
 
 @dataclass(frozen=True)

{cellgraph-0.2.0 → cellgraph-0.3.0}/src/cellgraph/keys.py

@@ -104,6 +104,18 @@ class Published:
     cell_ref: OutputRef | Unavailable | None = None
 
 
+@dataclass(frozen=True)
+class Item:
+    """One item of an ``@each`` cell: what the host calls it, and its current content.
+
+    The key is the instance's name, so it must be stable across runs; the digest is
+    what makes the instance run again.
+    """
+
+    key: str
+    digest: str
+
+
 @dataclass(frozen=True)
 class Resolved:
     """A resolver's answer for one input.
@@ -277,6 +289,7 @@ def _upstream(
     cell_id: str,
     instance: str | None,
     published: Callable[[str], Published],
+    without: str | None = None,
 ) -> _Assembly:
     graph = analysis.graph
     cell = analysis.cells[cell_id]
@@ -307,6 +320,8 @@ def _upstream(
         for var in names:
             if by_content and var not in reads:
                 continue
+            if var == without:
+                continue
             read = variant_read(analysis, cell_id, upstream_id, var, instance)
             if read is not None and read.fanout:
                 parts = []
@@ -356,7 +371,7 @@ def execution_key(
     analysis: Analysis,
     cell_id: str,
     *,
-    instance: str | None = None,
+    instance: str | Item | None = None,
     published: Callable[[str], Published],
     resolve: Mapping[str, Resolver] | None = None,
     env_hash: str,
@@ -364,7 +379,9 @@ def execution_key(
     """The key that says whether *cell_id* (or one fan-out *instance*) must run.
 
     ``published(cell_id)`` is what an upstream published; ``resolve`` has one resolver
-    per input kind. ``instance=None`` on a fan-out cell is its cell-level key.
+    per input kind. ``instance=None`` on a fan-out cell is its cell-level key. An ``@each``
+    cell's instance is an :class:`Item`, keyed on that item in place of its source, so
+    changing one item leaves every other item's key alone.
     """
     from cellgraph.analysis import STRUCTURAL_CODES, NotPlannableError
 
@@ -381,18 +398,32 @@ def execution_key(
         raise NotPlannableError(cell_id, structural)
     owns_key = analysis.analyzers[cell.language].owns_key
     fanout = analysis.graph.fanout_cells.get(cell_id)
-    if instance is not None:
+    source = analysis.graph.each_cells.get(cell_id)
+    item = instance if isinstance(instance, Item) else None
+    if item is not None:
+        if source is None:
+            raise ValueError(f"cell {cell_id} has no items, so it has no item {item.key!r}")
+        instance = item.key
+    elif instance is not None:
+        if source is not None:
+            raise ValueError(f"cell {cell_id} runs per item; {instance!r} is not an Item")
         if fanout is None:
             raise ValueError(f"cell {cell_id} does not fan out, so it has no instance {instance!r}")
         if instance not in {name for name, _ in fanout.variants}:
             raise ValueError(f"cell {cell_id} has no instance {instance!r}")
 
     legacy = analysis.digest == "legacy-1"
-    assembly = _upstream(analysis, cell_id, instance, published)
+    if item is not None:
+        assembly = _upstream(analysis, cell_id, None, published, without=source)
+        assembly.add(f"each:{source}", f"each:{item.key}={item.digest}")
+    else:
+        assembly = _upstream(analysis, cell_id, instance, published)
     inputs: list[tuple[str, str, str]] = []
     resolvers = resolve or {}
-    # A host-keyed cell's inputs are the host's to key.
+    # A host-keyed cell's inputs are the host's to key. An item stands in for its source.
     for spec in cell.inputs if owns_key else []:
+        if item is not None and spec.name == source:
+            continue
         if spec.kind not in resolvers:
             raise ValueError(f"no resolver for {spec.kind!r} inputs (cell {cell_id})")
         result = resolvers[spec.kind](spec)
@@ -434,7 +465,9 @@ def execution_key(
     components = [value for _, value in assembly.components] + [c for _, _, c in inputs]
     if legacy:
         digest = compute_provenance_hash(components, cell.definition_key, env_hash)
-        if instance is not None:
+        if item is not None:
+            digest = derive_subkey(digest, f"each={item.key}")
+        elif instance is not None:
             digest = derive_subkey(digest, f"variant={instance}")
     else:
         digest = _sha256(