dirigent-cli 0.23.4__tar.gz → 0.24.0__tar.gz

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/PKG-INFO

@@ -1,19 +1,19 @@
 Metadata-Version: 2.4
 Name: dirigent-cli
-Version: 0.23.4
+Version: 0.24.0
 Summary: The dirigent command line interface (dirigent / dg).
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: dirigent-block-execute==0.23.4
-Requires-Dist: dirigent-blocks==0.23.4
-Requires-Dist: dirigent-client==0.23.4
-Requires-Dist: dirigent-common==0.23.4
-Requires-Dist: dirigent-core==0.23.4
-Requires-Dist: dirigent-examples==0.23.4
-Requires-Dist: dirigent-plugin==0.23.4
-Requires-Dist: dirigent-server==0.23.4
+Requires-Dist: dirigent-block-execute==0.24.0
+Requires-Dist: dirigent-blocks==0.24.0
+Requires-Dist: dirigent-client==0.24.0
+Requires-Dist: dirigent-common==0.24.0
+Requires-Dist: dirigent-core==0.24.0
+Requires-Dist: dirigent-examples==0.24.0
+Requires-Dist: dirigent-plugin==0.24.0
+Requires-Dist: dirigent-server==0.24.0
 Requires-Dist: httpx2>=2.12.0
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: pyyaml>=6.0.3

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-cli"
-version = "0.23.4"
+version = "0.24.0"
 description = "The dirigent command line interface (dirigent / dg)."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,14 +11,14 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-block-execute==0.23.4",
-    "dirigent-blocks==0.23.4",
-    "dirigent-client==0.23.4",
-    "dirigent-common==0.23.4",
-    "dirigent-core==0.23.4",
-    "dirigent-examples==0.23.4",
-    "dirigent-plugin==0.23.4",
-    "dirigent-server==0.23.4",
+    "dirigent-block-execute==0.24.0",
+    "dirigent-blocks==0.24.0",
+    "dirigent-client==0.24.0",
+    "dirigent-common==0.24.0",
+    "dirigent-core==0.24.0",
+    "dirigent-examples==0.24.0",
+    "dirigent-plugin==0.24.0",
+    "dirigent-server==0.24.0",
     "httpx2>=2.12.0",
     "python-dotenv>=1.1.0",
     "pyyaml>=6.0.3",

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-cli"
-version = "0.23.4"
+version = "0.24.0"
 description = "The dirigent command line interface (dirigent / dg)."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,14 +11,14 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-block-execute==0.23.4",
-    "dirigent-blocks==0.23.4",
-    "dirigent-client==0.23.4",
-    "dirigent-common==0.23.4",
-    "dirigent-core==0.23.4",
-    "dirigent-examples==0.23.4",
-    "dirigent-plugin==0.23.4",
-    "dirigent-server==0.23.4",
+    "dirigent-block-execute==0.24.0",
+    "dirigent-blocks==0.24.0",
+    "dirigent-client==0.24.0",
+    "dirigent-common==0.24.0",
+    "dirigent-core==0.24.0",
+    "dirigent-examples==0.24.0",
+    "dirigent-plugin==0.24.0",
+    "dirigent-server==0.24.0",
     "httpx2>=2.12.0",
     "python-dotenv>=1.1.0",
     "pyyaml>=6.0.3",

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/commands.py

@@ -43,6 +43,8 @@ from dirigent_cli.messages import (
     INIT_CANCELLED,
     INIT_REFUSED,
     INIT_WAY,
+    INPUT_FILE_UNREADABLE,
+    INPUT_REFUSED,
     INSTANCE_ALREADY_HERE,
     KEEP_IS_LOCAL_ONLY,
     LOCAL_RUN_HAD_NO_OUTCOME,
@@ -309,6 +311,18 @@ def _print_plan(result: ApplyResult, origin: str) -> bool:
     return True
 
 
+def _print_source_tokens(result: ApplyResult, *, base_url: str) -> None:
+    """Print each token an apply minted for a source, once, with where it is presented.
+
+    The instance stores only a token's hash; there is no second chance to read it.
+    """
+    for minted in result.sources.tokens:
+        console.print(f"\n  source [bold]{minted.code}[/]  {base_url.rstrip('/')}{minted.url_path}")
+        console.print(f"  Authorization: Bearer [bold]{minted.token}[/]")
+    if result.sources.tokens:
+        console.print("\n[yellow]Each token is shown once.[/] The instance stores only its hash.\n")
+
+
 def _print_graph(steps: Sequence[GraphStep], title: str = "steps") -> None:
     """Print a pipeline's shape: roots at the margin, each step under what it waits for."""
     lines = render_graph(steps)
@@ -322,7 +336,7 @@ def _print_graph(steps: Sequence[GraphStep], title: str = "steps") -> None:
 def graph_steps(definition: PipelineDefinition) -> list[GraphStep]:
     """Read a definition's steps in the shape the tree is drawn from."""
     return [
-        GraphStep(name=name, block=step.block, depends_on=tuple(step.depends_on), rule=step.rule.value)
+        GraphStep(name=name, block=step.block_id, depends_on=tuple(step.depends_on), rule=step.rule.value)
         for name, step in definition.steps.items()
     ]
 
@@ -402,6 +416,8 @@ def apply_command(
             raise typer.Exit(code=1)
         return
     accepted = all(_print_plan(result, origin) for result, origin in results)
+    for result, _ in results:
+        _print_source_tokens(result, base_url=dg.url)
     if pruned is not None:
         for code in pruned.pruned:
             console.print(f"{styled('deactivated'):<10} {code}  [dim](absent from the project)[/]")
@@ -1080,6 +1096,23 @@ def parse_priority(value: str | None) -> RunPriority | None:
         )
 
 
+def parse_input(value: str | None) -> Any:
+    """Read what a hand run hands its source: JSON or YAML written out, or ``@FILE`` holding either."""
+    if value is None:
+        return None
+    text = value
+    if value.startswith("@"):
+        path = Path(value[1:])
+        try:
+            text = path.read_text()
+        except OSError as error:
+            fail(INPUT_FILE_UNREADABLE, path=str(path), detail=error.strerror or str(error))
+    try:
+        return yaml.safe_load(text)
+    except yaml.YAMLError as error:
+        fail(INPUT_REFUSED, detail=str(error).splitlines()[0])
+
+
 def parse_importance(value: str | None) -> Importance | None:
     """Read the importance a rule was given, naming the vocabulary when it is not one."""
     if value is None:
@@ -1164,6 +1197,20 @@ def run_command(
             "Omitted takes the pipeline's own.",
         ),
     ] = None,
+    source: Annotated[
+        str | None,
+        typer.Option(
+            "--source",
+            help="The source step a pipeline with sources is entered at. Omitted takes its only one.",
+        ),
+    ] = None,
+    given: Annotated[
+        str | None,
+        typer.Option(
+            "--input",
+            help="What arrives at that source, as JSON or YAML, or @FILE. Omitted takes the source's sample.",
+        ),
+    ] = None,
 ) -> None:
     """Run a pipeline by code, or apply and run a document in one command.
 
@@ -1182,6 +1229,7 @@ def run_command(
     wanted = parse_priority(priority)
     if wanted is not None and local:
         fail(PRIORITY_IS_NOT_LOCAL)
+    arrives = parse_input(given)
     if local:
         _run_locally(
             state,
@@ -1197,6 +1245,8 @@ def run_command(
             strict=strict,
             keep=keep,
             root=root,
+            source=source,
+            given=arrives,
         )
         return
     with client_for(state) as dg:
@@ -1224,7 +1274,11 @@ def run_command(
             stored = dg.call(dg.pipelines.get(code))
             schema = dict((stored.document or {}).get("params") or {})
         params = parse_params(param, schema=schema, files=params_file)
-        accepted = dg.call(dg.pipelines.run(code, params=params, window=covered, log_levels=levels, priority=wanted))
+        accepted = dg.call(
+            dg.pipelines.run(
+                code, params=params, window=covered, log_levels=levels, priority=wanted, source=source, input=arrives
+            )
+        )
         if accepted.run_id is None:
             if state.json_output:
                 emit_problem(
@@ -1315,6 +1369,8 @@ def _run_locally(
     strict: bool,
     keep: bool = False,
     root: Path | None = None,
+    source: str | None = None,
+    given: Any = None,
 ) -> None:
     """Apply and run a document in a throwaway instance, streaming what it logs."""
     if state.url is not None or state.profile is not None or state.token is not None:
@@ -1344,6 +1400,8 @@ def _run_locally(
             state=state,
             keep=keep,
             root=root,
+            source=source,
+            given=given,
         )
     )
     if outcome is None:  # pragma: no cover - the generator always ends with an outcome
@@ -1384,6 +1442,8 @@ async def _stream_local(
     state: CliState,
     keep: bool = False,
     root: Path | None = None,
+    source: str | None = None,
+    given: Any = None,
 ) -> LocalOutcome | None:
     """Drive a local run, printing each transition and log line as it lands."""
     outcome: LocalOutcome | None = None
@@ -1400,6 +1460,8 @@ async def _stream_local(
             inherited=settings,
             keep=keep,
             root=root,
+            source=source,
+            given=given,
         ):
             match item:
                 case LocalStarted():

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/explain.py

@@ -25,6 +25,7 @@ from dirigent_common import Duration, JsonMap, base_format_checker, format_durat
 from dirigent_core.engine.definition import ParameterError, PipelineDefinition, RetryPolicy, StepDefinition
 from dirigent_core.engine.failure import backoff_delay
 from dirigent_core.engine.references import ReferenceScope, UnknownReference, resolve
+from dirigent_core.engine.state import item_wise_parent
 
 #: What a cardinality says when the document alone cannot fix it.
 UNKNOWN: Final = "unknown"
@@ -65,7 +66,15 @@ def explain(definition: PipelineDefinition, catalog: Catalog | None = None) -> D
     order = definition.topological_order()
     shapes: dict[str, StepShape] = {}
     for name in order:
-        shapes[name] = _step_shape(name, definition.steps[name], params, shapes, catalog, definition.grid_source(name))
+        shapes[name] = _step_shape(
+            name,
+            definition.steps[name],
+            params,
+            shapes,
+            catalog,
+            definition.grid_source(name),
+            item_wise_parent(definition, name),
+        )
     rows = [shapes[name] for name in order]
     return DocumentShape(
         attempts_max=sum(row.max_attempts * (1 if row.elements is None else row.elements) for row in rows),
@@ -95,20 +104,23 @@ def _step_shape(
     done: Mapping[str, StepShape],
     catalog: Catalog | None,
     source: str | None,
+    paired: str | None = None,
 ) -> StepShape:
     """Read one step as the work it will become, against the steps already read.
 
-    ``source`` is the step a late grid waits for, which the row names.
+    ``source`` is the step a late grid waits for, and ``paired`` the fan-out whose items this
+    step's items wait for one by one; the row names both.
     """
     cardinality, elements = _cardinality(step, params, done, source)
     poll, deadline, from_block = _waits(step, _entry(step, catalog))
     return StepShape(
         name=name,
-        block=step.block,
+        block=step.block_id,
         depends_on=list(step.depends_on),
         cardinality=cardinality,
         elements=elements,
         grid_source=source,
+        item_wise_parent=paired,
         reference=step.for_each if isinstance(step.for_each, str) else None,
         items=step.items.value,
         max_attempts=step.retry.max_attempts,
@@ -147,7 +159,7 @@ def _cardinality(
 
 def _entry(step: StepDefinition, catalog: Catalog | None) -> BlockEntry | None:
     """Find what the catalog says about this step's block, where there is a catalog."""
-    return None if catalog is None else catalog.block(step.block)
+    return None if catalog is None or step.block is None else catalog.block(step.block)
 
 
 def _waits(step: StepDefinition, entry: BlockEntry | None) -> tuple[timedelta | None, timedelta | None, list[str]]:

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/graph.py

@@ -30,7 +30,7 @@ def steps_of_document(document: Mapping[str, Any]) -> list[GraphStep]:
         read.append(
             GraphStep(
                 name=str(name),
-                block=str(held.get("block", "-")),
+                block=_block_of(held),
                 depends_on=tuple(str(item) for item in cast("list[Any]", depends or ())),
                 rule=str(held.get("rule", DEFAULT_RULE)),
             )
@@ -38,6 +38,16 @@ def steps_of_document(document: Mapping[str, Any]) -> list[GraphStep]:
     return read
 
 
+def _block_of(step: Mapping[str, Any]) -> str:
+    """Name what a step runs: its block, a source's kind, or ``reply``."""
+    source = step.get("source")
+    if isinstance(source, dict):
+        return str(cast("dict[str, Any]", source).get("kind", "source"))
+    if "reply" in step:
+        return "reply"
+    return str(step.get("block", "-"))
+
+
 def render_graph(steps: Sequence[GraphStep]) -> list[str]:
     """Draw the graph as indented lines, each step under the last step it waits for.
 

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/local.py

@@ -4,6 +4,7 @@ import asyncio
 import contextlib
 import shutil
 import tempfile
+import time
 from collections.abc import AsyncIterator, Mapping, Sequence
 from datetime import datetime, timedelta
 from pathlib import Path
@@ -12,12 +13,12 @@ from uuid import UUID
 
 import sqlalchemy as sa
 import yaml
-from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, SecretStr
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
 from dirigent_client.enums import AttemptStatus, LogLevel, RunStatus, TriggerKind
 from dirigent_common import JsonMap
-from dirigent_core import migrations
+from dirigent_core import migrations, wakeups
 from dirigent_core.config import Settings, get_settings
 from dirigent_core.database import create_engine, create_session_factory, session_scope
 from dirigent_core.documents import DocumentError, load_pipeline_text, safe_load
@@ -26,11 +27,12 @@ from dirigent_core.engine.definition import PipelineDefinition
 from dirigent_core.engine.runs import RunCreationError, RunWindow
 from dirigent_core.engine.services import EngineServices
 from dirigent_core.engine.state import in_execution_order
-from dirigent_core.models import ArtifactRef, Connection, LogEntry, Run, RunItem, StepAttempt
+from dirigent_core.models import ArtifactRef, Connection, LogEntry, Run, RunItem, StepAttempt, utcnow
 from dirigent_core.pipelines import apply_document
 from dirigent_core.plugins import load_plugin_host
 from dirigent_core.schemas import SchemaRefused, resolve_identity, store_schema
 from dirigent_core.secrets import generate_key
+from dirigent_core.sources.entries import hand_entry
 from dirigent_core.storage import scratch_prefix
 from dirigent_core.worker import Worker
 
@@ -556,6 +558,28 @@ class Tick(BaseModel):
     error: str | None
 
 
+def _passed_through(attempt: StepAttempt, before: AttemptStatus | None) -> list[tuple[AttemptStatus, datetime]]:
+    """Name the states a started attempt went through on its way here that no poll saw, with when.
+
+    A poll reads where an attempt stands, so an attempt claimed and settled between two polls
+    would otherwise be reported only in the state it ended in. One that started was queued
+    and ran: those are reported in order, at the moment each began.
+    """
+    started = attempt.started_at
+    if started is None:
+        return []
+    passed: list[tuple[AttemptStatus, datetime]] = []
+    if before in (None, AttemptStatus.PENDING) and attempt.status is not AttemptStatus.QUEUED:
+        passed.append((AttemptStatus.QUEUED, attempt.available_at or started))
+    if before in (None, AttemptStatus.PENDING, AttemptStatus.QUEUED) and attempt.status not in (
+        AttemptStatus.QUEUED,
+        AttemptStatus.RUNNING,
+        AttemptStatus.WAITING,
+    ):
+        passed.append((AttemptStatus.RUNNING, started))
+    return passed
+
+
 async def _transitions(
     sessions: async_sessionmaker[AsyncSession],
     run_id: UUID,
@@ -582,9 +606,21 @@ async def _transitions(
         # announces one and swallows the rest. The id is the identity of the thing that moved.
         if seen.get(attempt.id) is attempt.status:
             continue
+        before = seen.get(attempt.id)
         seen[attempt.id] = attempt.status
         if attempt.status is AttemptStatus.PENDING:
             continue
+        for passed, at in _passed_through(attempt, before):
+            changed.append(
+                StepTransition(
+                    step=attempt.step_name,
+                    block=attempt.block_id,
+                    attempt=attempt.attempt,
+                    status=passed,
+                    at=at,
+                    item=labels.get(attempt.id),
+                )
+            )
         settled = attempt.status in SETTLED
         changed.append(
             StepTransition(
@@ -618,12 +654,16 @@ async def run_document(
     deadline_seconds: float = DEFAULT_DEADLINE_SECONDS,
     keep: bool = False,
     root: Path | None = None,
+    source: str | None = None,
+    given: JsonValue = None,
 ) -> AsyncIterator[LocalStarted | LogLine | StepTransition | LocalOutcome]:
     """Apply and run a document in a throwaway instance, yielding logs then the outcome.
 
     The last item yielded is always the outcome. ``also`` holds documents to apply first
     and not run, for a pipeline that composes another one. ``root`` names a directory to hold
-    the instance in and keep, so a later run reads what this one wrote.
+    the instance in and keep, so a later run reads what this one wrote. A pipeline with
+    sources is entered at ``source``, its only one when that is omitted, with ``given`` as the
+    input, or the source's sample when that is omitted.
     """
     held = prepare_root(root) if root is not None else Path(tempfile.mkdtemp(prefix="dirigent-local-"))
     leave = keep or root is not None
@@ -654,11 +694,13 @@ async def run_document(
                     )
                 version = await _current_version(session, definition.code)
                 try:
+                    entry = hand_entry(definition, services.sources, source, given, received_at=utcnow())
                     run = await create_run(
                         session,
                         services,
                         version,
                         params=params or {},
+                        entry=entry,
                         attribution=Attribution(kind=TriggerKind.USER, label="dg run --local"),
                         window=window,
                         log_levels=log_levels,
@@ -751,6 +793,9 @@ async def _drive(
     items = _Items()
     try:
         while waited < deadline_seconds:
+            # The mark is taken before the read, so a commit landing between the read and the
+            # wait below still wakes it.
+            mark = wakeups.settled_mark(run_id)
             # The status is read before the log is: a run that had settled by then wrote its
             # last entry before it settled, so the pages below carry the whole of it.
             tick = await _transitions(sessions, run_id, seen, items, order)
@@ -775,8 +820,9 @@ async def _drive(
                 )
                 return
             interval = POLL_SECONDS if moved else min(interval * POLL_WIDEN, POLL_CEILING)
-            await asyncio.sleep(interval)
-            waited += interval
+            paused = time.monotonic()
+            await wakeups.wait_settled(mark, timeout=interval)
+            waited += time.monotonic() - paused
         yield LocalOutcome(
             run_id=run_id,
             pipeline=pipeline,

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/main.py

@@ -14,7 +14,7 @@ from pydantic import ValidationError
 from typer import _click, rich_utils
 from typer.core import TyperGroup
 
-from dirigent_cli import commands, triggers
+from dirigent_cli import commands, source_commands, triggers
 from dirigent_cli.aliases import add_alias, add_list_aliases
 from dirigent_cli.context import CliState, state_of
 from dirigent_cli.formatters import DEFAULT, names
@@ -128,6 +128,7 @@ app.add_typer(commands.connection_app, rich_help_panel=CONNECT_PANEL)
 app.add_typer(triggers.schedule_app, rich_help_panel=TRIGGER_PANEL)
 app.add_typer(triggers.webhook_app, rich_help_panel=TRIGGER_PANEL)
 app.add_typer(triggers.watch_app, rich_help_panel=TRIGGER_PANEL)
+app.add_typer(source_commands.source_app, rich_help_panel=TRIGGER_PANEL)
 app.add_typer(triggers.trigger_document_app, rich_help_panel=TRIGGER_PANEL)
 app.add_typer(triggers.alerts_app, rich_help_panel=TRIGGER_PANEL)
 app.add_typer(docker_app, rich_help_panel=PROCESS_PANEL)
@@ -627,6 +628,7 @@ def prune_command(
     runs: Annotated[str | None, typer.Option(help="Keep settled runs younger than this, such as 30d.")] = None,
     logs: Annotated[str | None, typer.Option(help="Keep log entries younger than this.")] = None,
     deliveries: Annotated[str | None, typer.Option(help="Keep webhook deliveries younger than this.")] = None,
+    inputs: Annotated[str | None, typer.Option(help="Keep the record of inputs to a source younger than this.")] = None,
     firings: Annotated[str | None, typer.Option(help="Keep schedule firings younger than this.")] = None,
     notifications: Annotated[str | None, typer.Option(help="Keep sent alerts younger than this.")] = None,
     scratch: Annotated[
@@ -645,7 +647,7 @@ def prune_command(
     settings = get_settings()
     state = state_of(ctx)
     try:
-        policy = _prune_policy(settings, runs, logs, deliveries, firings, notifications, scratch=scratch)
+        policy = _prune_policy(settings, runs, logs, deliveries, inputs, firings, notifications, scratch=scratch)
     except ValueError as error:
         refuse(INVALID_AGE, status=2, title="Invalid age", detail=str(error))
         raise typer.Exit(code=2) from error
@@ -657,6 +659,7 @@ def _prune_policy(
     runs: str | None,
     logs: str | None,
     deliveries: str | None,
+    inputs: str | None,
     firings: str | None,
     notifications: str | None,
     *,
@@ -673,6 +676,7 @@ def _prune_policy(
         runs=age(runs, settings.retention_runs),
         logs=age(logs, settings.retention_logs),
         deliveries=age(deliveries, settings.retention_deliveries),
+        inputs=age(inputs, settings.retention_inputs),
         firings=age(firings, settings.retention_firings),
         notifications=age(notifications, settings.retention_notifications),
         scratch=scratch,
@@ -939,9 +943,11 @@ def dev(
     checkout is not needed; both may be given, and the directories go in first.
 
     --seed fills the instance from a directory of documents the moment it answers: the
-    connections a file or a document declares are created first, then every document is
-    applied with its schedules and watches paused. A document an instance will not store is reported
-    and passed over, because a corpus holds those on purpose.
+    connections a file or a document declares and the standalone JSON Schema files are created
+    first, then every document is applied with its schedules and watches paused, and one refused
+    for naming something a later document supplies is applied again once the rest are in. A
+    document an instance will not store is reported and passed over, because a corpus holds
+    those on purpose.
     """
     import asyncio
     import os

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/messages.py

@@ -86,6 +86,10 @@ CANNOT_PROMPT_SET = CLI.define("cannot_prompt_set", "--json cannot prompt for {f
 
 PARAMS_REFUSED = CLI.define("params_refused", "{detail}")
 
+INPUT_REFUSED = CLI.define("input_refused", "--input is not JSON or YAML: {detail}")
+
+INPUT_FILE_UNREADABLE = CLI.define("input_file_unreadable", "--input names {path}, which cannot be read: {detail}")
+
 TRIGGERS_DOCUMENT_NOT_RUNNABLE = CLI.define(
     "triggers_document_not_runnable",
     "a triggers document declares clocks for a pipeline and cannot be run; run the pipeline it names",
@@ -320,3 +324,14 @@ DATABASE_BEHIND = HEALTH.define(
 )
 
 DATABASE_OK = HEALTH.define("database.ok", "the database at {where} answers, schema {stamped}")
+
+ONE_INPUT_BODY = CLI.define("one_input_body", "a call takes one body: --body, --text or --form")
+
+TAKES_NAME_EQUALS_VALUE = CLI.define("takes_name_equals_value", "{entry} is not name=value")
+
+SOURCE_NOT_CALLABLE = CLI.define(
+    "source_not_callable",
+    "source {code} of pipeline {pipeline} is of kind {kind}, which this instance does not receive",
+)
+
+SOURCE_UNREACHABLE = CLI.define("source_unreachable", "{url} could not be called: {detail}")

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/schemas.py

@@ -73,6 +73,10 @@ class StepShape(BaseModel):
     grid_source: str | None = None
     """The step a late grid waits for before it expands, or null for a grid fixed at creation."""
 
+    item_wise_parent: str | None = None
+    """The fan-out whose matching item each of this step's items waits for, one by one, or null
+    when the step waits for its prerequisites whole."""
+
     reference: str | None = None
     """The ``for_each`` this step maps over, where it names one rather than listing it."""
 

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/seeding.py

@@ -2,7 +2,9 @@
 
 The seeding goes through the API, so every apply meets the same preflight a person's apply
 meets, and a refusal is a record rather than a failure: a corpus is expected to hold
-documents an instance will not store.
+documents an instance will not store. A corpus goes in as connection files, then the
+standalone JSON Schemas beside its documents, then the documents; a document refused for
+naming something a later one supplies is applied again once the rest are in.
 """
 
 from collections.abc import AsyncIterator, Mapping, Sequence
@@ -13,8 +15,9 @@ from typing import Any, cast
 from dirigent_cli.local import ConnectionSpec
 from dirigent_client import Dirigent, DirigentError, PlanAction, ProvenanceSource, declared_label
 from dirigent_common import JsonMap
-from dirigent_core.documents import CARRIED, SUFFIXES, is_document, readable, safe_load
+from dirigent_core.documents import CARRIED, SUFFIXES, is_document, readable
 from dirigent_core.protocol import Record, make
+from dirigent_core.schemas import code_from_id
 
 __all__ = ["CARRIED", "SUFFIXES", "is_document", "readable", "seed_directories", "seed_installed", "specs"]
 
@@ -36,49 +39,117 @@ async def seed_directories(client: Dirigent, directories: Sequence[Path]) -> Asy
     The closing record's counts are read off the records the seeding emitted, so a reader of
     the stream and this summary never disagree.
     """
-    pipelines = 0
-    refused = 0
-    connections: set[str] = set()
-    async for record in _walk(client, directories):
-        kind = record["kind"]
-        if kind == "seed.applied":
-            pipelines += 1
-        elif kind == "seed.refused":
-            refused += 1
-        elif kind == "seed.connection":
-            connections.add(str(record["connection"]))
-        yield record
+    files = [(str(path), raw) for directory in directories for path, raw in readable(directory)]
+    tally = _Tally()
+    async for record in _seed(client, files):
+        yield tally.count(record)
     yield make(
         "seed.done",
         at=datetime.now(UTC),
         message="seeded",
         directories=[str(one) for one in directories],
-        pipelines=pipelines,
-        refused=refused,
-        connections=sorted(connections),
+        **tally.counts(),
     )
 
 
-async def _walk(client: Dirigent, directories: Sequence[Path]) -> AsyncIterator[Record]:
-    """Seed each directory in the order given: the connections it declares, then its documents.
-
-    A directory's own connection files go in before any of its documents, so a document that
-    names one of them passes the preflight that reads it.
+class _Tally:
+    """What a seeding did, read off the records it emitted, so a reader and the summary agree."""
+
+    def __init__(self) -> None:
+        """Start from nothing seeded."""
+        self.pipelines = 0
+        self.refused = 0
+        self.connections: set[str] = set()
+        self.schemas: set[str] = set()
+
+    def count(self, record: Record) -> Record:
+        """Count one record, and hand it back to be emitted."""
+        match record["kind"]:
+            case "seed.applied":
+                self.pipelines += 1
+            case "seed.refused":
+                self.refused += 1
+            case "seed.connection":
+                self.connections.add(str(record["connection"]))
+            case "seed.schema":
+                self.schemas.add(str(record["schema"]))
+            case _:
+                pass
+        return record
+
+    def counts(self) -> dict[str, Any]:
+        """The closing record's fields."""
+        return {
+            "pipelines": self.pipelines,
+            "refused": self.refused,
+            "connections": sorted(self.connections),
+            "schemas": sorted(self.schemas),
+        }
+
+
+async def _seed(client: Dirigent, files: Sequence[tuple[str, JsonMap]]) -> AsyncIterator[Record]:
+    """Seed a corpus: its connection files, its standalone schemas, then its documents.
+
+    The connections and the schemas go in before any document, so a document that names one
+    passes the preflight that reads it. A document refused in one pass is applied again in
+    the next, since what it names may be a pipeline or a schema a later document supplied, and
+    the passes end when one stores nothing new; what is still refused is reported then.
+    """
+    for origin, raw in files:
+        if not is_document(raw):
+            async for record in _connections(client, specs(raw.get("connections")), origin):
+                yield record
+    for origin, raw in files:
+        code = _schema_file(raw, origin)
+        if code is not None:
+            yield await _standalone_schema(client, code, raw, origin)
+    waiting: list[tuple[str, JsonMap, Record]] = []
+    for origin, raw in files:
+        if is_document(raw):
+            async for record in _carried(client, raw, origin):
+                yield record
+            applied = await _apply(client, raw, origin)
+            if applied["kind"] == "seed.applied":
+                yield applied
+            else:
+                waiting.append((origin, raw, applied))
+    while waiting:
+        still: list[tuple[str, JsonMap, Record]] = []
+        for origin, raw, _refusal in waiting:
+            record = await _apply(client, raw, origin)
+            if record["kind"] == "seed.applied":
+                yield record
+            else:
+                still.append((origin, raw, record))
+        if len(still) == len(waiting):
+            break
+        waiting = still
+    for _origin, _raw, refusal in waiting:
+        yield refusal
+
+
+def _schema_file(raw: JsonMap, origin: str) -> str | None:
+    """Name the code a standalone JSON Schema file is stored under, or None when the file is not one.
+
+    A file that is neither a document nor a connection file and declares a string ``$id`` is a
+    schema, coded the way ``dg schema create`` codes it: from the ``$id``, else the filename.
     """
-    for directory in directories:
-        files = readable(directory)
-        for path, raw in files:
-            if not is_document(raw):
-                async for record in _connections(client, specs(raw.get("connections")), str(path)):
-                    yield record
-        for path, raw in files:
-            if is_document(raw):
-                async for record in _document(client, raw, str(path)):
-                    yield record
-
-
-async def _document(client: Dirigent, raw: JsonMap, origin: str) -> AsyncIterator[Record]:
-    """Create what one document carries, then apply the document without those sections."""
+    if is_document(raw) or "connections" in raw or not isinstance(raw.get("$id"), str):
+        return None
+    return code_from_id(cast("str", raw["$id"])) or code_from_id(Path(origin.rpartition(":")[2]).name)
+
+
+async def _standalone_schema(client: Dirigent, code: str, body: JsonMap, origin: str) -> Record:
+    """Store one schema file, and say so, or say why the instance would not."""
+    try:
+        await _schema(client, code, body)
+    except DirigentError as error:
+        return _refused(origin, f"schema {code}: {_reason(error)}")
+    return make("seed.schema", at=datetime.now(UTC), message="schema", schema=code, origin=origin)
+
+
+async def _carried(client: Dirigent, raw: JsonMap, origin: str) -> AsyncIterator[Record]:
+    """Create the connections and the schemas one document carries."""
     async for record in _connections(client, specs(raw.get("connections")), origin):
         yield record
     for code, body in _schemas(raw).items():
@@ -86,7 +157,6 @@ async def _document(client: Dirigent, raw: JsonMap, origin: str) -> AsyncIterato
             await _schema(client, code, body)
         except DirigentError as error:
             yield _refused(origin, f"schema {code}: {_reason(error)}")
-    yield await _apply(client, raw, origin)
 
 
 def _schemas(raw: JsonMap) -> dict[str, JsonMap]:
@@ -182,37 +252,22 @@ def _reason(error: DirigentError) -> str:
 
 
 async def seed_installed(client: Dirigent) -> AsyncIterator[Record]:
-    """Apply every document of every installed corpus, the way a directory's documents go in.
+    """Seed every installed corpus the way a directory is seeded, files beside the documents included.
 
-    The catalogue is what this build ships, so nothing is named on the command line and no
-    checkout has to be present. A document is attributed to the plugin that carries it.
+    The corpus is what this build ships, so nothing is named on the command line and no
+    checkout has to be present. A file is attributed to the plugin that carries it.
     """
     from dirigent_core.plugins import load_plugin_host
 
-    pipelines = 0
-    refused = 0
-    connections: set[str] = set()
-    plugins: set[str] = set()
-    for entry in load_plugin_host().examples():
-        parsed = safe_load(entry.source)
-        if not isinstance(parsed, dict):
-            continue
-        plugins.add(entry.plugin)
-        async for record in _document(client, cast("JsonMap", parsed), f"{entry.plugin}:{entry.path}"):
-            kind = record["kind"]
-            if kind == "seed.applied":
-                pipelines += 1
-            elif kind == "seed.refused":
-                refused += 1
-            elif kind == "seed.connection":
-                connections.add(str(record["connection"]))
-            yield record
+    shipped = load_plugin_host().example_files()
+    files = [(f"{one.plugin}:{one.path}", one.raw) for one in shipped]
+    tally = _Tally()
+    async for record in _seed(client, files):
+        yield tally.count(record)
     yield make(
         "seed.done",
         at=datetime.now(UTC),
         message="seeded",
-        plugins=sorted(plugins),
-        pipelines=pipelines,
-        refused=refused,
-        connections=sorted(connections),
+        plugins=sorted({one.plugin for one in shipped if is_document(one.raw)}),
+        **tally.counts(),
     )

dirigent_cli-0.24.0/src/dirigent_cli/source_commands.py

@@ -0,0 +1,203 @@
+"""The source command group: where a pipeline's inputs enter, what has arrived, and calling one."""
+
+import json
+from pathlib import Path
+from typing import Annotated, Final
+
+import httpx2
+import typer
+from pydantic import JsonValue
+
+from dirigent_cli.commands import fail, paged
+from dirigent_cli.context import client_for, state_of
+from dirigent_cli.messages import ONE_INPUT_BODY, SOURCE_NOT_CALLABLE, SOURCE_UNREACHABLE, TAKES_NAME_EQUALS_VALUE
+from dirigent_cli.output import emit_fact, emit_one, emit_records
+from dirigent_client import DEFAULT_CONNECT_TIMEOUT
+from dirigent_plugin.sources import Route
+
+#: Where ``dg source call`` reads the token a source was given when ``--source-token`` names none.
+TOKEN_ENV: Final = "DIRIGENT_SOURCE_TOKEN"
+
+#: How long a call waits for its answer: an endpoint answers once its run exists, not once it settles.
+CALL_TIMEOUT: Final = httpx2.Timeout(30.0, connect=DEFAULT_CONNECT_TIMEOUT)
+
+source_app = typer.Typer(
+    name="source",
+    help="Where a pipeline's inputs enter: what each listens to, what has arrived, and calling one.",
+    no_args_is_help=True,
+)
+
+PipelineArg = Annotated[str, typer.Argument(help="The pipeline the source belongs to.")]
+SourceArg = Annotated[str, typer.Argument(help="The source's step code.")]
+
+
+@source_app.command("list")
+def source_list(
+    ctx: typer.Context,
+    pipeline: Annotated[
+        str | None, typer.Argument(help="The pipeline whose sources to list; every one when unset.")
+    ] = None,
+) -> None:
+    """List sources with their state and what they have taken, never their tokens."""
+    with client_for(state_of(ctx)) as dg:
+        rows = list(paged(lambda after, size: dg.call(dg.sources.list(pipeline, after=after, limit=size)), None))
+    emit_records("source", rows)
+
+
+@source_app.command("show")
+def source_show(ctx: typer.Context, pipeline: PipelineArg, code: SourceArg) -> None:
+    """Show one source: its route, its state, and what it has taken."""
+    with client_for(state_of(ctx)) as dg:
+        row = dg.call(dg.sources.get(pipeline, code))
+    emit_one("source", row)
+
+
+@source_app.command("pause")
+def source_pause(ctx: typer.Context, pipeline: PipelineArg, code: SourceArg) -> None:
+    """Refuse new inputs with 503, keeping the token and the counts; runs in flight carry on."""
+    with client_for(state_of(ctx)) as dg:
+        row = dg.call(dg.sources.pause(pipeline, code))
+    emit_fact("source.paused", message="paused", code=code, pipeline=pipeline, state=row.state.value)
+
+
+@source_app.command("resume")
+def source_resume(ctx: typer.Context, pipeline: PipelineArg, code: SourceArg) -> None:
+    """Take inputs again on the token that was already issued."""
+    with client_for(state_of(ctx)) as dg:
+        row = dg.call(dg.sources.resume(pipeline, code))
+    emit_fact("source.resumed", message="resumed", code=code, pipeline=pipeline, state=row.state.value)
+
+
+@source_app.command("rotate-token")
+def source_rotate(ctx: typer.Context, pipeline: PipelineArg, code: SourceArg) -> None:
+    """Mint a new token and forget the old one; every caller has to be given the new one."""
+    with client_for(state_of(ctx)) as dg:
+        minted = dg.call(dg.sources.rotate_token(pipeline, code))
+        base = dg.url
+    emit_fact(
+        "source.token_rotated",
+        message="rotated",
+        code=code,
+        pipeline=pipeline,
+        url=f"{base.rstrip('/')}{minted.url_path}",
+        token=minted.token,
+    )
+
+
+@source_app.command("inputs")
+def source_inputs(
+    ctx: typer.Context,
+    pipeline: PipelineArg,
+    code: SourceArg,
+    limit: Annotated[int, typer.Option(help="How many inputs to show, newest first.")] = 50,
+) -> None:
+    """List what has arrived at a source, newest first, refusals included."""
+    with client_for(state_of(ctx)) as dg:
+        rows = list(
+            paged(lambda after, size: dg.call(dg.sources.inputs(pipeline, code, after=after, limit=size)), limit)
+        )
+    emit_records("source.input", rows)
+
+
+@source_app.command("call")
+def source_call(
+    ctx: typer.Context,
+    pipeline: PipelineArg,
+    code: SourceArg,
+    data: Annotated[str | None, typer.Option("--body", help="A JSON body, or @file to read one from a file.")] = None,
+    text: Annotated[str | None, typer.Option("--text", help="A text body, or @file to read one from a file.")] = None,
+    form: Annotated[
+        list[str] | None, typer.Option("--form", help="One name=value of a urlencoded form body; repeat for more.")
+    ] = None,
+    method: Annotated[
+        str | None, typer.Option("--method", "-X", help="The method to call with; the source's first unless set.")
+    ] = None,
+    path_params: Annotated[
+        list[str] | None, typer.Option("--path-param", "-p", help="One name=value the path template takes.")
+    ] = None,
+    query: Annotated[list[str] | None, typer.Option("--query", "-q", help="One name=value of the query.")] = None,
+    token: Annotated[
+        str | None,
+        typer.Option("--source-token", envvar=TOKEN_ENV, help="The source's own token, not the CLI's credential."),
+    ] = None,
+) -> None:
+    """Call a received source the way its callers do, and write down what it answered.
+
+    The call carries the source's own token, never this CLI's: an endpoint authenticates its
+    caller as the source, not as a person.
+    """
+    if sum(value is not None for value in (data, text, form)) > 1:
+        fail(ONE_INPUT_BODY)
+    with client_for(state_of(ctx)) as dg:
+        row = dg.call(dg.sources.get(pipeline, code))
+        base = dg.url
+    if row.url_path is None or not row.methods:
+        fail(SOURCE_NOT_CALLABLE, code=repr(code), pipeline=repr(pipeline), kind=repr(row.kind))
+    chosen = (method or row.methods[0]).upper()
+    url = base.rstrip("/") + Route(methods=tuple(row.methods), template=row.url_path).fill(_pairs(path_params))
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    content: bytes | None = None
+    if data is not None:
+        content = _read(data).encode()
+        headers["Content-Type"] = "application/json"
+    elif text is not None:
+        content = _read(text).encode()
+        headers["Content-Type"] = "text/plain; charset=utf-8"
+    try:
+        with httpx2.Client(timeout=CALL_TIMEOUT) as http:
+            answered = http.request(
+                chosen,
+                url,
+                headers=headers,
+                params=_pairs(query),
+                content=content,
+                data=_pairs(form) if form is not None else None,
+            )
+    except httpx2.HTTPError as error:
+        fail(SOURCE_UNREACHABLE, url=url, detail=str(error))
+    emit_fact(
+        "source.called",
+        message=f"answered {answered.status_code}",
+        code=code,
+        pipeline=pipeline,
+        method=chosen,
+        url=url,
+        status=answered.status_code,
+        body=_body_of(answered),
+    )
+    if answered.status_code >= 400:
+        raise typer.Exit(code=1)
+
+
+def _read(value: str) -> str:
+    """Read a value, or the file an ``@`` before it names."""
+    if value.startswith("@"):
+        return Path(value[1:]).expanduser().read_text()
+    return value
+
+
+def _pairs(entries: list[str] | None) -> dict[str, str]:
+    """Read repeated ``name=value`` options into a mapping."""
+    pairs: dict[str, str] = {}
+    for entry in entries or ():
+        name, equals, value = entry.partition("=")
+        if not equals or not name:
+            fail(TAKES_NAME_EQUALS_VALUE, entry=repr(entry))
+        pairs[name] = value
+    return pairs
+
+
+def _body_of(answered: httpx2.Response) -> JsonValue:
+    """Read what an endpoint answered with: JSON when it says so, its text otherwise, nothing when empty."""
+    if not answered.content:
+        return None
+    if "json" in answered.headers.get("content-type", ""):
+        try:
+            parsed: JsonValue = json.loads(answered.content)
+        except ValueError:
+            return answered.text
+        return parsed
+    return answered.text
+
+
+__all__ = ["TOKEN_ENV", "source_app"]

{dirigent_cli-0.23.4 → dirigent_cli-0.24.0}/src/dirigent_cli/summaries.py

@@ -250,6 +250,20 @@ def _issued(record: Record) -> RenderableType | None:
     )
 
 
+def _source_token(record: Record) -> RenderableType | None:
+    """Hand over a source's minted token with where it is presented, and that it is shown once."""
+    token = record.get("token")
+    if not token:
+        return None
+    return Group(
+        "",
+        f"  [bold]{escape(str(record.get('url') or ''))}[/]",
+        f"  [bold]Authorization: Bearer {escape(str(token))}[/]",
+        "",
+        "[yellow]This is the only time the token is shown.[/] The instance stores only its hash.",
+    )
+
+
 def _files(record: Record) -> list[RenderableType]:
     """List what was written, named against the directory it was written into."""
     written = _texts(record, "files")
@@ -467,8 +481,10 @@ def _shape(record: Record) -> RenderableType | None:
 
 
 def _width(row: schemas.StepShape) -> str:
-    """Say how wide a step is, and where one bad item leaves the rest of the batch running."""
+    """Say how wide a step is, whether its items go one by one, and where one bad item leaves the rest running."""
     drawn = escape(str(row.cardinality))
+    if row.item_wise_parent is not None:
+        drawn = f"{drawn}  {muted('item by item')}"
     return f"{drawn}  {muted('continue')}" if row.items == "continue" else drawn
 
 
@@ -548,6 +564,7 @@ RENDERERS: Final[Mapping[str, Callable[[Record], RenderableType | None]]] = {
     "validation.shape": _shape,
     "error": _refusal,
     "token.issued": _issued,
+    "source.token_rotated": _source_token,
     "process": _issued,
     "project.scaffolded": _scaffolded,
     "instance.initialised": _initialised,