dirigent-cli 0.14.1__tar.gz → 0.15.0__tar.gz

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirigent-cli
-Version: 0.14.1
+Version: 0.15.0
 Summary: The dirigent command line interface (dirigent / dg).
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
@@ -8,6 +8,7 @@ Requires-Dist: dirigent-blocks
 Requires-Dist: dirigent-client
 Requires-Dist: dirigent-common
 Requires-Dist: dirigent-core
+Requires-Dist: dirigent-examples
 Requires-Dist: dirigent-plugin
 Requires-Dist: dirigent-server
 Requires-Dist: httpx2>=2.12.0

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-cli"
-version = "0.14.1"
+version = "0.15.0"
 description = "The dirigent command line interface (dirigent / dg)."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,6 +11,7 @@ dependencies = [
     "dirigent-client",
     "dirigent-common",
     "dirigent-core",
+    "dirigent-examples",
     "dirigent-plugin",
     "dirigent-server",
     "httpx2>=2.12.0",
@@ -40,6 +41,9 @@ workspace = true
 [tool.uv.sources.dirigent-core]
 workspace = true
 
+[tool.uv.sources.dirigent-examples]
+workspace = true
+
 [tool.uv.sources.dirigent-plugin]
 workspace = true
 

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-cli"
-version = "0.14.1"
+version = "0.15.0"
 description = "The dirigent command line interface (dirigent / dg)."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,6 +11,7 @@ dependencies = [
     "dirigent-client",
     "dirigent-common",
     "dirigent-core",
+    "dirigent-examples",
     "dirigent-plugin",
     "dirigent-server",
     "httpx2>=2.12.0",
@@ -35,5 +36,6 @@ build-backend = "uv_build"
 dirigent-blocks = { workspace = true }
 dirigent-client = { workspace = true }
 dirigent-core = { workspace = true }
+dirigent-examples = { workspace = true }
 dirigent-plugin = { workspace = true }
 dirigent-server = { workspace = true }

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/src/dirigent_cli/commands.py

@@ -13,7 +13,7 @@ import typer
 import yaml
 from rich.markup import escape
 
-from dirigent_cli import schemas
+from dirigent_cli import schemas, starters
 from dirigent_cli.context import CliState, Session, client_for, state_of
 from dirigent_cli.graph import GraphStep, render_graph, steps_of_document
 from dirigent_cli.local import (
@@ -56,6 +56,7 @@ from dirigent_cli.output import (
 )
 from dirigent_cli.params import ParamError, build_params
 from dirigent_cli.project import (
+    DEFAULT_PIPELINES_DIR,
     InitChoices,
     ProjectError,
     check_choices,
@@ -76,6 +77,8 @@ from dirigent_client import (
     BlockKind,
     Catalog,
     DocumentKind,
+    ExampleDetail,
+    ExampleOut,
     ItemOut,
     LogEntryOut,
     LogLevel,
@@ -103,6 +106,7 @@ from dirigent_core.documents import (
 from dirigent_core.engine.definition import PipelineDefinition, TriggersDefinition, load_definition
 from dirigent_core.engine.runs import RunWindow
 from dirigent_core.engine.state import in_execution_order
+from dirigent_core.examples import STARTER_TAG, ExampleEntry
 from dirigent_core.schemas import code_from_id
 
 RUNS_PAGE = 50
@@ -121,6 +125,7 @@ connection_app = typer.Typer(
 )
 schema_app = typer.Typer(name="schema", help="Named JSON Schemas the instance holds.", no_args_is_help=True)
 blocks_app = typer.Typer(name="blocks", help="The block catalog every plugin contributes to.", no_args_is_help=True)
+examples_app = typer.Typer(name="examples", help="The documents every installed plugin ships.", no_args_is_help=True)
 token_app = typer.Typer(name="token", help="API tokens.", no_args_is_help=True)
 user_app = typer.Typer(name="user", help="Local accounts.", no_args_is_help=True)
 auth_app = typer.Typer(name="auth", help="Logging in, and which identity the CLI holds.", no_args_is_help=True)
@@ -492,6 +497,13 @@ def init_command(
     pack: Annotated[
         list[str] | None, typer.Option("--pack", help="A pack to add, such as dirigent-dhis2. Repeatable.")
     ] = None,
+    pipeline: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--pipeline",
+            help="A starter to copy into pipelines/, as `dg examples list --starter` names it. Repeatable.",
+        ),
+    ] = None,
     admin: Annotated[str, typer.Option(help="The first admin account's username.")] = "admin",
     password: Annotated[str | None, typer.Option(help="Its password; asked for when omitted.")] = None,
 ) -> None:
@@ -511,7 +523,7 @@ def init_command(
 
     root = directory.resolve()
     version = cli_version()
-    unasked = template is None and service is None and pack is None and not workflow
+    unasked = template is None and service is None and pack is None and pipeline is None and not workflow
     asked = unasked and sys.stdin.isatty() and sys.stdout.isatty()
     # Refusing happens before anything is written or asked for, so a run that cannot finish
     # has not half-created a project, and nobody fills a form for a command that was going
@@ -533,6 +545,7 @@ def init_command(
             services=tuple(service) if service is not None else InitChoices().services,
             workflow=workflow,
             packs=tuple(pack or ()),
+            pipelines=tuple(pipeline or ()),
             admin=admin,
             password=password or os.environ.get(BOOTSTRAP_PASSWORD_ENV) or "",
         )
@@ -557,6 +570,7 @@ def init_command(
         **({"services": list(choices.services)} if choices.stack else {}),
         **({"workflow": True} if choices.workflow else {}),
         **({"packs": list(choices.packs)} if choices.packs else {}),
+        **({"pipelines": list(choices.pipelines)} if choices.pipelines else {}),
     }
     if not choices.instance:
         starting = [
@@ -2598,3 +2612,158 @@ def auth_status(ctx: typer.Context) -> None:
         role=me.role.value,
         via=me.via.value if me.via else "-",
     )
+
+
+def installed_examples() -> list[ExampleDetail]:
+    """Read the corpus installed beside this `dg`, without addressing any server."""
+    from dirigent_core.plugins import load_plugin_host
+
+    return [as_detail(entry) for entry in load_plugin_host().examples()]
+
+
+def as_detail(entry: ExampleEntry) -> ExampleDetail:
+    """Render one entry of the installed catalogue the way the API renders one."""
+    return ExampleDetail(**entry.model_dump())
+
+
+def _example_rows(
+    state: CliState,
+    *,
+    local: bool,
+    tags: Sequence[str] = (),
+    shelf: str | None = None,
+    plugin: str | None = None,
+    starter: bool | None = None,
+) -> list[ExampleOut]:
+    """List the catalogue, from the installed corpus or from the instance's own."""
+    if local:
+        wanted = set(tags)
+        return [
+            row
+            for row in installed_examples()
+            if wanted <= set(row.tags)
+            and (shelf is None or row.shelf == shelf)
+            and (plugin is None or row.plugin == plugin)
+            and (starter is None or row.starter is starter)
+        ]
+    with client_for(state) as dg:
+        return list(
+            paged(
+                lambda after, size: dg.call(
+                    dg.examples.list(tags=tags, shelf=shelf, plugin=plugin, starter=starter, after=after, limit=size)
+                ),
+                None,
+            )
+        )
+
+
+def _example(state: CliState, code: str, *, local: bool) -> ExampleDetail:
+    """Resolve one example by code, from the installed corpus or from the instance's own."""
+    if not local:
+        with client_for(state) as dg:
+            return dg.call(dg.examples.get(code))
+    from dirigent_core.plugins import UnknownExample, load_plugin_host
+
+    try:
+        return as_detail(load_plugin_host().example(code))
+    except UnknownExample as error:
+        refuse(str(error), title="No such example")
+        raise typer.Exit(code=1) from error
+
+
+LOCAL_CATALOGUE = "Read the corpus installed beside dg, instead of the instance's own."
+
+
+@examples_app.command("list")
+def examples_list(
+    ctx: typer.Context,
+    tag: Annotated[
+        list[str] | None, typer.Option("--tag", help="Only documents wearing this tag; repeat it to name more.")
+    ] = None,
+    shelf: Annotated[str | None, typer.Option("--shelf", help="Only documents on this shelf.")] = None,
+    plugin: Annotated[str | None, typer.Option("--plugin", help="Only documents this distribution ships.")] = None,
+    starter: Annotated[bool, typer.Option("--starter", help="Only the documents dg pipeline new may copy.")] = False,
+    local: Annotated[bool, typer.Option("--local", help=LOCAL_CATALOGUE)] = False,
+) -> None:
+    """List the documents every installed plugin ships, and which of them are starters."""
+    state = state_of(ctx)
+    rows = _example_rows(
+        state, local=local, tags=tag or [], shelf=shelf, plugin=plugin, starter=True if starter else None
+    )
+    if state.json_output:
+        return emit_records("example", rows)
+    table(
+        "examples",
+        ["code", "name", "tags", "plugin", "starter", "needs"],
+        [
+            [
+                row.code,
+                row.name or "-",
+                " ".join(one for one in row.tags if one != STARTER_TAG) or "-",
+                row.plugin,
+                "[green]*[/]" if row.starter else "",
+                starters.summary(row.requires),
+            ]
+            for row in rows
+        ],
+    )
+
+
+@examples_app.command("show")
+def examples_show(
+    ctx: typer.Context,
+    code: Annotated[str, typer.Argument(help="The example to read.")],
+    local: Annotated[bool, typer.Option("--local", help=LOCAL_CATALOGUE)] = False,
+) -> None:
+    """Print one example's document, verbatim, as the shelf holds it."""
+    state = state_of(ctx)
+    entry = _example(state, code, local=local)
+    if state.json_output:
+        return emit_fact("example.source", message="example", **entry.model_dump(mode="json"))
+    console.print(entry.source, end="", highlight=False, markup=False)
+
+
+@pipeline_app.command("new")
+def pipeline_new(
+    ctx: typer.Context,
+    starter: Annotated[str, typer.Argument(help="The starter to copy, as `dg examples list --starter` names it.")],
+    code: Annotated[
+        str | None, typer.Option("--code", help="Register the copy under this code; the starter's own when omitted.")
+    ] = None,
+    directory: Annotated[Path, typer.Option("--dir", help="Where to write it; pipelines/ when omitted.")] = Path(
+        DEFAULT_PIPELINES_DIR
+    ),
+    local: Annotated[bool, typer.Option("--local", help=LOCAL_CATALOGUE)] = False,
+) -> None:
+    """Copy a starter into this project as a pipeline of its own.
+
+    The copy is the starter's text verbatim, with the top-level `code:` rewritten and the
+    `starter` tag dropped, so every teaching comment in it survives. What it needs from the
+    instance is the copy's own `requires`, which this prints as the list to work through.
+    """
+    entry = _example(state_of(ctx), starter, local=local)
+    if not entry.starter:
+        refuse(
+            f"{entry.code!r} is an example, not a starter",
+            title="Not a starter",
+            problems=[f"a document is copyable only when it wears the {STARTER_TAG!r} tag"],
+        )
+        raise typer.Exit(code=1)
+    new_code = code or entry.code
+    path = directory / f"{new_code}.yaml"
+    if path.exists():
+        refuse(f"{path} is already there", title="Already there", problems=["name another code with --code"])
+        raise typer.Exit(code=1)
+    text = starters.instantiate(entry.source, new_code)
+    directory.mkdir(parents=True, exist_ok=True)
+    path.write_text(text)
+    copied = load_pipeline_text(text).requires
+    emit_fact(
+        "pipeline.created",
+        message="created",
+        path=str(path),
+        code=new_code,
+        starter=entry.code,
+        requires=copied.model_dump(mode="json"),
+        preflight=starters.preflight(copied),
+    )

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/src/dirigent_cli/init_form.py

@@ -9,7 +9,7 @@ Nothing here touches the disk.
 from __future__ import annotations
 
 from pathlib import Path
-from typing import cast
+from typing import TYPE_CHECKING, cast
 
 from textual.app import App, ComposeResult
 from textual.binding import Binding
@@ -20,11 +20,14 @@ from textual.widgets import Button, Checkbox, Input, Label, RadioButton, RadioSe
 from dirigent_cli.project import (
     DEFAULT_SERVICES,
     PACKS,
-    SERVICE_EXAMPLES,
     SERVICES,
     InitChoices,
+    installed_starters,
 )
 
+if TYPE_CHECKING:
+    from dirigent_core.examples import ExampleEntry
+
 MIN_PASSWORD_LENGTH = 8
 
 #: The web UI's dark palette, so the form is the same product as the screen behind the door:
@@ -63,6 +66,24 @@ KINDS = (
 )
 
 
+#: How much of a starter's description fits on the line that offers it.
+DESCRIPTION_WIDTH = 60
+
+
+def starter_label(entry: ExampleEntry) -> str:
+    """Name one starter the way every screen names an addressable thing.
+
+    The title is the name where there is one and the code otherwise, the code is on the line
+    exactly once, and the description is the rest of it.
+    """
+    described = (entry.description or "").strip().splitlines()
+    first = described[0] if described else ""
+    if len(first) > DESCRIPTION_WIDTH:
+        first = first[: DESCRIPTION_WIDTH - 1].rstrip() + "\u2026"
+    trailing = "  ".join(part for part in ([entry.code] if entry.name else []) + ([first] if first else []))
+    return f"{entry.name or entry.code}  [dim]{trailing}[/]" if trailing else entry.name or entry.code
+
+
 class InitForm(App[InitChoices | None]):
     """The scaffolding form: kind, services, packs, workflow, admin, on one screen."""
 
@@ -81,7 +102,7 @@ class InitForm(App[InitChoices | None]):
     Checkbox > .toggle--button { color: $surface; background: $surface; }
     Checkbox.-on > .toggle--button { color: $success; background: $surface; }
     #kind { height: auto; }
-    #services, #packs { height: auto; border: round $border; }
+    #services, #packs, #starters { height: auto; max-height: 12; border: round $border; }
     #admin-block { height: auto; }
     #admin-row { height: auto; }
     #admin-row Input { width: 1fr; margin-right: 2; }
@@ -105,6 +126,7 @@ class InitForm(App[InitChoices | None]):
         self._version = version
         self._admin = admin
         self._password = password
+        self._starters = installed_starters()
 
     def compose(self) -> ComposeResult:
         """Lay the whole form out on one screen."""
@@ -128,6 +150,12 @@ class InitForm(App[InitChoices | None]):
                 id="services",
             )
 
+            yield Label("First pipelines  (space toggles; copied from the installed corpus)", classes="section")
+            yield SelectionList[str](
+                *((starter_label(entry), entry.code) for entry in self._starters),
+                id="starters",
+            )
+
             yield Label("Packs  (space toggles; pinned at this version)", classes="section")
             yield SelectionList[str](*((f"{pack.name}  [dim]{pack.what}[/]", pack.name) for pack in PACKS), id="packs")
 
@@ -185,8 +213,10 @@ class InitForm(App[InitChoices | None]):
         # subscripted generic: the widgets are fetched untyped and narrowed here.
         services = cast("SelectionList[str]", self.query_one("#services"))
         packs = cast("SelectionList[str]", self.query_one("#packs"))
+        chosen = cast("SelectionList[str]", self.query_one("#starters"))
         picked_services: list[str] = list(services.selected)
         picked_packs: list[str] = list(packs.selected)
+        picked_starters: list[str] = list(chosen.selected)
         kind = self.kind
         chosen_services = tuple(code for code in (s.code for s in SERVICES) if code in picked_services)
         return InitChoices(
@@ -194,6 +224,7 @@ class InitForm(App[InitChoices | None]):
             services=chosen_services if kind == "compose" else DEFAULT_SERVICES,
             workflow=self.query_one("#workflow", Checkbox).value,
             packs=tuple(name for name in (p.name for p in PACKS) if name in picked_packs),
+            pipelines=tuple(entry.code for entry in self._starters if entry.code in picked_starters),
             admin=self.query_one("#admin", Input).value.strip() or "admin",
             password=self.query_one("#password", Input).value,
         )
@@ -203,13 +234,12 @@ class InitForm(App[InitChoices | None]):
         choices = self._collect()
         files = [
             "dirigent.yaml",
-            "pipelines/hello-world.yaml",
+            *(f"pipelines/{code}.yaml" for code in choices.pipelines),
             ".dirigent/profiles.yaml",
             "pyproject.toml",
             "README.md",
         ]
         if choices.stack:
-            files += [f"pipelines/{SERVICE_EXAMPLES[code][0]}" for code in choices.services]
             files += ["compose.yaml", "Dockerfile", ".env"]
         elif choices.instance:
             files += [".env", ".dirigent/state/"]

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/src/dirigent_cli/main.py

@@ -87,6 +87,7 @@ docker_app = typer.Typer(name="docker", help="The docker daemon this host's work
 app.add_typer(commands.runs_app, rich_help_panel=RUN_PANEL)
 app.add_typer(commands.pipeline_app, rich_help_panel=DEFINE_PANEL)
 app.add_typer(commands.blocks_app, rich_help_panel=DEFINE_PANEL)
+app.add_typer(commands.examples_app, rich_help_panel=DEFINE_PANEL)
 app.add_typer(commands.schema_app, rich_help_panel=DEFINE_PANEL)
 app.add_typer(commands.connection_app, rich_help_panel=CONNECT_PANEL)
 app.add_typer(triggers.schedule_app, rich_help_panel=TRIGGER_PANEL)
@@ -824,6 +825,13 @@ def dev(
             "with its schedules paused; name it more than once to seed one directory after another.",
         ),
     ] = None,
+    seed_installed: Annotated[
+        bool,
+        typer.Option(
+            "--seed-installed",
+            help="Apply every document of every installed corpus, naming no directory at all.",
+        ),
+    ] = False,
 ) -> None:
     """Run the API and an embedded worker in one process, on SQLite, with no dependencies.
 
@@ -832,6 +840,9 @@ def dev(
     An instance that is there, made by dg init or by an earlier start, is the one that runs;
     --wipe-state deletes it first, and only a directory dirigent named itself is removed.
 
+    --seed-installed does the same with the corpus every installed plugin ships, so a
+    checkout is not needed; both may be given, and the directories go in first.
+
     --seed fills the instance from a directory of documents the moment it answers: the
     connections a file or a document declares are created first, then every document is
     applied with its schedules paused. A document an instance will not store is reported
@@ -870,8 +881,8 @@ def dev(
     bound = f"http://{address}:{listening}"
     emit(dev_started(settings, bound=bound, admin=admin, token=token, migrated=migrated))
     directories = seed or []
-    bearer = token or (asyncio.run(seed_token(settings)) if directories else None)
-    asyncio.run(_dev(settings, address, listening, seed=directories, bearer=bearer))
+    bearer = token or (asyncio.run(seed_token(settings)) if directories or seed_installed else None)
+    asyncio.run(_dev(settings, address, listening, seed=directories, installed=seed_installed, bearer=bearer))
 
 
 def clear_state(settings: Settings) -> Path | None:
@@ -1035,6 +1046,7 @@ async def _dev(
     port: int,
     *,
     seed: Sequence[Path] = (),
+    installed: bool = False,
     bearer: str | None = None,
 ) -> None:
     """Run the API, the scheduler, a worker, and any seeding as tasks in one event loop.
@@ -1054,7 +1066,9 @@ async def _dev(
     api = uvicorn.Server(server_config)
     worker_task = asyncio.create_task(worker.run())
     ready = asyncio.create_task(_announce_ready(api))
-    seeding = asyncio.create_task(_seed(api, local_url(host, port), bearer, seed)) if seed else None
+    seeding = (
+        asyncio.create_task(_seed(api, local_url(host, port), bearer, seed, installed)) if seed or installed else None
+    )
     try:
         await api.serve()
     finally:
@@ -1081,19 +1095,29 @@ async def _announce_ready(api: "uvicorn.Server") -> None:
     emit(make("process", at=datetime.now(UTC), message="ready", process="dev"))
 
 
-async def _seed(api: "uvicorn.Server", url: str, bearer: str | None, directories: Sequence[Path]) -> None:
-    """Apply what --seed named, once the port is accepting, and write a record for each."""
+async def _seed(
+    api: "uvicorn.Server",
+    url: str,
+    bearer: str | None,
+    directories: Sequence[Path],
+    installed: bool = False,
+) -> None:
+    """Apply what the seeding flags named, once the port is accepting, and record each document."""
     import asyncio
 
-    from dirigent_cli.seeding import seed_directories
+    from dirigent_cli.seeding import seed_directories, seed_installed
     from dirigent_client import Dirigent, DirigentError
 
     while not api.started:
         await asyncio.sleep(0.05)
     try:
         async with Dirigent(url=url, token=bearer) as client:
-            async for record in seed_directories(client, directories):
-                emit(record)
+            if directories:
+                async for record in seed_directories(client, directories):
+                    emit(record)
+            if installed:
+                async for record in seed_installed(client):
+                    emit(record)
     except DirigentError as error:
         emit(make("error", at=datetime.now(UTC), level="error", message=f"seeding stopped: {error.message}"))
 

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/src/dirigent_cli/project.py

@@ -3,14 +3,18 @@
 import base64
 import os
 import re
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import yaml
 from pydantic import BaseModel, ConfigDict, Field
 
 from dirigent_core.configdocs import example_document, project_document
 
+if TYPE_CHECKING:
+    from dirigent_core.examples import ExampleEntry
+
 #: The one file a project hand-edits: where its documents are, and what this instance sets.
 PROJECT_FILE: Final = "dirigent.yaml"
 
@@ -157,28 +161,6 @@ jobs:
           DG_TOKEN: ${{ secrets.DG_TOKEN }}
 """
 
-EXAMPLE_TEMPLATE = """\
-# The smallest thing dirigent can run: one step, one block, no parameters.
-#
-# value.const emits its configured value and touches nothing, so this runs with nothing
-# on the unsafe allowlist:
-#
-#   dg apply
-#   dg run hello-world --watch
-
-format: dirigent/v1
-kind: pipeline
-code: hello-world
-name: Hello world
-description: Emit a greeting, and nothing else.
-
-steps:
-  greet:
-    block: value.const
-    config:
-      value: "hello from dirigent"
-"""
-
 
 class Service(BaseModel):
     """One optional service of the container stack, as the form and the flag name it."""
@@ -201,184 +183,6 @@ SERVICES: Final = (
 
 DEFAULT_SERVICES: Final = ("s3",)
 
-#: The example each service brings into `pipelines/`, so a stack with the service has one
-#: document that uses it the day it is made.
-SERVICE_EXAMPLES: Final = {
-    "s3": (
-        "s3-hello.yaml",
-        """\
-# A greeting written to the stack's own bucket and read back, through the s3:// scheme.
-#
-# There is no S3 block: storage.write puts text in an object, storage.copy moves bytes between
-# schemes, and the stack's `migrate` service bootstraps the `artifacts` connection that serves
-# s3://. The bucket comes from the URI, so this one is the stack's:
-#
-#   dg run s3-hello --watch
-
-format: dirigent/v1
-kind: pipeline
-code: s3-hello
-name: Hello, object storage
-description: Write a greeting to the bucket and copy it back, through s3://.
-
-requires:
-  blocks:
-    - storage.write
-    - storage.copy
-  connections:
-    - artifacts
-  storage:
-    - s3
-
-steps:
-  greet:
-    block: storage.write
-    config:
-      target: "${run.scratch}/hello.txt"
-      text: "hello from object storage"
-
-  upload:
-    block: storage.copy
-    depends_on: [greet]
-    config:
-      source: "${steps.greet.output.uri}"
-      target: "s3://dirigent/hello/${run.id}.txt"
-
-  download:
-    block: storage.copy
-    depends_on: [upload]
-    config:
-      source: "s3://dirigent/hello/${run.id}.txt"
-      target: "${run.scratch}/hello-back.txt"
-""",
-    ),
-    "docker": (
-        "docker-hello.yaml",
-        """\
-# A command run in a container on the workers' own daemon, the `docker` service.
-#
-# docker.run is on the stack's allowlist (DIRIGENT_ENABLED_UNSAFE_BLOCKS in .env) because the
-# daemon it reaches is the sidecar, never the host's. The image is pulled first, gets no
-# network, and is capped in memory and processes:
-#
-#   dg run docker-hello --watch
-
-format: dirigent/v1
-kind: pipeline
-code: docker-hello
-name: Hello from a container
-description: Run a command in a container, with no network and the image pulled first.
-
-requires:
-  blocks:
-    - docker.run
-  workers:
-    - docker
-
-steps:
-  greet:
-    block: docker.run
-    deadline: 5m
-    config:
-      image: alpine:3
-      pull: true
-      network: none
-      argv: [echo, "hello from a container"]
-      memory: 64mb
-      pids_limit: 64
-""",
-    ),
-    "kafka": (
-        "kafka-hello.yaml",
-        """\
-# Three records published to the stack's Kafka broker and read back off the topic.
-#
-# The `migrate` service bootstraps the `kafka` connection and the `kafka-topic` service
-# creates the `hello` topic, so nothing has to be arranged first:
-#
-#   dg run kafka-hello --watch
-
-format: dirigent/v1
-kind: pipeline
-code: kafka-hello
-name: Hello, Kafka
-description: Publish records to a topic and consume the same batch back.
-
-requires:
-  blocks:
-    - kafka.produce
-    - kafka.consume
-  connections:
-    - kafka
-
-steps:
-  publish:
-    block: kafka.produce
-    config:
-      connection: kafka
-      topic: hello
-      records:
-        - {greeting: hello, n: 1}
-        - {greeting: hello, n: 2}
-        - {greeting: hello, n: 3}
-      timeout: 30s
-
-  consume:
-    block: kafka.consume
-    depends_on: [publish]
-    poll: 5s
-    deadline: 5m
-    config:
-      connection: kafka
-      topic: hello
-      start: earliest
-      min_messages: 3
-      max_messages: 50
-      poll_timeout: 5s
-      value_format: json
-""",
-    ),
-    "rabbitmq": (
-        "rabbitmq-hello.yaml",
-        """\
-# A run that waits for a message on the stack's RabbitMQ queue, then hands the batch on.
-#
-# The `migrate` service bootstraps the `rabbitmq` connection and the `rabbitmq-queue`
-# service declares the `hello` queue. Publish something to it from the management UI at
-# http://127.0.0.1:15672 (dirigent / dirigent), and the run that is waiting takes it:
-#
-#   dg run rabbitmq-hello --watch
-
-format: dirigent/v1
-kind: pipeline
-code: rabbitmq-hello
-name: Hello, RabbitMQ
-description: Wait for a message on a queue and hand the batch on.
-
-requires:
-  blocks:
-    - rabbitmq.consume
-  connections:
-    - rabbitmq
-
-steps:
-  wait:
-    block: rabbitmq.consume
-    poll: 10s
-    deadline: 1h
-    on_timeout: skip
-    config:
-      connection: rabbitmq
-      queue: hello
-      min_messages: 1
-      max_messages: 50
-      poll_timeout: 5s
-      ack: on_success
-      value_format: json
-""",
-    ),
-}
-
 
 class Pack(BaseModel):
     """One published pack the form and the flag can add to a project."""
@@ -400,6 +204,26 @@ TEMPLATES: Final = ("local", "compose", "documents")
 COMPOSE_TEMPLATE_NAME: Final = "compose"
 
 
+def installed_starters() -> "list[ExampleEntry]":
+    """List the starters installed beside ``dg`` itself, which is what a new project may copy.
+
+    ``dg init`` writes the ``pyproject.toml`` that installs the packs, so a pack's own
+    starters are not installed yet when it runs; those arrive with ``dg pipeline new``.
+    """
+    from dirigent_core.examples import STARTER_TAG
+    from dirigent_core.plugins import load_plugin_host
+
+    return [entry for entry in load_plugin_host().examples() if STARTER_TAG in entry.tags]
+
+
+def starter_documents(codes: Sequence[str]) -> dict[str, str]:
+    """Read each chosen starter as the text a copy of it writes, keyed by its code."""
+    from dirigent_cli.starters import instantiate
+
+    held = {entry.code: entry for entry in installed_starters()}
+    return {code: instantiate(held[code].source, code) for code in codes if code in held}
+
+
 class InitChoices(BaseModel):
     """Everything ``dg init`` decides, from the form or from the flags, before it writes."""
 
@@ -409,6 +233,9 @@ class InitChoices(BaseModel):
     services: tuple[str, ...] = DEFAULT_SERVICES
     workflow: bool = False
     packs: tuple[str, ...] = ()
+    pipelines: tuple[str, ...] = ()
+    """The starters this project opens with, copied from the corpus installed beside ``dg``."""
+
     admin: str = "admin"
     password: str = ""
 
@@ -441,6 +268,13 @@ def check_choices(choices: InitChoices) -> None:
             raise ProjectError(f"no pack named {pack!r}; the packs are {', '.join(sorted(packs))}")
     if choices.services != DEFAULT_SERVICES and not choices.stack:
         raise ProjectError("--service applies to the compose template alone; the other two run no stack")
+    starters = {entry.code for entry in installed_starters()}
+    for code in choices.pipelines:
+        if code not in starters:
+            raise ProjectError(
+                f"no starter named {code!r}; a document is copyable only when it wears the 'starter' tag, "
+                f"and `dg examples list --starter` names the ones installed here"
+            )
 
 
 COMPOSE_HEAD = """\
@@ -611,7 +445,7 @@ COMPOSE_S3 = """\
 
   # The bucket has to exist before a run writes to it, and nothing else creates it.
   s3-bucket:
-    image: minio/mc:RELEASE.2025-04-16T18-13-26Z
+    image: quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z
     depends_on:
       s3:
         condition: service_healthy
@@ -1030,20 +864,20 @@ README_RUN: Final = {
     "local": (
         "uv sync",
         "uv run dg dev",
+        "uv run dg pipeline new <starter>",
         "uv run dg apply",
-        "uv run dg run hello-world --watch",
     ),
     "documents": (
         "uv sync",
+        "uv run dg pipeline new <starter>",
         "uv run dg apply --dry-run",
         "uv run dg apply",
-        "uv run dg run hello-world --watch",
     ),
     "compose": (
         "uv sync",
         "docker compose up -d",
         "uv run dg auth login --username admin",
-        "uv run dg run hello-world --watch",
+        "uv run dg pipeline new <starter>",
     ),
 }
 
@@ -1070,11 +904,9 @@ def scaffold(directory: Path, choices: InitChoices, *, version: str = "0.0.0") -
     skipped: list[Path] = []
     directory.mkdir(parents=True, exist_ok=True)
     written.append(_write(directory / PROJECT_FILE, PROJECT_TEMPLATE + "\n" + project_document()))
-    written.append(_write(directory / DEFAULT_PIPELINES_DIR / "hello-world.yaml", EXAMPLE_TEMPLATE))
-    if choices.stack:
-        for code in choices.services:
-            filename, document = SERVICE_EXAMPLES[code]
-            written.append(_write(directory / DEFAULT_PIPELINES_DIR / filename, document))
+    (directory / DEFAULT_PIPELINES_DIR).mkdir(parents=True, exist_ok=True)
+    for code, document in starter_documents(choices.pipelines).items():
+        written.append(_write(directory / DEFAULT_PIPELINES_DIR / f"{code}.yaml", document))
     written.append(_write(directory / ".dirigent" / "profiles.yaml", PROFILES_TEMPLATE))
     written.append(_write(directory / ".dirigent" / ".gitignore", STATE_IGNORE_TEMPLATE))
     written.append(_write(directory / EXAMPLE_CONFIG_FILE, example_document()))

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/src/dirigent_cli/seeding.py

@@ -8,43 +8,15 @@ documents an instance will not store.
 from collections.abc import AsyncIterator, Mapping, Sequence
 from datetime import UTC, datetime
 from pathlib import Path
-from typing import Any, Final, cast
-
-import yaml
+from typing import Any, cast
 
 from dirigent_cli.local import ConnectionSpec
 from dirigent_client import Dirigent, DirigentError, PlanAction, ProvenanceSource
 from dirigent_common import JsonMap
-from dirigent_core.documents import safe_load
-from dirigent_core.engine.definition import FORMAT_V1
+from dirigent_core.documents import CARRIED, SUFFIXES, is_document, readable, safe_load
 from dirigent_core.protocol import Record, make
 
-#: The file endings a seed reads. Anything else under a seed directory is passed over, and so
-#: is a file of one of these that does not parse as a mapping.
-SUFFIXES: Final = (".yaml", ".yml", ".json")
-
-#: The sections a document may carry so that it runs alone under ``dg run --local``. An
-#: instance refuses to store a document carrying one, so the seed creates what they declare
-#: and applies the document without them.
-CARRIED: Final = ("connections", "schemas")
-
-
-def readable(directory: Path) -> list[tuple[Path, JsonMap]]:
-    """Read every file under a directory that parses as a mapping, in a stable order."""
-    found: list[tuple[Path, JsonMap]] = []
-    for path in sorted(one for one in directory.rglob("*") if one.suffix in SUFFIXES and one.is_file()):
-        try:
-            parsed = safe_load(path.read_text())
-        except (OSError, UnicodeDecodeError, yaml.YAMLError):
-            continue
-        if isinstance(parsed, dict):
-            found.append((path, cast("JsonMap", parsed)))
-    return found
-
-
-def is_document(raw: JsonMap) -> bool:
-    """Say whether a parsed file is a document this instance reads."""
-    return raw.get("format") == FORMAT_V1
+__all__ = ["CARRIED", "SUFFIXES", "is_document", "readable", "seed_directories", "seed_installed", "specs"]
 
 
 def specs(declared: object) -> list[ConnectionSpec]:
@@ -196,3 +168,40 @@ def _refused(origin: str, reason: str) -> Record:
 def _reason(error: DirigentError) -> str:
     """Read a refusal as the one line a record carries."""
     return "; ".join(error.problems) or error.message
+
+
+async def seed_installed(client: Dirigent) -> AsyncIterator[Record]:
+    """Apply every document of every installed corpus, the way a directory's documents go in.
+
+    The catalogue is what this build ships, so nothing is named on the command line and no
+    checkout has to be present. A document is attributed to the plugin that carries it.
+    """
+    from dirigent_core.plugins import load_plugin_host
+
+    pipelines = 0
+    refused = 0
+    connections: set[str] = set()
+    plugins: set[str] = set()
+    for entry in load_plugin_host().examples():
+        parsed = safe_load(entry.source)
+        if not isinstance(parsed, dict):
+            continue
+        plugins.add(entry.plugin)
+        async for record in _document(client, cast("JsonMap", parsed), f"{entry.plugin}:{entry.path}"):
+            kind = record["kind"]
+            if kind == "seed.applied":
+                pipelines += 1
+            elif kind == "seed.refused":
+                refused += 1
+            elif kind == "seed.connection":
+                connections.add(str(record["connection"]))
+            yield record
+    yield make(
+        "seed.done",
+        at=datetime.now(UTC),
+        message="seeded",
+        plugins=sorted(plugins),
+        pipelines=pipelines,
+        refused=refused,
+        connections=sorted(connections),
+    )

dirigent_cli-0.15.0/src/dirigent_cli/starters.py

@@ -0,0 +1,121 @@
+"""Copying a starter: the two lines a copy rewrites, and nothing else.
+
+A starter is a plain ``dirigent/v1`` document, not a template, so instantiating one is a
+verbatim copy of its text with the top-level ``code:`` changed and ``starter`` taken off the
+top-level ``tags:``. The edit is at text level rather than through a parser because the
+teaching comments, the blank lines and the quoting are the point of copying a document
+instead of generating one.
+"""
+
+import re
+from typing import Final
+
+from dirigent_client import Requirements
+from dirigent_core.examples import STARTER_TAG
+
+#: The top-level ``code:`` line: no indentation, so a step's own ``code`` is never touched.
+_CODE = re.compile(r"^code:[^\S\n]*(.*)$", re.MULTILINE)
+
+#: The top-level ``tags:`` line, and whatever it carries on the same line.
+_TAGS = re.compile(r"^tags:[^\S\n]*(.*)$", re.MULTILINE)
+
+#: One entry of a block list under ``tags:``: two spaces, a dash, the value.
+_TAG_ITEM = re.compile(r"^[^\S\n]*-[^\S\n]*(\S.*?)[^\S\n]*$")
+
+_FLOW: Final = ("[", "]")
+
+
+def instantiate(source: str, code: str) -> str:
+    """Copy a starter's text under a new code, with the ``starter`` tag dropped."""
+    return _retag(_recode(source, code))
+
+
+def _recode(source: str, code: str) -> str:
+    """Rewrite the one top-level ``code:`` line, leaving every other line alone."""
+    return _CODE.sub(lambda _: f"code: {code}", source, count=1)
+
+
+def _retag(source: str) -> str:
+    """Take ``starter`` off the top-level ``tags:``, whichever shape the list is written in.
+
+    A document whose only tag was ``starter`` loses the whole ``tags:`` entry: an empty list
+    says less than no list at all.
+    """
+    found = _TAGS.search(source)
+    if found is None:
+        return source
+    rest = found.group(1).strip()
+    if rest.startswith(_FLOW[0]):
+        return _rewrite_flow(source, found.start(), found.end(), rest)
+    if rest:
+        return source
+    return _rewrite_block(source, found.start(), found.end())
+
+
+def _rewrite_flow(source: str, start: int, end: int, rest: str) -> str:
+    """Rewrite ``tags: [a, b, starter]``, dropping the entry when nothing is left."""
+    inner = rest.removeprefix(_FLOW[0]).removesuffix(_FLOW[1])
+    kept = [tag for tag in (one.strip() for one in inner.split(",")) if tag and tag != STARTER_TAG]
+    if not kept:
+        return _drop_line(source, start, end)
+    return source[:start] + f"tags: [{', '.join(kept)}]" + source[end:]
+
+
+def _rewrite_block(source: str, start: int, end: int) -> str:
+    """Rewrite a block list under ``tags:``, dropping the whole entry when nothing is left."""
+    lines = source[end:].split("\n")
+    items: list[tuple[int, str]] = []
+    for index, line in enumerate(lines):
+        if index == 0 and not line.strip():
+            continue
+        matched = _TAG_ITEM.match(line)
+        if matched is None:
+            break
+        items.append((index, matched.group(1)))
+    if not items:
+        return source
+    dropped = [index for index, tag in items if tag == STARTER_TAG]
+    if not dropped:
+        return source
+    if len(dropped) == len(items):
+        return _drop_line(source, start, end + len("\n".join(lines[: items[-1][0] + 1])))
+    kept = [line for index, line in enumerate(lines) if index not in dropped]
+    return source[:end] + "\n".join(kept)
+
+
+def _drop_line(source: str, start: int, end: int) -> str:
+    """Remove a whole entry, including the newline that ended it."""
+    tail = source[end:]
+    return source[:start] + tail.removeprefix("\n")
+
+
+#: How a requirement's field is named when a summary counts it, singular and plural.
+_COUNTED: Final = (
+    ("connections", "connection", "connections"),
+    ("schemas", "schema", "schemas"),
+    ("pipelines", "pipeline", "pipelines"),
+    ("storage", "storage scheme", "storage schemes"),
+    ("blocks", "block", "blocks"),
+    ("workers", "worker tag", "worker tags"),
+)
+
+
+def summary(requires: Requirements) -> str:
+    """Say what a document needs in one line: the counts, or nothing when it needs nothing."""
+    counted = [
+        f"{len(held)} {one if len(held) == 1 else many}"
+        for name, one, many in _COUNTED
+        if (held := getattr(requires, name))
+    ]
+    return ", ".join(counted) or "-"
+
+
+def preflight(requires: Requirements) -> list[str]:
+    """List what has to exist on the instance before a copy of this document will apply."""
+    steps = [f"dg connection create KIND {code}" for code in requires.connections]
+    steps += [f"dg schema create {code}.json --code {code}" for code in requires.schemas]
+    steps += [f"apply the pipeline {code} it starts" for code in requires.pipelines]
+    steps += [f"a storage backend claiming {scheme}://" for scheme in requires.storage]
+    steps += [f"a pack contributing {block}" for block in requires.blocks]
+    steps += [f"a worker carrying the {tag} tag" for tag in requires.workers]
+    return steps

{dirigent_cli-0.14.1 → dirigent_cli-0.15.0}/src/dirigent_cli/summaries.py

@@ -36,7 +36,20 @@ from dirigent_core.protocol import Record
 #: Fields a record carries for what is drawn beneath its line rather than for the line. A
 #: line carries whole values, and a run's every step is not a line's worth of them.
 BULKY: Final = frozenset(
-    {"steps", "failures", "windows", "packages", "settings", "history", "problems", "issues", "files", "token"}
+    {
+        "steps",
+        "failures",
+        "windows",
+        "packages",
+        "settings",
+        "history",
+        "problems",
+        "issues",
+        "files",
+        "token",
+        "preflight",
+        "requires",
+    }
 )
 
 
@@ -299,14 +312,31 @@ def _initialised(record: Record) -> RenderableType | None:
         "\n  [bold]uv sync[/]"
         "\n  [bold]uv run dg dev[/]"
         f"\nThe UI is at http://127.0.0.1:3333 and {admin} logs in with the password you gave."
-        "\n\nThen, in this directory, apply the example and run it:"
+        "\n\nThen pick a first pipeline out of the corpus, apply it and run it:"
+        "\n  [bold]uv run dg examples list --starter[/]"
+        "\n  [bold]uv run dg pipeline new <starter>[/]"
         "\n  [bold]uv run dg apply[/]"
-        "\n  [bold]uv run dg run hello-world --watch[/]"
         "\n\n[dim]No DIRIGENT_SECRET_KEY is set, so a connection carrying a credential cannot be stored until it is.[/]"
     )
     return Group(*parts)
 
 
+def _created(record: Record) -> RenderableType | None:
+    """Render a copied starter: where it landed, and what the instance must hold first."""
+    path = record.get("path")
+    if not path:
+        return None
+    parts: list[RenderableType] = [
+        f"Copied [bold]{escape(str(record.get('starter')))}[/] to [bold]{escape(str(path))}[/]."
+    ]
+    steps = _texts(record, "preflight")
+    if steps:
+        parts.append("\nIt will not apply until the instance holds what it needs:")
+        parts.extend(f"  [yellow]-[/] {escape(step)}" for step in steps)
+    parts.append(f"\nThen:\n  [bold]dg apply {escape(str(record.get('code')))}[/]")
+    return Group(*parts)
+
+
 def _config(record: Record) -> RenderableType | None:
     """Render the effective configuration as a setting-per-row table."""
     settings = record.get("settings")
@@ -411,4 +441,5 @@ RENDERERS: Final[Mapping[str, Callable[[Record], RenderableType | None]]] = {
     "process": _issued,
     "project.scaffolded": _scaffolded,
     "instance.initialised": _initialised,
+    "pipeline.created": _created,
 }