dirigent-cli 0.13.0__tar.gz → 0.14.0__tar.gz

{dirigent_cli-0.13.0 → dirigent_cli-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirigent-cli
-Version: 0.13.0
+Version: 0.14.0
 Summary: The dirigent command line interface (dirigent / dg).
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE

{dirigent_cli-0.13.0 → dirigent_cli-0.14.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-cli"
-version = "0.13.0"
+version = "0.14.0"
 description = "The dirigent command line interface (dirigent / dg)."
 readme = "README.md"
 requires-python = ">=3.13"

{dirigent_cli-0.13.0 → dirigent_cli-0.14.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-cli"
-version = "0.13.0"
+version = "0.14.0"
 description = "The dirigent command line interface (dirigent / dg)."
 readme = "README.md"
 requires-python = ">=3.13"

{dirigent_cli-0.13.0 → dirigent_cli-0.14.0}/src/dirigent_cli/main.py

@@ -192,6 +192,10 @@ DEV_ADMIN = "dev"
 DEV_PASSWORD = "dirigent-dev"  # noqa: S105
 DEV_TOKEN_NAME = "dev"
 
+#: The credential `dg dev --seed` authenticates its own applies with, and how long it lasts.
+SEED_TOKEN_NAME = "dev seed"
+SEED_TOKEN_LIFETIME = timedelta(hours=1)
+
 SCHEDULER_ENV = "DIRIGENT_SCHEDULER_ENABLED"
 
 UI_ENV = "DIRIGENT_UI_ENABLED"
@@ -812,6 +816,14 @@ def dev(
             help="Delete the state directory before starting, instead of running the instance that is there.",
         ),
     ] = False,
+    seed: Annotated[
+        list[Path] | None,
+        typer.Option(
+            "--seed",
+            help="Apply every dirigent/v1 document under this directory once the API answers, "
+            "with its schedules paused; name it more than once to seed one directory after another.",
+        ),
+    ] = None,
 ) -> None:
     """Run the API and an embedded worker in one process, on SQLite, with no dependencies.
 
@@ -819,6 +831,11 @@ def dev(
     starting this somewhere else means a different instance, with none of the same runs.
     An instance that is there, made by dg init or by an earlier start, is the one that runs;
     --wipe-state deletes it first, and only a directory dirigent named itself is removed.
+
+    --seed fills the instance from a directory of documents the moment it answers: the
+    connections a file or a document declares are created first, then every document is
+    applied with its schedules paused. A document an instance will not store is reported
+    and passed over, because a corpus holds those on purpose.
     """
     import asyncio
     import os
@@ -848,9 +865,13 @@ def dev(
             emit(state_cleared(cleared))
     migrated = _migrate_quietly(settings)
     admin, token = asyncio.run(dev_admin(settings))
-    bound = f"http://{host or settings.host}:{port or settings.port}"
+    address = host or settings.host
+    listening = port or settings.port
+    bound = f"http://{address}:{listening}"
     emit(dev_started(settings, bound=bound, admin=admin, token=token, migrated=migrated))
-    asyncio.run(_dev(settings, host or settings.host, port or settings.port))
+    directories = seed or []
+    bearer = token or (asyncio.run(seed_token(settings)) if directories else None)
+    asyncio.run(_dev(settings, address, listening, seed=directories, bearer=bearer))
 
 
 def clear_state(settings: Settings) -> Path | None:
@@ -980,10 +1001,46 @@ async def dev_admin(settings: Settings) -> tuple[str | None, str | None]:
         await engine.dispose()
 
 
-async def _dev(settings: Settings, host: str, port: int) -> None:
-    """Run the API, the scheduler, and a worker as tasks in one event loop.
+async def seed_token(settings: Settings) -> str:
+    """Mint the short-lived admin token the seeding authenticates with.
 
-    The scheduler is started by the application's own lifespan, not here.
+    The seeding goes through the API rather than the database, so it needs a bearer even
+    though it runs inside the process that is serving. A --keep-state instance kept its own
+    admin and handed its token over long ago, so one is minted here for the seeding alone.
+    """
+    from dirigent_core.auth import issue_token, list_users
+    from dirigent_core.database import create_engine, create_session_factory, session_scope
+
+    engine = create_engine(settings)
+    try:
+        async with session_scope(create_session_factory(engine)) as session:
+            admins = [user for user in await list_users(session) if user.role is UserRole.ADMIN and user.active]
+            if not admins:
+                refuse(
+                    "dg dev --seed has no admin account to seed as",
+                    status=commands.GUARD_EXIT,
+                    title="No admin",
+                    problems=["create one with dg admin user create --role admin"],
+                )
+                raise typer.Exit(code=commands.GUARD_EXIT)
+            issued = await issue_token(session, admins[0], name=SEED_TOKEN_NAME, lifetime=SEED_TOKEN_LIFETIME)
+            return issued.secret.get_secret_value()
+    finally:
+        await engine.dispose()
+
+
+async def _dev(
+    settings: Settings,
+    host: str,
+    port: int,
+    *,
+    seed: Sequence[Path] = (),
+    bearer: str | None = None,
+) -> None:
+    """Run the API, the scheduler, a worker, and any seeding as tasks in one event loop.
+
+    The scheduler is started by the application's own lifespan, not here. The seeding is a
+    task rather than a step before the loop, because it talks to the API this loop serves.
     """
     import asyncio
 
@@ -997,15 +1054,24 @@ async def _dev(settings: Settings, host: str, port: int) -> None:
     api = uvicorn.Server(server_config)
     worker_task = asyncio.create_task(worker.run())
     ready = asyncio.create_task(_announce_ready(api))
+    seeding = asyncio.create_task(_seed(api, local_url(host, port), bearer, seed)) if seed else None
     try:
         await api.serve()
     finally:
         ready.cancel()
+        if seeding is not None:
+            seeding.cancel()
         worker.request_stop()
         await worker_task
         await engine.dispose()
 
 
+def local_url(host: str, port: int) -> str:
+    """Address the instance this process serves, however widely it is bound."""
+    wildcards = {"0.0.0.0", "::", ""}  # noqa: S104 - naming the bind-all addresses, not binding one
+    return f"http://{'127.0.0.1' if host in wildcards else host}:{port}"
+
+
 async def _announce_ready(api: "uvicorn.Server") -> None:
     """Say ready once the port is actually accepting, and not a moment before."""
     import asyncio
@@ -1015,6 +1081,23 @@ async def _announce_ready(api: "uvicorn.Server") -> None:
     emit(make("process", at=datetime.now(UTC), message="ready", process="dev"))
 
 
+async def _seed(api: "uvicorn.Server", url: str, bearer: str | None, directories: Sequence[Path]) -> None:
+    """Apply what --seed named, once the port is accepting, and write a record for each."""
+    import asyncio
+
+    from dirigent_cli.seeding import seed_directories
+    from dirigent_client import Dirigent, DirigentError
+
+    while not api.started:
+        await asyncio.sleep(0.05)
+    try:
+        async with Dirigent(url=url, token=bearer) as client:
+            async for record in seed_directories(client, directories):
+                emit(record)
+    except DirigentError as error:
+        emit(make("error", at=datetime.now(UTC), level="error", message=f"seeding stopped: {error.message}"))
+
+
 @app.command(rich_help_panel=PROCESS_PANEL)
 def worker(
     ctx: typer.Context,

dirigent_cli-0.14.0/src/dirigent_cli/seeding.py

@@ -0,0 +1,198 @@
+"""Filling a live instance from directories of documents, which ``dg dev --seed`` does at boot.
+
+The seeding goes through the API, so every apply meets the same preflight a person's apply
+meets, and a refusal is a record rather than a failure: a corpus is expected to hold
+documents an instance will not store.
+"""
+
+from collections.abc import AsyncIterator, Mapping, Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Final, cast
+
+import yaml
+
+from dirigent_cli.local import ConnectionSpec
+from dirigent_client import Dirigent, DirigentError, PlanAction, ProvenanceSource
+from dirigent_common import JsonMap
+from dirigent_core.documents import safe_load
+from dirigent_core.engine.definition import FORMAT_V1
+from dirigent_core.protocol import Record, make
+
+#: The file endings a seed reads. Anything else under a seed directory is passed over, and so
+#: is a file of one of these that does not parse as a mapping.
+SUFFIXES: Final = (".yaml", ".yml", ".json")
+
+#: The sections a document may carry so that it runs alone under ``dg run --local``. An
+#: instance refuses to store a document carrying one, so the seed creates what they declare
+#: and applies the document without them.
+CARRIED: Final = ("connections", "schemas")
+
+
+def readable(directory: Path) -> list[tuple[Path, JsonMap]]:
+    """Read every file under a directory that parses as a mapping, in a stable order."""
+    found: list[tuple[Path, JsonMap]] = []
+    for path in sorted(one for one in directory.rglob("*") if one.suffix in SUFFIXES and one.is_file()):
+        try:
+            parsed = safe_load(path.read_text())
+        except (OSError, UnicodeDecodeError, yaml.YAMLError):
+            continue
+        if isinstance(parsed, dict):
+            found.append((path, cast("JsonMap", parsed)))
+    return found
+
+
+def is_document(raw: JsonMap) -> bool:
+    """Say whether a parsed file is a document this instance reads."""
+    return raw.get("format") == FORMAT_V1
+
+
+def specs(declared: object) -> list[ConnectionSpec]:
+    """Read a ``connections:`` section, of a file or of a document, into one spec per code."""
+    if not isinstance(declared, Mapping):
+        return []
+    read: list[ConnectionSpec] = []
+    for code, body in sorted(cast("Mapping[str, object]", declared).items()):
+        fields = cast("dict[str, Any]", body) if isinstance(body, Mapping) else {}
+        read.append(ConnectionSpec.model_validate({**fields, "code": code}))
+    return read
+
+
+async def seed_directories(client: Dirigent, directories: Sequence[Path]) -> AsyncIterator[Record]:
+    """Apply every document under each directory, paused, and create what they need first.
+
+    The closing record's counts are read off the records the seeding emitted, so a reader of
+    the stream and this summary never disagree.
+    """
+    pipelines = 0
+    refused = 0
+    connections: set[str] = set()
+    async for record in _walk(client, directories):
+        kind = record["kind"]
+        if kind == "seed.applied":
+            pipelines += 1
+        elif kind == "seed.refused":
+            refused += 1
+        elif kind == "seed.connection":
+            connections.add(str(record["connection"]))
+        yield record
+    yield make(
+        "seed.done",
+        at=datetime.now(UTC),
+        message="seeded",
+        directories=[str(one) for one in directories],
+        pipelines=pipelines,
+        refused=refused,
+        connections=sorted(connections),
+    )
+
+
+async def _walk(client: Dirigent, directories: Sequence[Path]) -> AsyncIterator[Record]:
+    """Seed each directory in the order given: the connections it declares, then its documents.
+
+    A directory's own connection files go in before any of its documents, so a document that
+    names one of them passes the preflight that reads it.
+    """
+    for directory in directories:
+        files = readable(directory)
+        for path, raw in files:
+            if not is_document(raw):
+                async for record in _connections(client, specs(raw.get("connections")), str(path)):
+                    yield record
+        for path, raw in files:
+            if is_document(raw):
+                async for record in _document(client, raw, str(path)):
+                    yield record
+
+
+async def _document(client: Dirigent, raw: JsonMap, origin: str) -> AsyncIterator[Record]:
+    """Create what one document carries, then apply the document without those sections."""
+    async for record in _connections(client, specs(raw.get("connections")), origin):
+        yield record
+    for code, body in _schemas(raw).items():
+        try:
+            await _schema(client, code, body)
+        except DirigentError as error:
+            yield _refused(origin, f"schema {code}: {_reason(error)}")
+    yield await _apply(client, raw, origin)
+
+
+def _schemas(raw: JsonMap) -> dict[str, JsonMap]:
+    """Read a document's ``schemas:`` section as one body per code."""
+    declared = raw.get("schemas")
+    if not isinstance(declared, Mapping):
+        return {}
+    read = cast("Mapping[str, object]", declared)
+    return {code: cast("JsonMap", body) for code, body in sorted(read.items()) if isinstance(body, Mapping)}
+
+
+async def _schema(client: Dirigent, code: str, body: JsonMap) -> None:
+    """Store one named schema, replacing the body of one this instance already holds."""
+    try:
+        await client.schemas.create(body, code=code)
+    except DirigentError:
+        await client.schemas.update(code, body=body)
+
+
+async def _connections(client: Dirigent, declared: Sequence[ConnectionSpec], origin: str) -> AsyncIterator[Record]:
+    """Create each declared connection, or replace the config of one that is already there."""
+    for spec in declared:
+        try:
+            action = await _connection(client, spec)
+        except DirigentError as error:
+            yield _refused(origin, f"connection {spec.code}: {_reason(error)}")
+            continue
+        yield make(
+            "seed.connection",
+            at=datetime.now(UTC),
+            message="connection",
+            connection=spec.code,
+            connection_kind=spec.kind,
+            action=action,
+            origin=origin,
+        )
+
+
+async def _connection(client: Dirigent, spec: ConnectionSpec) -> str:
+    """Store one connection and say whether it was new, letting a refusal through."""
+    try:
+        await client.connections.create(spec.code, kind=spec.kind, config=spec.config, name=spec.name)
+    except DirigentError:
+        await client.connections.update(spec.code, config=spec.config)
+        return "updated"
+    return "created"
+
+
+async def _apply(client: Dirigent, raw: JsonMap, origin: str) -> Record:
+    """Apply one document with its schedules paused, without the sections it carries."""
+    document = {name: value for name, value in raw.items() if name not in CARRIED}
+    try:
+        result = await client.pipelines.apply(
+            document,
+            source=ProvenanceSource.FILE,
+            source_ref=origin,
+            pause_schedules=True,
+        )
+    except DirigentError as error:
+        return _refused(origin, _reason(error))
+    if result.plan.action is PlanAction.INVALID:
+        return _refused(origin, "; ".join(str(issue) for issue in result.plan.issues))
+    return make(
+        "seed.applied",
+        at=datetime.now(UTC),
+        message="applied",
+        document=origin,
+        pipeline=result.plan.code,
+        action=result.plan.action.value,
+        schedules_paused=result.triggers.schedules_created,
+    )
+
+
+def _refused(origin: str, reason: str) -> Record:
+    """Build the record saying what the instance would not take, and why."""
+    return make("seed.refused", at=datetime.now(UTC), message="refused", document=origin, reason=reason)
+
+
+def _reason(error: DirigentError) -> str:
+    """Read a refusal as the one line a record carries."""
+    return "; ".join(error.problems) or error.message