dirigent-client 0.9.0__py3-none-any.whl

dirigent_client/schemas/pipelines.py

@@ -0,0 +1,326 @@
+"""Pipelines: what a listing shows, what an apply sends, and what an apply reports back."""
+
+from datetime import datetime
+from enum import StrEnum
+from uuid import UUID
+
+from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
+
+from dirigent_client.enums import DocumentKind, LogLevel, ProvenanceSource, RunPriority, RunStatus
+from dirigent_client.schemas.common import WireModel
+from dirigent_common import JsonMap
+
+
+class LastRun(WireModel):
+    """The most recent run of a pipeline, as a listing shows it."""
+
+    id: UUID
+    status: RunStatus
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    failed_step: str | None = None
+    """The step whose first failed attempt this run holds, when it did not end well."""
+
+
+class PipelineOut(WireModel):
+    """A pipeline as a listing shows it."""
+
+    id: UUID
+    code: str
+    name: str | None = None
+    description: str | None = None
+    tags: list[str] = Field(default_factory=list[str])
+    """What the current document says this pipeline is for."""
+
+    active: bool
+    current_version: int | None = None
+    active_runs: int = 0
+    schedules: int = 0
+    """How many clocks fire this pipeline."""
+
+    webhooks: int = 0
+    """How many inbound endpoints fire this pipeline."""
+
+    last_run: LastRun | None = None
+    """How the newest run of this pipeline went, or nothing when it has never run."""
+
+    created_at: datetime
+    updated_at: datetime
+
+
+class PipelineVersionOut(WireModel):
+    """One immutable version, its digest, and where it came from."""
+
+    id: UUID
+    version: int
+    digest: str
+    provenance_source: ProvenanceSource
+    provenance_ref: str | None = None
+    applied_by: str | None = None
+    created_at: datetime
+
+
+class PipelineDetail(PipelineOut):
+    """A pipeline plus the document its current version holds."""
+
+    document: JsonMap | None = None
+
+
+class ApplyRequest(BaseModel):
+    """A whole ``dirigent/v1`` document, optionally registered under a different code."""
+
+    document: JsonMap
+    code: str | None = Field(default=None, description="Register under this code instead of the document's own.")
+    source: ProvenanceSource = ProvenanceSource.API
+    source_ref: str | None = Field(default=None, description="The file path or URL the document came from.")
+    pause_schedules: bool = Field(
+        default=False,
+        description="Create the schedules this apply brings into being already paused.",
+    )
+
+
+class PruneRequest(BaseModel):
+    """The codes a reconcile keeps: everything else it once applied is deactivated."""
+
+    keep: list[str]
+    """Every code the reconciled directory holds. A directory-provenance pipeline absent
+    from this list is deactivated; a pipeline applied any other way is never touched."""
+
+
+class PruneResult(BaseModel):
+    """What a prune deactivated and deleted, or would have."""
+
+    pruned: list[str]
+    trigger_documents_removed: list[str] = Field(default_factory=list[str])
+    """The triggers documents a directory no longer holds, deleted with the rows they owned."""
+
+    dry_run: bool = False
+
+
+class RunRequest(BaseModel):
+    """Ad hoc run parameters, validated against the pipeline's own schema."""
+
+    params: JsonMap = Field(default_factory=dict)
+    priority: RunPriority | None = Field(
+        default=None,
+        description=(
+            "How far ahead of other runs this run's attempts are claimed. Omitted takes the pipeline's own priority."
+        ),
+    )
+    window_start: datetime | None = Field(
+        default=None,
+        description="Start of the logical data interval this run covers, inclusive.",
+    )
+    window_end: datetime | None = Field(
+        default=None,
+        description="End of that interval, exclusive. Both are given or neither is.",
+    )
+    log_levels: dict[str, LogLevel] | None = Field(
+        default=None,
+        description=(
+            "Which log levels this run keeps, as a map of block-id pattern to level; the most "
+            "specific pattern wins. Omitted keeps info and up."
+        ),
+    )
+
+    @field_validator("log_levels")
+    @classmethod
+    def _a_pattern_names_something(cls, value: dict[str, LogLevel] | None) -> dict[str, LogLevel] | None:
+        """Refuse an empty pattern, which would match nothing and say nothing."""
+        if value is not None and any(not pattern for pattern in value):
+            raise ValueError("log_levels: a pattern may not be empty")
+        return value
+
+    @model_validator(mode="after")
+    def _a_window_has_two_ends(self) -> "RunRequest":
+        """Refuse a half-declared window, which no step could read as an interval."""
+        if (self.window_start is None) != (self.window_end is None):
+            raise ValueError("a window has two ends: give both window_start and window_end, or neither")
+        if self.window_start is not None and self.window_end is not None and self.window_start >= self.window_end:
+            raise ValueError(
+                f"a window runs forwards and covers something: {self.window_start.isoformat()} "
+                f"is not before {self.window_end.isoformat()}"
+            )
+        return self
+
+
+class RunAccepted(WireModel):
+    """What starting a run returns: the run id, or why nothing started."""
+
+    run_id: UUID | None = None
+    status: str
+    detail: str | None = None
+
+
+class BackfillRequest(BaseModel):
+    """One backfill: whose cadence defines the windows, what interval to fill, and with what.
+
+    ``from_`` and ``to`` bound the firings enumerated, half-open, and each enumerated firing
+    becomes one run carrying the window that firing would have covered.
+    """
+
+    schedule: str = Field(description="The schedule on this pipeline whose cadence defines the windows.")
+    #: ``from`` is a Python keyword, so the field is ``from_`` here and ``from`` on the wire.
+    from_: datetime = Field(
+        serialization_alias="from",
+        validation_alias=AliasChoices("from", "from_"),
+        description="First instant of the interval to enumerate, inclusive.",
+    )
+    to: datetime = Field(description="Last instant of the interval to enumerate, exclusive.")
+    params: JsonMap | None = Field(
+        default=None,
+        description="Parameters every run created here is given; the schedule's own pins when omitted.",
+    )
+    dry_run: bool = Field(default=False, description="Answer with the plan and create nothing.")
+
+    @model_validator(mode="after")
+    def _runs_forwards(self) -> "BackfillRequest":
+        """Refuse an empty or backwards interval, which encloses no firing at all."""
+        if self.from_ >= self.to:
+            raise ValueError(
+                f"a backfill runs forwards and covers something: {self.from_.isoformat()} "
+                f"is not before {self.to.isoformat()}"
+            )
+        return self
+
+
+class BackfilledRun(WireModel):
+    """One window a backfill enumerated, and the run it created for it."""
+
+    window_start: datetime
+    window_end: datetime
+    run_id: UUID | None = None
+    """The run created for this window, or None on a dry run and where the policy refused one."""
+
+    detail: str | None = None
+    """Why no run was created, when none was."""
+
+
+class BackfillAccepted(WireModel):
+    """What a backfill amounted to: the windows it enumerated, in chronological order."""
+
+    pipeline: str
+    schedule: str
+    dry_run: bool = False
+    windows: list[BackfilledRun] = Field(default_factory=list[BackfilledRun])
+
+    @property
+    def created(self) -> int:
+        """How many runs this backfill actually created."""
+        return sum(1 for window in self.windows if window.run_id is not None)
+
+
+class ValidationIssue(WireModel):
+    """One problem with a document, addressed at the place in it that is wrong."""
+
+    location: str
+    """Where the problem is, as a dotted document path such as ``steps.push.config.method``."""
+
+    message: str
+    """What is wrong, in the terms the person editing the document is thinking in."""
+
+    def __str__(self) -> str:
+        """Render the issue as one line."""
+        return f"{self.location}: {self.message}"
+
+
+class PlanAction(StrEnum):
+    """What applying a document would do to the instance."""
+
+    CREATE = "create"
+    """No pipeline holds this code; applying mints one with version 1."""
+
+    UPDATE = "update"
+    """The code exists and the document differs; applying writes the next version."""
+
+    UNCHANGED = "unchanged"
+    """The digest matches the current version; applying writes nothing."""
+
+    INVALID = "invalid"
+    """The document did not validate against this instance; applying is refused."""
+
+
+class DiffSummary(WireModel):
+    """What changed between the current version and the document being applied."""
+
+    steps_added: list[str] = Field(default_factory=list[str])
+    steps_removed: list[str] = Field(default_factory=list[str])
+    steps_changed: list[str] = Field(default_factory=list[str])
+    params_changed: bool = False
+    triggers_changed: bool = False
+    settings_changed: bool = False
+    """Name, description, or concurrency policy differs."""
+
+    @property
+    def empty(self) -> bool:
+        """Report whether the two definitions are the same in every respect summarised here."""
+        return not (
+            self.steps_added
+            or self.steps_removed
+            or self.steps_changed
+            or self.params_changed
+            or self.triggers_changed
+            or self.settings_changed
+        )
+
+
+class Materialized(WireModel):
+    """What applying a document did to its triggers."""
+
+    schedules_created: list[str] = Field(default_factory=list[str])
+    schedules_updated: list[str] = Field(default_factory=list[str])
+    schedules_removed: list[str] = Field(default_factory=list[str])
+    webhooks_created: list[str] = Field(default_factory=list[str])
+    webhooks_updated: list[str] = Field(default_factory=list[str])
+    webhooks_removed: list[str] = Field(default_factory=list[str])
+
+    @property
+    def empty(self) -> bool:
+        """Report whether the apply left every trigger exactly as it was."""
+        return not (
+            self.schedules_created
+            or self.schedules_updated
+            or self.schedules_removed
+            or self.webhooks_created
+            or self.webhooks_updated
+            or self.webhooks_removed
+        )
+
+
+class PipelinePlan(WireModel):
+    """What applying a document would do, before anything is written."""
+
+    code: str
+    action: PlanAction
+    digest: str
+    pipeline: str | None = None
+    """The pipeline a triggers document declares clocks for; a pipeline document names none."""
+
+    current_version: int | None = None
+    next_version: int | None = None
+    issues: list[ValidationIssue] = Field(default_factory=list[ValidationIssue])
+    warnings: list[ValidationIssue] = Field(default_factory=list[ValidationIssue])
+    """What is worth saying but does not refuse the apply, such as a worker tag nobody carries."""
+
+    diff: DiffSummary | None = None
+
+    @property
+    def ok(self) -> bool:
+        """Report whether this document could be applied as it stands."""
+        return self.action is not PlanAction.INVALID
+
+
+class ApplyResult(WireModel):
+    """The outcome of an apply."""
+
+    kind: DocumentKind = DocumentKind.PIPELINE
+    """Which kind of document was applied; a triggers document writes no version."""
+
+    plan: PipelinePlan
+    pipeline_id: UUID | None = None
+    """The pipeline this apply wrote, or the one a triggers document declares clocks for."""
+
+    version: int | None = None
+    dry_run: bool = False
+    triggers: Materialized = Field(default_factory=Materialized)
+    """Which schedules and webhooks the document's ``triggers:`` section created, changed, or retired."""

dirigent_client/schemas/runs.py

@@ -0,0 +1,203 @@
+"""Runs: the listing row, the detail with its DAG view model, log pages, and the report."""
+
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import Field
+
+from dirigent_client.enums import (
+    AttemptKind,
+    AttemptStatus,
+    LogLevel,
+    RunItemStatus,
+    RunPriority,
+    RunStatus,
+    TriggerKind,
+)
+from dirigent_client.schemas.common import WireModel
+from dirigent_common import JsonMap
+
+TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset(
+    {
+        RunStatus.SUCCEEDED,
+        RunStatus.COMPLETED_WITH_ERRORS,
+        RunStatus.FAILED,
+        RunStatus.CANCELLED,
+    }
+)
+
+
+class RunOut(WireModel):
+    """A run as a listing shows it."""
+
+    id: UUID
+    pipeline: str
+    pipeline_version: int
+    status: RunStatus
+    priority: RunPriority = RunPriority.NORMAL
+    """The priority the claim takes this run's attempts at, pinned when it was created."""
+    params: JsonMap = Field(default_factory=dict)
+    log_levels: dict[str, LogLevel] | None = None
+    """Which log levels this run keeps, by block-id pattern; None keeps info and up."""
+    triggered_by_kind: TriggerKind
+    triggered_by_label: str | None = None
+    trace_id: str | None = None
+    error: str | None = None
+    failed_step: str | None = None
+    """The step whose first failed attempt this run holds, when it did not end well."""
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    window_start: datetime | None = None
+    """The start of the logical data interval this run covers, when it carries one."""
+
+    window_end: datetime | None = None
+    """The end of that interval, exclusive. Both are set or neither is."""
+
+    created_at: datetime
+
+    @property
+    def terminal(self) -> bool:
+        """Report whether this run has reached a status nothing will move it out of."""
+        return self.status in TERMINAL_RUN_STATUSES
+
+
+class AttemptOut(WireModel):
+    """One try of one step, which is the unit a retry addresses."""
+
+    id: UUID
+    step_name: str
+    block_id: str
+    attempt: int
+    kind: AttemptKind
+    status: AttemptStatus
+    run_item_id: UUID | None = None
+    error: str | None = None
+    error_class: str | None = None
+    output: JsonMap | None = None
+    output_uri: str | None = None
+    """Where the output was written, when it was too large to inline on the artifact."""
+
+    output_bytes: int | None = None
+    waiting_message: str | None = None
+    waiting_progress: float | None = None
+    poke_count: int = 0
+    """How many times a probe answered that the work was not done yet."""
+
+    next_poll_at: datetime | None = None
+    available_at: datetime | None = None
+    """When this attempt became claimable, which a retry's backoff pushes into the future."""
+
+    deadline_at: datetime | None = None
+    """When the engine gives up on this attempt, which is the budget its waiting is spent against."""
+
+    heartbeat_at: datetime | None = None
+    """When a worker last took this attempt up, which on a parked attempt is its last probe."""
+
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    created_at: datetime | None = None
+    """When the attempt was written down, which is where its queued time is measured from."""
+
+
+class AttemptEvent(AttemptOut):
+    """One attempt state on the run's event stream, with the label a person knows the item by."""
+
+    item: str | None = None
+    item_index: int | None = None
+    """Where the element sits in the fan-out, which is the order a reader lists the elements in."""
+
+
+class ItemOut(WireModel):
+    """One element of a fan-out step."""
+
+    id: UUID
+    step_name: str
+    item_index: int
+    item_key: str
+    status: RunItemStatus
+    failing_step: str | None = None
+    error: str | None = None
+
+
+class DagNode(WireModel):
+    """One node of the run's graph."""
+
+    code: str
+    """The step's key in the document, which is what every edge and every attempt references."""
+
+    name: str | None = None
+    """The step's human title, when the document gave it one."""
+
+    block: str
+    outcome: str
+    depends_on: list[str] = Field(default_factory=list[str])
+    rule: str
+    fan_out: bool = False
+    items_total: int = 0
+    items_failed: int = 0
+    attempts: int = 0
+
+
+class DagView(WireModel):
+    """The run's graph: its nodes and the edges between them."""
+
+    nodes: list[DagNode] = Field(default_factory=list[DagNode])
+    edges: list[tuple[str, str]] = Field(default_factory=list[tuple[str, str]])
+
+
+class RunDetail(WireModel):
+    """A run, the DAG view model, and how big the grids its sub-resources page are."""
+
+    run: RunOut
+    dag: DagView
+    items_total: int = 0
+    attempts_total: int = 0
+    waiting_for_workers: list[str] | None = None
+    """The tags this queued run needs that no live worker carries; null when nothing blocks it."""
+
+
+class LogEntryOut(WireModel):
+    """One product-telemetry entry."""
+
+    id: int
+    run_id: UUID
+    step_name: str | None = None
+    step_attempt_id: UUID | None = None
+    level: LogLevel
+    message: str
+    fields: JsonMap | None = None
+    created_at: datetime
+
+
+class StepReport(WireModel):
+    """What one step amounted to, for the run report."""
+
+    step: str
+    block: str
+    outcome: str
+    attempts: int
+    depends_on: list[str] = Field(default_factory=list[str])
+    """The steps this one waited for, which is why it ran when it did."""
+
+    warnings: int = 0
+    """How many warnings or errors this step logged, whatever it settled as."""
+
+    duration_ms: int | None = None
+    error: str | None = None
+
+
+class RunReport(WireModel):
+    """A plain summary of one run."""
+
+    run_id: UUID
+    pipeline: str
+    pipeline_version: int
+    status: RunStatus
+    triggered_by: str | None = None
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    duration_ms: int | None = None
+    steps: list[StepReport] = Field(default_factory=list[StepReport])
+    items_total: int = 0
+    items_failed: int = 0
+    error: str | None = None

dirigent_client/schemas/schemas.py

@@ -0,0 +1,54 @@
+"""Schemas: named JSON Schemas an instance holds, addressable by code and referenced by name."""
+
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+from dirigent_client.schemas.common import WireModel
+from dirigent_common import EntityName, JsonMap
+
+
+class SchemaIn(BaseModel):
+    """What storing a schema sends: the JSON Schema itself, and an identity it need not repeat.
+
+    The body is a JSON Schema in its own right. Its identity is read from the schema's own
+    keywords when the caller leaves it out -- ``$id`` for the code, ``title`` for the name,
+    ``description`` for the description -- so a portable schema file stores as one without a
+    dirigent wrapper. A code given here wins over ``$id``; a code given neither way, with no
+    ``$id`` and no filename to fall back on, is refused.
+    """
+
+    code: EntityName | None = None
+    name: str | None = None
+    description: str | None = None
+    body: JsonMap = Field(default_factory=dict)
+    """The JSON Schema, checked to be a valid schema before anything is stored."""
+
+
+class SchemaUpdate(BaseModel):
+    """What editing a schema may change; the code is fixed once minted.
+
+    A field left out is left alone, and a field sent as null is cleared. The body, when
+    sent, replaces the stored schema and is checked the same way a create's is.
+    """
+
+    name: str | None = None
+    description: str | None = None
+    body: JsonMap | None = None
+
+    def changing(self, field: str) -> bool:
+        """Report whether the request said anything about this field at all."""
+        return field in self.model_fields_set
+
+
+class SchemaOut(WireModel):
+    """A schema as every response shows it: the identity quartet and the schema body."""
+
+    id: UUID
+    code: str
+    name: str | None = None
+    description: str | None = None
+    body: JsonMap
+    created_at: datetime
+    updated_at: datetime

dirigent_client/schemas/system.py

@@ -0,0 +1,100 @@
+"""What this instance is, whether it is well, and which nodes are picking work up."""
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+from dirigent_client.enums import WorkerStatus
+from dirigent_client.schemas.common import WireModel
+from dirigent_common import JsonMap
+
+
+class ConnectionHealth(WireModel):
+    """What one connection reported when it was asked."""
+
+    code: str
+    name: str | None = None
+    kind: str
+    connected: bool
+    detail: str | None = None
+    version: str | None = None
+
+
+class SystemInfo(WireModel):
+    """What this instance is, and whether it is well."""
+
+    name: str = "dirigent"
+    version: str
+    environment: str
+    database: str
+    checked_at: datetime
+    plugins: list[str] = Field(default_factory=list[str])
+    blocks: int = 0
+    storage_schemes: list[str] = Field(default_factory=list[str])
+    """Every scheme a URI in a document may address, including the one core ships."""
+
+    notifiers: list[str] = Field(default_factory=list[str])
+    connection_kinds: list[str] = Field(default_factory=list[str])
+    workers_live: int = 0
+    unsafe_blocks_enabled: list[str] = Field(default_factory=list[str])
+    secrets_configured: bool = False
+    connections: list[ConnectionHealth] = Field(default_factory=list[ConnectionHealth])
+
+
+class WorkerOut(WireModel):
+    """One worker as the registry knows it."""
+
+    id: UUID
+    name: str
+    hostname: str
+    version: str
+    status: WorkerStatus
+    concurrency: int
+    tags: list[str] = Field(default_factory=list[str])
+    plugins: JsonMap = Field(default_factory=dict)
+    catalog_digest: str | None = None
+    code_matches_server: bool = True
+    """Whether this worker's catalog is the same code the server is running."""
+
+    stale: bool = False
+    """Whether its heartbeat is old enough that it is presumed gone."""
+
+    created_at: datetime
+    last_seen_at: datetime
+
+
+class CheckStatus(StrEnum):
+    """How one dependency is doing; the worst status across checks becomes the answer."""
+
+    HEALTHY = "healthy"
+    """Reachable and behaving."""
+
+    DEGRADED = "degraded"
+    """Usable, but not as it should be; readiness still succeeds."""
+
+    UNHEALTHY = "unhealthy"
+    """Unusable; readiness fails and the process should not receive traffic."""
+
+
+class CheckResult(BaseModel):
+    """What one health check reports."""
+
+    status: CheckStatus
+    detail: str | None = None
+
+
+class Health(BaseModel):
+    """The liveness answer: the process is running and can serve requests."""
+
+    status: Literal["ok"] = "ok"
+    version: str
+
+
+class Readiness(BaseModel):
+    """The readiness answer: the worst status across the checks, and each check's own."""
+
+    status: CheckStatus
+    checks: dict[str, CheckResult]