crawfish 0.1.0__py3-none-any.whl

crawfish/__init__.py

@@ -0,0 +1,420 @@
+"""Crawfish — agents for bulk work over your data.
+
+``Source → Batch (fan-out) → Aggregator (reduce) → Router (branch) → Sink``,
+authored as directories and run locally via ``claude -p``.
+
+This module re-exports the stable public surface. As primitives land (M1–M5) they
+are added here, each placed in its stability tier.
+"""
+
+from __future__ import annotations
+
+from crawfish.artifacts import (
+    ArtifactRef,
+    ArtifactStore,
+    LocalArtifactStore,
+    offload_if_large,
+)
+from crawfish.batch import Anomaly, Batch, Task
+from crawfish.build import BuildPlan, generate_containerfile, plan_build, write_containerfile
+from crawfish.ccexport import (
+    ClaudeCodeAgent,
+    ClaudeCodeSkill,
+    definition_to_cc_agent,
+    export_claude_code,
+    map_tools,
+    model_alias,
+)
+from crawfish.config import ProfileConfig, ProjectManifest, ProjectPaths, load_manifest
+from crawfish.core import (
+    BudgetExceeded,
+    Cancelled,
+    CancelToken,
+    CostBudget,
+    Flow,
+    JSONValue,
+    Node,
+    NodeKind,
+    Parameter,
+    Policy,
+    PolicyKind,
+    RunContext,
+    new_id,
+    parameters_compatible,
+)
+from crawfish.cost import (
+    Budget,
+    BudgetState,
+    CostEstimate,
+    CostMeter,
+    estimate_cost,
+    spent_today,
+)
+from crawfish.definition import (
+    AgentSpec,
+    Coordination,
+    Definition,
+    DefinitionAssets,
+    DefinitionLoadError,
+    DefinitionRef,
+    MarketplacePackage,
+    MCPConnection,
+    Prompt,
+    TeamSpec,
+    load_definition,
+)
+from crawfish.deploy import (
+    DeployEntry,
+    DeployRegistry,
+    DeployStatus,
+    Supervisor,
+    deploy,
+    stop,
+)
+from crawfish.discovery import Registry, UnitRef
+from crawfish.doctor import DoctorFinding, DoctorReport, diagnose
+from crawfish.engine import Engine, run_pipeline
+from crawfish.eval import (
+    EvalCase,
+    GoldenSet,
+    LLMJudge,
+    capture_case,
+    gate_against_baseline,
+    grade_output,
+    load_baseline,
+    save_baseline,
+)
+from crawfish.executor import (
+    BatchExecutor,
+    BatchRunResult,
+    CycleError,
+    DependencyGraph,
+    ExecutionPlan,
+    Roadmap,
+)
+from crawfish.inspector import RunReport, format_report, inspect_run, tail_events
+from crawfish.ledger import ExecState, ExecutionLedger
+from crawfish.manage import PipelineStatus, format_table, manage_list, restart_target
+from crawfish.memory import Memory
+from crawfish.metrics import (
+    Benchmark,
+    Metric,
+    Rubric,
+    compare,
+    confidence_threshold,
+    field_present,
+    is_nonempty,
+    is_regression,
+    output_number,
+)
+from crawfish.nodes import (
+    Aggregator,
+    ApprovalRequired,
+    Classifier,
+    Filter,
+    GitHubPRSink,
+    LinearSink,
+    PullRequestSource,
+    RepoSource,
+    Router,
+    Sink,
+    Source,
+    TargetMustBeStaticError,
+    UnroutableLabelError,
+    collect,
+    concat,
+    count,
+    dedupe,
+    definition_reducer,
+    fan_in,
+    fan_out,
+    field_equals,
+    field_matches,
+    limit,
+    title_contains,
+)
+from crawfish.observe import (
+    ObserverEvent,
+    ObserverSurface,
+    RunInfo,
+    Severity,
+    parse_since,
+)
+from crawfish.observer import (
+    CostSpike,
+    FailureRateAbove,
+    Observer,
+    ObserverContext,
+    Rule,
+    StuckRun,
+)
+from crawfish.output import Output, WireError, check_wire, output_satisfies_inputs
+from crawfish.retry import ItemResult, ItemStatus, RetryPolicy
+from crawfish.run import InputBindingError, Run, RunStatus, RunSuspended
+from crawfish.runtime import (
+    AgentRuntime,
+    ClientRuntime,
+    CommandRuntime,
+    ManagedRuntime,
+    MockRuntime,
+    RecordReplayRuntime,
+    RunRequest,
+    RunResult,
+    RuntimeEvent,
+    get_runtime,
+)
+from crawfish.sandbox import EgressBroker, EgressDenied, run_out_of_process
+from crawfish.scaffold import scaffold_project
+from crawfish.secrets import (
+    Capabilities,
+    ScrubbingStore,
+    SecretManager,
+    load_env,
+    read_capabilities,
+    redact,
+    resolve_secret,
+)
+from crawfish.stability import (
+    Stability,
+    deprecated,
+    experimental,
+    is_breaking,
+    stability_of,
+    stable,
+)
+from crawfish.store import SqliteStore, Store
+from crawfish.testing import (
+    assert_rubric,
+    assert_snapshot,
+    replaying,
+    run_fixtures,
+    snapshot_match,
+)
+from crawfish.triggers import (
+    Cron,
+    CronSchedule,
+    CronTrigger,
+    Trigger,
+    WebhookTrigger,
+    verify_webhook,
+)
+from crawfish.typesystem import TypeDef, TypeKind, TypeRegistry, default_registry
+from crawfish.versioning import Freezable, FrozenError, Version
+from crawfish.visualize import dashboard_state, serve_dashboard
+from crawfish.workflow import Workflow
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    # core
+    "JSONValue",
+    "new_id",
+    "Flow",
+    "Parameter",
+    "NodeKind",
+    "Node",
+    "PolicyKind",
+    "Policy",
+    "parameters_compatible",
+    "RunContext",
+    "CostBudget",
+    "CancelToken",
+    "BudgetExceeded",
+    "Cancelled",
+    # type system
+    "TypeDef",
+    "TypeKind",
+    "TypeRegistry",
+    "default_registry",
+    # versioning
+    "Version",
+    "FrozenError",
+    "Freezable",
+    # store
+    "Store",
+    "SqliteStore",
+    # engine
+    "Engine",
+    "run_pipeline",
+    # output
+    "Output",
+    "output_satisfies_inputs",
+    "check_wire",
+    "WireError",
+    # definition
+    "Definition",
+    "AgentSpec",
+    "TeamSpec",
+    "Coordination",
+    "Prompt",
+    "DefinitionRef",
+    "DefinitionAssets",
+    "MarketplacePackage",
+    "MCPConnection",
+    "load_definition",
+    "DefinitionLoadError",
+    # runtime
+    "AgentRuntime",
+    "CommandRuntime",
+    "MockRuntime",
+    "ClientRuntime",
+    "ManagedRuntime",
+    "RecordReplayRuntime",
+    "RunRequest",
+    "RunResult",
+    "RuntimeEvent",
+    "get_runtime",
+    # nodes (M2)
+    "Source",
+    "RepoSource",
+    "PullRequestSource",
+    "fan_out",
+    "Sink",
+    "LinearSink",
+    "GitHubPRSink",
+    "TargetMustBeStaticError",
+    "ApprovalRequired",
+    "Filter",
+    "title_contains",
+    "field_equals",
+    "field_matches",
+    "limit",
+    "Memory",
+    # run (M2)
+    "Run",
+    "RunStatus",
+    "InputBindingError",
+    "RunSuspended",
+    # pipelines (M3)
+    "Batch",
+    "Task",
+    "Anomaly",
+    "Aggregator",
+    "collect",
+    "concat",
+    "count",
+    "dedupe",
+    "definition_reducer",
+    "fan_in",
+    "Router",
+    "Classifier",
+    "UnroutableLabelError",
+    "ArtifactRef",
+    "ArtifactStore",
+    "LocalArtifactStore",
+    "offload_if_large",
+    "DependencyGraph",
+    "CycleError",
+    "Roadmap",
+    "ExecutionPlan",
+    "BatchExecutor",
+    "BatchRunResult",
+    "ExecutionLedger",
+    "ObserverEvent",
+    "ObserverSurface",
+    "RunInfo",
+    "Severity",
+    "parse_since",
+    # operate / observe / integrate
+    "DeployEntry",
+    "DeployRegistry",
+    "DeployStatus",
+    "Supervisor",
+    "deploy",
+    "stop",
+    "PipelineStatus",
+    "manage_list",
+    "format_table",
+    "restart_target",
+    "Observer",
+    "ObserverContext",
+    "Rule",
+    "FailureRateAbove",
+    "CostSpike",
+    "StuckRun",
+    "dashboard_state",
+    "serve_dashboard",
+    "ClaudeCodeAgent",
+    "ClaudeCodeSkill",
+    "definition_to_cc_agent",
+    "export_claude_code",
+    "map_tools",
+    "model_alias",
+    "ExecState",
+    "RetryPolicy",
+    "ItemResult",
+    "ItemStatus",
+    "Workflow",
+    # measurement (M4)
+    "Metric",
+    "Rubric",
+    "Benchmark",
+    "output_number",
+    "field_present",
+    "is_nonempty",
+    "confidence_threshold",
+    "compare",
+    "is_regression",
+    "estimate_cost",
+    "CostEstimate",
+    "Budget",
+    "BudgetState",
+    "CostMeter",
+    "spent_today",
+    "inspect_run",
+    "tail_events",
+    "format_report",
+    "RunReport",
+    # eval data lifecycle (M4)
+    "EvalCase",
+    "GoldenSet",
+    "LLMJudge",
+    "capture_case",
+    "grade_output",
+    "save_baseline",
+    "load_baseline",
+    "gate_against_baseline",
+    # authoring / packaging / ship (M5)
+    "Registry",
+    "UnitRef",
+    "ProfileConfig",
+    "ProjectManifest",
+    "ProjectPaths",
+    "load_manifest",
+    "DoctorFinding",
+    "DoctorReport",
+    "diagnose",
+    "Cron",
+    "CronSchedule",
+    "scaffold_project",
+    "resolve_secret",
+    "load_env",
+    "SecretManager",
+    "ScrubbingStore",
+    "redact",
+    "read_capabilities",
+    "Capabilities",
+    "snapshot_match",
+    "assert_snapshot",
+    "run_fixtures",
+    "assert_rubric",
+    "replaying",
+    "generate_containerfile",
+    "plan_build",
+    "write_containerfile",
+    "BuildPlan",
+    "Trigger",
+    "CronTrigger",
+    "WebhookTrigger",
+    "verify_webhook",
+    "Stability",
+    "stable",
+    "experimental",
+    "deprecated",
+    "stability_of",
+    "is_breaking",
+    "EgressBroker",
+    "EgressDenied",
+    "run_out_of_process",
+]

crawfish/artifacts/__init__.py

@@ -0,0 +1,12 @@
+"""Artifact store — blob/file Outputs carried by reference.
+
+Public surface: the :class:`ArtifactRef` envelope, the :class:`ArtifactStore`
+protocol seam, the local-disk reference impl, and the ``offload_if_large`` helper.
+"""
+
+from __future__ import annotations
+
+from crawfish.artifacts.base import ArtifactRef, ArtifactStore
+from crawfish.artifacts.local import LocalArtifactStore, offload_if_large
+
+__all__ = ["ArtifactRef", "ArtifactStore", "LocalArtifactStore", "offload_if_large"]

crawfish/artifacts/base.py

@@ -0,0 +1,62 @@
+"""The ``ArtifactStore`` seam — blob/file Outputs by reference.
+
+Large payloads (files, blobs, big JSON) don't belong inline in an ``Output``.
+Instead an Output carries a small, content-addressed :class:`ArtifactRef` (a dict
+on ``Output.value``) and the bytes live in an :class:`ArtifactStore`. The product
+model imports the *protocol*, never a concrete backend, so local-disk → S3/GCS is
+a driver swap (mirrors the ``Store`` seam, ADR 0001/0003). Every operation carries
+an ``org_id`` tenancy key (defaulted ``"local"``) so cloud multi-tenancy is a
+driver swap, not a schema migration.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel
+
+__all__ = ["ArtifactRef", "ArtifactStore"]
+
+
+class ArtifactRef(BaseModel):
+    """A content-addressed pointer to artifact bytes held in an ``ArtifactStore``.
+
+    This is what an ``Output`` carries instead of inline bytes. ``uri`` and
+    ``sha256`` both derive from the content hash, so identical content dedupes.
+    """
+
+    uri: str
+    sha256: str
+    size: int
+    content_type: str = "application/octet-stream"
+
+
+@runtime_checkable
+class ArtifactStore(Protocol):
+    """Blob persistence contract: content-addressed, tenant-scoped, GC-able."""
+
+    def put(
+        self,
+        data: bytes,
+        *,
+        content_type: str = "application/octet-stream",
+        org_id: str = "local",
+    ) -> ArtifactRef:
+        """Store ``data`` and return a content-addressed :class:`ArtifactRef`."""
+        ...
+
+    def get(self, ref: ArtifactRef, *, org_id: str = "local") -> bytes:
+        """Return the bytes for ``ref``. Raises if absent for this ``org_id``."""
+        ...
+
+    def exists(self, ref: ArtifactRef, *, org_id: str = "local") -> bool:
+        """True iff ``ref``'s content is stored under this ``org_id``."""
+        ...
+
+    def delete(self, ref: ArtifactRef, *, org_id: str = "local") -> None:
+        """Delete ``ref``'s content for this ``org_id`` (no-op if absent)."""
+        ...
+
+    def gc(self, live_refs: set[str], *, org_id: str = "local") -> int:
+        """Delete artifacts whose sha256 is not in ``live_refs``; return count."""
+        ...

crawfish/artifacts/local.py

@@ -0,0 +1,93 @@
+"""Local-filesystem reference implementation of the ``ArtifactStore`` seam.
+
+Content-addressed storage under ``root/<org_id>/<sha[:2]>/<sha>``: identical bytes
+dedupe to one file, and each ``org_id`` gets a separate subtree (tenancy is a path
+prefix, not a schema change — mirrors the SQLite ``Store`` impl, ADR 0001/0003).
+All filesystem layout lives here; call sites use the protocol.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from crawfish.artifacts.base import ArtifactRef, ArtifactStore
+from crawfish.core.types import JSONValue
+
+__all__ = ["LocalArtifactStore", "offload_if_large"]
+
+
+class LocalArtifactStore:
+    """An ``ArtifactStore`` backed by the local filesystem, addressed by sha256."""
+
+    def __init__(self, root: str | Path) -> None:
+        self._root = Path(root)
+        self._root.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, sha256: str, org_id: str) -> Path:
+        return self._root / org_id / sha256[:2] / sha256
+
+    def put(
+        self,
+        data: bytes,
+        *,
+        content_type: str = "application/octet-stream",
+        org_id: str = "local",
+    ) -> ArtifactRef:
+        sha256 = hashlib.sha256(data).hexdigest()
+        dest = self._path(sha256, org_id)
+        if not dest.exists():
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            # Atomic write: stage to a temp file, then rename into place.
+            tmp = dest.with_name(dest.name + ".tmp")
+            tmp.write_bytes(data)
+            tmp.replace(dest)
+        return ArtifactRef(
+            uri=dest.as_uri(),
+            sha256=sha256,
+            size=len(data),
+            content_type=content_type,
+        )
+
+    def get(self, ref: ArtifactRef, *, org_id: str = "local") -> bytes:
+        return self._path(ref.sha256, org_id).read_bytes()
+
+    def exists(self, ref: ArtifactRef, *, org_id: str = "local") -> bool:
+        return self._path(ref.sha256, org_id).is_file()
+
+    def delete(self, ref: ArtifactRef, *, org_id: str = "local") -> None:
+        self._path(ref.sha256, org_id).unlink(missing_ok=True)
+
+    def gc(self, live_refs: set[str], *, org_id: str = "local") -> int:
+        base = self._root / org_id
+        if not base.is_dir():
+            return 0
+        removed = 0
+        for shard in base.iterdir():
+            if not shard.is_dir():
+                continue
+            for blob in shard.iterdir():
+                if blob.is_file() and blob.name not in live_refs:
+                    blob.unlink()
+                    removed += 1
+        return removed
+
+
+def offload_if_large(
+    value: JSONValue,
+    store: ArtifactStore,
+    *,
+    threshold: int = 65536,
+    org_id: str = "local",
+) -> JSONValue | ArtifactRef:
+    """Offload ``value`` to ``store`` if its JSON form exceeds ``threshold`` bytes.
+
+    Returns an :class:`ArtifactRef` (content_type ``application/json``) when the
+    serialized value is larger than ``threshold``; otherwise returns ``value``
+    unchanged. This is how an Output keeps large payloads out of the record.
+    """
+    data = json.dumps(value).encode("utf-8")
+    if len(data) <= threshold:
+        return value
+    return store.put(data, content_type="application/json", org_id=org_id)