roborean-storage-dict 0.1.2__tar.gz

roborean_storage_dict-0.1.2/.gitignore

@@ -0,0 +1,27 @@
+venv/
+venv-qt5/
+venv-qt6/
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+*.tsbuildinfo
+node_modules/
+.pnpm-store/
+.DS_Store
+.idea/
+.vscode/*.local
+playground/
+.e2e-ai/
+**/test-results/
+**/playwright-report/
+research/
+.roborean/
+.roborean-sql-artifacts/
+*.db

roborean_storage_dict-0.1.2/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## [Unreleased]
+
+### Added
+
+- Filesystem project package loader/saver with JSON and YAML (`safe_load`)
+  support, plus durable run artifact directories and idempotency indexes.
+
+### Changed
+
+### Fixed
+
+- Lower inter-package dependency pins to `>=0.1.1` to match lockstep
+  release versions.

roborean_storage_dict-0.1.2/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: roborean-storage-dict
+Version: 0.1.2
+Summary: JSON/YAML filesystem persistence for Roborean
+Project-URL: Homepage, https://github.com/TNick/roborean
+Project-URL: Repository, https://github.com/TNick/roborean
+License-Expression: MIT
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: roborean-spec>=0.1.1
+Requires-Dist: roborean-storage-base>=0.1.1
+Provides-Extra: dev
+Requires-Dist: black>=24.0; extra == 'dev'
+Requires-Dist: flake8>=7.0; extra == 'dev'
+Requires-Dist: isort>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# roborean-storage-dict
+
+Filesystem JSON/YAML persistence for projects and runs.
+
+YAML loading uses `yaml.safe_load` only.
+
+```python
+from roborean_storage_dict import DictProjectRepository, DictRunRepository
+
+projects = DictProjectRepository(Path(".roborean"))
+runs = DictRunRepository(Path(".roborean"))
+```

roborean_storage_dict-0.1.2/README.md

@@ -0,0 +1,12 @@
+# roborean-storage-dict
+
+Filesystem JSON/YAML persistence for projects and runs.
+
+YAML loading uses `yaml.safe_load` only.
+
+```python
+from roborean_storage_dict import DictProjectRepository, DictRunRepository
+
+projects = DictProjectRepository(Path(".roborean"))
+runs = DictRunRepository(Path(".roborean"))
+```

roborean_storage_dict-0.1.2/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "roborean-storage-dict"
+version = "0.1.2"
+description = "JSON/YAML filesystem persistence for Roborean"
+requires-python = ">=3.11"
+license = "MIT"
+readme = "README.md"
+dependencies = [
+  "PyYAML>=6.0",
+  "roborean-spec>=0.1.1",
+  "roborean-storage-base>=0.1.1",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "black>=24.0", "isort>=5.0", "flake8>=7.0"]
+
+
+[project.urls]
+Homepage = "https://github.com/TNick/roborean"
+Repository = "https://github.com/TNick/roborean"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/roborean_storage_dict"]

roborean_storage_dict-0.1.2/src/roborean_storage_dict/__init__.py

@@ -0,0 +1,21 @@
+"""Filesystem JSON/YAML persistence for Roborean."""
+
+from .json_store import (
+    DictArtifactStore,
+    DictProjectRepository,
+    DictRunRepository,
+)
+from .project_package import load_project_dir, save_project_dir
+from .yaml_store import dump_yaml, load_yaml
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "DictArtifactStore",
+    "DictProjectRepository",
+    "DictRunRepository",
+    "dump_yaml",
+    "load_project_dir",
+    "load_yaml",
+    "save_project_dir",
+]

roborean_storage_dict-0.1.2/src/roborean_storage_dict/json_store.py

@@ -0,0 +1,228 @@
+"""Dictionary/filesystem repositories for projects and runs."""
+
+import json
+import logging
+from pathlib import Path
+
+from roborean_spec import Project, RunRecord, project_from_dict
+from roborean_storage_base import ConflictError, NotFoundError
+
+from .project_package import load_project_dir, save_project_dir, write_revision
+
+logger = logging.getLogger(__name__)
+
+
+def _read_json(path: Path) -> dict:
+    """Load a JSON object from disk."""
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _write_json(path: Path, data: dict) -> None:
+    """Write a JSON object with a trailing newline."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(data, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+
+
+class DictProjectRepository:
+    """Store projects under ``<root>/projects/<id>/``."""
+
+    def __init__(self, root: Path) -> None:
+        """Create repositories rooted at ``root``."""
+        self.root = root
+        self._projects = root / "projects"
+
+    def _project_dir(self, project_id: str) -> Path:
+        """Return the package directory for one project."""
+        return self._projects / project_id
+
+    def get(self, project_id: str) -> Project:
+        """Load the current project package."""
+        package_dir = self._project_dir(project_id)
+        if not package_dir.is_dir():
+            raise NotFoundError(project_id)
+        return load_project_dir(package_dir)
+
+    def get_revision(self, project_id: str, revision: str) -> Project:
+        """Load a pinned revision."""
+        path = (
+            self._project_dir(project_id)
+            / "revisions"
+            / revision
+            / "project.json"
+        )
+        if not path.is_file():
+            raise NotFoundError(f"{project_id}@{revision}")
+        return project_from_dict(_read_json(path))
+
+    def save(self, project: Project, *, revision: str | None = None) -> str:
+        """Save the current package and optionally a revision snapshot."""
+        rev = revision or "1"
+        package_dir = self._project_dir(project.id)
+        save_project_dir(package_dir, project)
+        write_revision(package_dir, rev, project)
+        meta_path = package_dir / "current_revision.txt"
+        meta_path.write_text(rev + "\n", encoding="utf-8")
+        return rev
+
+    def list_ids(self) -> list[str]:
+        """List project package directory names."""
+        if not self._projects.is_dir():
+            return []
+        return sorted(
+            path.name for path in self._projects.iterdir() if path.is_dir()
+        )
+
+    def delete(self, project_id: str) -> None:
+        """Remove a project package tree."""
+        package_dir = self._project_dir(project_id)
+        if not package_dir.exists():
+            raise NotFoundError(project_id)
+        for path in sorted(package_dir.rglob("*"), reverse=True):
+            if path.is_file():
+                path.unlink()
+            elif path.is_dir():
+                path.rmdir()
+        package_dir.rmdir()
+
+
+class DictRunRepository:
+    """Store runs under ``<root>/runs/<projectId>/<runId>/``."""
+
+    def __init__(self, root: Path) -> None:
+        """Create a run repository under ``root``."""
+        self.root = root
+        self._runs = root / "runs"
+        self._idempotency = root / "idempotency"
+
+    def _run_dir(self, project_id: str, run_id: str) -> Path:
+        """Return the artifact directory for one run."""
+        return self._runs / project_id / run_id
+
+    def _idempotency_path(self, project_id: str, key: str) -> Path:
+        """Return the idempotency index file path."""
+        safe = key.replace(":", "_")
+        return self._idempotency / project_id / f"{safe}.json"
+
+    def get(self, run_id: str) -> RunRecord:
+        """Find a run by scanning project run directories."""
+        if not self._runs.is_dir():
+            raise NotFoundError(run_id)
+        for project_dir in self._runs.iterdir():
+            candidate = project_dir / run_id / "run-record.json"
+            if candidate.is_file():
+                return RunRecord.model_validate(_read_json(candidate))
+        raise NotFoundError(run_id)
+
+    def get_by_idempotency(
+        self, project_id: str, idempotency_key: str
+    ) -> RunRecord | None:
+        """Resolve an idempotency key to a run record."""
+        index = self._idempotency_path(project_id, idempotency_key)
+        if not index.is_file():
+            return None
+        payload = _read_json(index)
+        run_id = payload["runId"]
+        try:
+            return self.get(run_id)
+        except NotFoundError:
+            logger.debug(
+                "Idempotency index pointed at missing run %s",
+                run_id,
+                exc_info=True,
+            )
+            return None
+
+    def save(self, record: RunRecord) -> None:
+        """Insert a new run and its idempotency index entry."""
+        existing = self.get_by_idempotency(
+            record.project_id, record.idempotency_key
+        )
+        if existing is not None:
+            if existing.request_digest != record.request_digest:
+                raise ConflictError(
+                    "idempotency key reused with a different request body"
+                )
+            raise ConflictError("idempotency key already exists")
+        self._write_record(record)
+        _write_json(
+            self._idempotency_path(record.project_id, record.idempotency_key),
+            {
+                "runId": record.run_id,
+                "createdAt": record.created_at,
+                "requestDigest": record.request_digest,
+            },
+        )
+
+    def update(self, record: RunRecord) -> None:
+        """Overwrite an existing run artifact set."""
+        run_dir = self._run_dir(record.project_id, record.run_id)
+        if not (run_dir / "run-record.json").is_file():
+            raise NotFoundError(record.run_id)
+        self._write_record(record)
+
+    def list_for_project(
+        self, project_id: str, *, limit: int = 50
+    ) -> list[RunRecord]:
+        """List runs for a project, newest first by createdAt."""
+        project_dir = self._runs / project_id
+        if not project_dir.is_dir():
+            return []
+        records: list[RunRecord] = []
+        for run_dir in project_dir.iterdir():
+            path = run_dir / "run-record.json"
+            if path.is_file():
+                records.append(RunRecord.model_validate(_read_json(path)))
+        records.sort(key=lambda item: item.created_at, reverse=True)
+        return records[:limit]
+
+    def _write_record(self, record: RunRecord) -> None:
+        """Write run-record and optional result/diff sidecars."""
+        run_dir = self._run_dir(record.project_id, record.run_id)
+        run_dir.mkdir(parents=True, exist_ok=True)
+        _write_json(
+            run_dir / "run-record.json",
+            record.model_dump(mode="json", by_alias=True, exclude_none=True),
+        )
+        if record.results is not None:
+            _write_json(
+                run_dir / "run-results.json",
+                record.results.model_dump(
+                    mode="json", by_alias=True, exclude_none=True
+                ),
+            )
+        if record.diff is not None:
+            _write_json(
+                run_dir / "run-diff.json",
+                record.diff.model_dump(
+                    mode="json", by_alias=True, exclude_none=True
+                ),
+            )
+
+
+class DictArtifactStore:
+    """Store opaque artifacts under ``<root>/artifacts/``."""
+
+    def __init__(self, root: Path) -> None:
+        """Create an artifact store under ``root``."""
+        self.root = root / "artifacts"
+
+    def put_bytes(self, key: str, data: bytes, *, content_type: str) -> str:
+        """Write artifact bytes and a small sidecar for content type."""
+        path = self.root / key
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(data)
+        _write_json(
+            path.with_suffix(path.suffix + ".meta.json"),
+            {"contentType": content_type},
+        )
+        return key
+
+    def get_bytes(self, key: str) -> bytes:
+        """Read artifact bytes."""
+        path = self.root / key
+        if not path.is_file():
+            raise NotFoundError(key)
+        return path.read_bytes()

roborean_storage_dict-0.1.2/src/roborean_storage_dict/project_package.py

@@ -0,0 +1,82 @@
+"""Load and save on-disk project packages."""
+
+import json
+from pathlib import Path
+
+from roborean_spec import (
+    CompiledProject,
+    Project,
+    project_from_dict,
+    project_to_dict,
+)
+from roborean_storage_base import IntegrityError, NotFoundError
+
+from .yaml_store import dump_yaml, load_yaml
+
+
+def detect_entry(package_dir: Path) -> Path:
+    """Return the project entry file inside a package directory."""
+    json_path = package_dir / "project.json"
+    yaml_path = package_dir / "project.yaml"
+    if json_path.is_file():
+        return json_path
+    if yaml_path.is_file():
+        return yaml_path
+    raise NotFoundError(f"No project.json or project.yaml in {package_dir}")
+
+
+def load_project_dir(package_dir: Path) -> Project:
+    """Load a project from a package directory."""
+    entry = detect_entry(package_dir)
+    if entry.suffix.lower() == ".yaml":
+        data = load_yaml(entry)
+    else:
+        data = json.loads(entry.read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise IntegrityError(f"Project entry is not an object: {entry}")
+    return project_from_dict(data)
+
+
+def save_project_dir(
+    package_dir: Path,
+    project: Project,
+    *,
+    as_yaml: bool = False,
+) -> None:
+    """Write a project package entry file."""
+    package_dir.mkdir(parents=True, exist_ok=True)
+    data = project_to_dict(project)
+    if as_yaml:
+        dump_yaml(package_dir / "project.yaml", data)
+        return
+    path = package_dir / "project.json"
+    path.write_text(
+        json.dumps(data, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+
+
+def write_revision(
+    package_dir: Path,
+    revision: str,
+    project: Project,
+    compiled: CompiledProject | None = None,
+) -> None:
+    """Persist one immutable project revision under revisions/."""
+    revision_dir = package_dir / "revisions" / revision
+    revision_dir.mkdir(parents=True, exist_ok=True)
+    (revision_dir / "project.json").write_text(
+        json.dumps(project_to_dict(project), indent=2, ensure_ascii=False)
+        + "\n",
+        encoding="utf-8",
+    )
+    if compiled is not None:
+        (revision_dir / "compiled-project.json").write_text(
+            json.dumps(
+                compiled.model_dump(mode="json", by_alias=True),
+                indent=2,
+                ensure_ascii=False,
+            )
+            + "\n",
+            encoding="utf-8",
+        )

roborean_storage_dict-0.1.2/src/roborean_storage_dict/yaml_store.py

@@ -0,0 +1,28 @@
+"""YAML helpers that only use safe_load."""
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from roborean_storage_base import IntegrityError
+
+
+def load_yaml(path: Path) -> dict[str, Any]:
+    """Load YAML using yaml.safe_load only."""
+    # Reject non-mapping roots so package files stay object-shaped.
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as error:
+        raise IntegrityError(f"Invalid YAML: {path}") from error
+    if not isinstance(data, dict):
+        raise IntegrityError(f"YAML root must be a mapping: {path}")
+    return data
+
+
+def dump_yaml(path: Path, data: dict[str, Any]) -> None:
+    """Write a YAML mapping with a trailing newline."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        yaml.safe_dump(data, sort_keys=False, allow_unicode=True),
+        encoding="utf-8",
+    )

roborean_storage_dict-0.1.2/tests/conftest.py

@@ -0,0 +1,59 @@
+"""Shared fixtures for dict storage tests."""
+
+from pathlib import Path
+
+import pytest
+from roborean_spec import project_from_dict
+
+
+@pytest.fixture
+def minimal_project():
+    """Return a tiny valid project model."""
+    return project_from_dict(
+        {
+            "schemaVersion": "1.0.0",
+            "id": "example.minimal",
+            "name": "Minimal",
+            "pluginRequirements": [],
+            "workspace": {
+                "variables": [
+                    {
+                        "key": "title",
+                        "schema": {"type": "string"},
+                        "defaultValue": {
+                            "kind": "public_literal",
+                            "dataType": "string",
+                            "value": "Hello",
+                        },
+                        "const": False,
+                        "exposure": "clientVisible",
+                    }
+                ]
+            },
+            "bits": [
+                {
+                    "id": "b1",
+                    "type": "roborean.noop",
+                    "when": True,
+                    "config": {},
+                    "reads": [],
+                    "writes": [],
+                    "emits": [],
+                    "effectClass": "pure",
+                    "onError": "abort",
+                    "capabilities": [],
+                }
+            ],
+            "documents": [],
+            "templates": [],
+            "metadata": {},
+        }
+    )
+
+
+@pytest.fixture
+def store_root(tmp_path: Path) -> Path:
+    """Return a temporary store root directory."""
+    root = tmp_path / "store"
+    root.mkdir()
+    return root

roborean_storage_dict-0.1.2/tests/project_package_test.py

@@ -0,0 +1,59 @@
+"""Tests for on-disk project packages."""
+
+from pathlib import Path
+
+import pytest
+import yaml
+from roborean_storage_base import IntegrityError
+from roborean_storage_dict import (
+    DictProjectRepository,
+    dump_yaml,
+    load_project_dir,
+    load_yaml,
+    save_project_dir,
+)
+
+
+class TestProjectPackage:
+    """Round-trip package loading."""
+
+    def test_json_roundtrip(self, tmp_path: Path, minimal_project) -> None:
+        """JSON packages reload identically."""
+        package = tmp_path / "pkg"
+        save_project_dir(package, minimal_project)
+        loaded = load_project_dir(package)
+        assert loaded.id == minimal_project.id
+        assert loaded.name == minimal_project.name
+
+    def test_yaml_roundtrip(self, tmp_path: Path, minimal_project) -> None:
+        """YAML packages reload identically."""
+        package = tmp_path / "pkg"
+        save_project_dir(package, minimal_project, as_yaml=True)
+        loaded = load_project_dir(package)
+        assert loaded.id == minimal_project.id
+
+    def test_repo_save_get(self, store_root: Path, minimal_project) -> None:
+        """Dict project repository saves and loads."""
+        repo = DictProjectRepository(store_root)
+        rev = repo.save(minimal_project, revision="1")
+        assert rev == "1"
+        assert repo.get(minimal_project.id).id == minimal_project.id
+        loaded = repo.get_revision(minimal_project.id, "1")
+        assert loaded.id == minimal_project.id
+
+
+class TestYamlSafe:
+    """YAML loading must reject unsafe tags."""
+
+    def test_rejects_python_object_tag(self, tmp_path: Path) -> None:
+        """Python object tags are rejected by safe_load."""
+        path = tmp_path / "evil.yaml"
+        path.write_text("!!python/object/apply:os.system ['echo hi']\n")
+        with pytest.raises((IntegrityError, yaml.YAMLError)):
+            load_yaml(path)
+
+    def test_accepts_mapping(self, tmp_path: Path) -> None:
+        """Plain mappings load."""
+        path = tmp_path / "ok.yaml"
+        dump_yaml(path, {"a": 1})
+        assert load_yaml(path) == {"a": 1}

roborean_storage_dict-0.1.2/tests/run_store_test.py

@@ -0,0 +1,52 @@
+"""Tests for durable run artifact storage."""
+
+import pytest
+from roborean_spec import (
+    RunRecord,
+    RunRequest,
+    RunStatus,
+    RunTrigger,
+)
+from roborean_storage_base import ConflictError
+from roborean_storage_dict import DictRunRepository
+
+
+def _record(run_id: str, key: str = "k1", digest: str = "abc") -> RunRecord:
+    """Build a minimal queued run record."""
+    return RunRecord(
+        runId=run_id,
+        idempotencyKey=key,
+        projectId="example.minimal",
+        projectRevision="1",
+        compiledDigest="",
+        status=RunStatus.QUEUED,
+        request=RunRequest(
+            projectId="example.minimal",
+            idempotencyKey=key,
+            trigger=RunTrigger.TEST,
+        ),
+        attempt=1,
+        engineVersion="0.2.0",
+        pluginVersions={},
+        createdAt="2026-01-01T00:00:00+00:00",
+        requestDigest=digest,
+    )
+
+
+class TestDictRunRepository:
+    """Filesystem run repository behavior."""
+
+    def test_save_and_get(self, store_root) -> None:
+        """Runs persist under project/run directories."""
+        repo = DictRunRepository(store_root)
+        repo.save(_record("r1"))
+        loaded = repo.get("r1")
+        assert loaded.run_id == "r1"
+        assert repo.get_by_idempotency("example.minimal", "k1").run_id == "r1"
+
+    def test_idempotency_conflict_on_digest_mismatch(self, store_root) -> None:
+        """Same key with a different body digest conflicts."""
+        repo = DictRunRepository(store_root)
+        repo.save(_record("r1", digest="aaa"))
+        with pytest.raises(ConflictError):
+            repo.save(_record("r2", digest="bbb"))