metaspn-store 0.1.0__tar.gz

metaspn_store-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MetaSPN
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

metaspn_store-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: metaspn-store
+Version: 0.1.0
+Summary: Minimal append-only durability layer and replay API for MetaSPN signals and emissions
+Author: MetaSPN
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/MetaSPN/metaspn-store
+Project-URL: Repository, https://github.com/MetaSPN/metaspn-store
+Project-URL: Issues, https://github.com/MetaSPN/metaspn-store/issues
+Keywords: metaspn,event-store,replay,append-only,jsonl
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: metaspn-schemas>=0.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: build>=1.2.0; extra == "dev"
+Requires-Dist: twine>=5.0.0; extra == "dev"
+Requires-Dist: wheel>=0.43.0; extra == "dev"
+Dynamic: license-file
+
+# metaspn-store
+
+`metaspn-store` provides a minimal append-only event store for MetaSPN signals and emissions.
+
+## Features
+- Filesystem JSONL backend (partitioned by UTC date)
+- Append-only writes for signals and emissions
+- Snapshot writes for deterministic state rebuild checkpoints
+- Streaming replay by time window, entity reference, and source
+
+## Layout
+```text
+workspace/
+  store/
+    signals/
+      2026-02-05.jsonl
+    emissions/
+      2026-02-05.jsonl
+    snapshots/
+      system_state__2026-02-05T120000Z.json
+```
+
+## Release
+```bash
+python -m pip install -e ".[dev]"
+pytest -q
+python -m build
+python -m twine check dist/*
+python -m twine upload dist/*
+```

metaspn_store-0.1.0/README.md

@@ -0,0 +1,30 @@
+# metaspn-store
+
+`metaspn-store` provides a minimal append-only event store for MetaSPN signals and emissions.
+
+## Features
+- Filesystem JSONL backend (partitioned by UTC date)
+- Append-only writes for signals and emissions
+- Snapshot writes for deterministic state rebuild checkpoints
+- Streaming replay by time window, entity reference, and source
+
+## Layout
+```text
+workspace/
+  store/
+    signals/
+      2026-02-05.jsonl
+    emissions/
+      2026-02-05.jsonl
+    snapshots/
+      system_state__2026-02-05T120000Z.json
+```
+
+## Release
+```bash
+python -m pip install -e ".[dev]"
+pytest -q
+python -m build
+python -m twine check dist/*
+python -m twine upload dist/*
+```

metaspn_store-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "metaspn-store"
+version = "0.1.0"
+description = "Minimal append-only durability layer and replay API for MetaSPN signals and emissions"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "MetaSPN" }]
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["metaspn", "event-store", "replay", "append-only", "jsonl"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Operating System :: OS Independent",
+]
+dependencies = ["metaspn-schemas>=0.1.0"]
+
+[project.urls]
+Homepage = "https://github.com/MetaSPN/metaspn-store"
+Repository = "https://github.com/MetaSPN/metaspn-store"
+Issues = "https://github.com/MetaSPN/metaspn-store/issues"
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "build>=1.2.0", "twine>=5.0.0", "wheel>=0.43.0"]
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["test"]

metaspn_store-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

metaspn_store-0.1.0/src/metaspn_store/__init__.py

@@ -0,0 +1,3 @@
+from metaspn_store.store import FileSystemStore
+
+__all__ = ["FileSystemStore"]

metaspn_store-0.1.0/src/metaspn_store/store.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import json
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Iterator
+
+from metaspn_schemas import EmissionEnvelope, EntityRef, SignalEnvelope
+
+
+def _ensure_utc(value: datetime) -> datetime:
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc)
+
+
+def _iter_days(start_day: date, end_day: date) -> Iterator[date]:
+    current = start_day
+    while current <= end_day:
+        yield current
+        current += timedelta(days=1)
+
+
+class FileSystemStore:
+    """Append-only filesystem store for SignalEnvelope and EmissionEnvelope."""
+
+    def __init__(self, workspace: str | Path) -> None:
+        self.workspace = Path(workspace)
+        self.store_root = self.workspace / "store"
+        self.signals_dir = self.store_root / "signals"
+        self.emissions_dir = self.store_root / "emissions"
+        self.snapshots_dir = self.store_root / "snapshots"
+        self._ensure_dirs()
+
+    def _ensure_dirs(self) -> None:
+        self.signals_dir.mkdir(parents=True, exist_ok=True)
+        self.emissions_dir.mkdir(parents=True, exist_ok=True)
+        self.snapshots_dir.mkdir(parents=True, exist_ok=True)
+
+    def write_signal(self, signal: SignalEnvelope) -> Path:
+        """Append a signal to its UTC day partition and return written file path."""
+        if not signal.signal_id:
+            raise ValueError("signal_id is required for stable IDs")
+        if not signal.schema_version:
+            raise ValueError("schema_version is required")
+
+        signal_ts = _ensure_utc(signal.timestamp)
+        destination = self.signals_dir / f"{signal_ts.date().isoformat()}.jsonl"
+        with destination.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(signal.to_dict(), sort_keys=True, separators=(",", ":")))
+            handle.write("\n")
+        return destination
+
+    def write_emission(self, emission: EmissionEnvelope) -> Path:
+        """Append an emission to its UTC day partition and return written file path."""
+        if not emission.emission_id:
+            raise ValueError("emission_id is required for stable IDs")
+        if not emission.schema_version:
+            raise ValueError("schema_version is required")
+
+        emission_ts = _ensure_utc(emission.timestamp)
+        destination = self.emissions_dir / f"{emission_ts.date().isoformat()}.jsonl"
+        with destination.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(emission.to_dict(), sort_keys=True, separators=(",", ":")))
+            handle.write("\n")
+        return destination
+
+    def write_snapshot(
+        self,
+        name: str,
+        snapshot_state: dict[str, Any],
+        snapshot_time: datetime | None = None,
+    ) -> Path:
+        """Write a point-in-time snapshot JSON document."""
+        ts = _ensure_utc(snapshot_time or datetime.now(timezone.utc))
+        timestamp_token = ts.strftime("%Y-%m-%dT%H%M%SZ")
+        destination = self.snapshots_dir / f"{name}__{timestamp_token}.json"
+        with destination.open("w", encoding="utf-8") as handle:
+            json.dump(snapshot_state, handle, sort_keys=True, separators=(",", ":"))
+            handle.write("\n")
+        return destination
+
+    def iter_signals(
+        self,
+        start: datetime,
+        end: datetime,
+        entity_ref: EntityRef | None = None,
+        sources: list[str] | None = None,
+    ) -> Iterator[SignalEnvelope]:
+        """Stream signals in [start, end], optionally filtered by entity_ref and source."""
+        start_utc = _ensure_utc(start)
+        end_utc = _ensure_utc(end)
+        if end_utc < start_utc:
+            raise ValueError("end must be greater than or equal to start")
+
+        source_set = set(sources) if sources else None
+        for day in _iter_days(start_utc.date(), end_utc.date()):
+            partition = self.signals_dir / f"{day.isoformat()}.jsonl"
+            if not partition.exists():
+                continue
+            with partition.open("r", encoding="utf-8") as handle:
+                for line in handle:
+                    if not line.strip():
+                        continue
+                    record = json.loads(line)
+                    signal = SignalEnvelope.from_dict(record)
+                    signal_ts = _ensure_utc(signal.timestamp)
+                    if signal_ts < start_utc or signal_ts > end_utc:
+                        continue
+                    if source_set is not None and signal.source not in source_set:
+                        continue
+                    if entity_ref is not None and entity_ref not in signal.entity_refs:
+                        continue
+                    yield signal
+
+    def iter_emissions(
+        self,
+        start: datetime,
+        end: datetime,
+        entity_ref: EntityRef | None = None,
+        emission_types: list[str] | None = None,
+    ) -> Iterator[EmissionEnvelope]:
+        """Stream emissions in [start, end], optionally filtered by entity_ref and type."""
+        start_utc = _ensure_utc(start)
+        end_utc = _ensure_utc(end)
+        if end_utc < start_utc:
+            raise ValueError("end must be greater than or equal to start")
+
+        emission_type_set = set(emission_types) if emission_types else None
+        for day in _iter_days(start_utc.date(), end_utc.date()):
+            partition = self.emissions_dir / f"{day.isoformat()}.jsonl"
+            if not partition.exists():
+                continue
+            with partition.open("r", encoding="utf-8") as handle:
+                for line in handle:
+                    if not line.strip():
+                        continue
+                    record = json.loads(line)
+                    emission = EmissionEnvelope.from_dict(record)
+                    emission_ts = _ensure_utc(emission.timestamp)
+                    if emission_ts < start_utc or emission_ts > end_utc:
+                        continue
+                    if emission_type_set is not None and emission.emission_type not in emission_type_set:
+                        continue
+                    if entity_ref is not None and entity_ref not in emission.entity_refs:
+                        continue
+                    yield emission

metaspn_store-0.1.0/src/metaspn_store.egg-info/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: metaspn-store
+Version: 0.1.0
+Summary: Minimal append-only durability layer and replay API for MetaSPN signals and emissions
+Author: MetaSPN
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/MetaSPN/metaspn-store
+Project-URL: Repository, https://github.com/MetaSPN/metaspn-store
+Project-URL: Issues, https://github.com/MetaSPN/metaspn-store/issues
+Keywords: metaspn,event-store,replay,append-only,jsonl
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: metaspn-schemas>=0.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: build>=1.2.0; extra == "dev"
+Requires-Dist: twine>=5.0.0; extra == "dev"
+Requires-Dist: wheel>=0.43.0; extra == "dev"
+Dynamic: license-file
+
+# metaspn-store
+
+`metaspn-store` provides a minimal append-only event store for MetaSPN signals and emissions.
+
+## Features
+- Filesystem JSONL backend (partitioned by UTC date)
+- Append-only writes for signals and emissions
+- Snapshot writes for deterministic state rebuild checkpoints
+- Streaming replay by time window, entity reference, and source
+
+## Layout
+```text
+workspace/
+  store/
+    signals/
+      2026-02-05.jsonl
+    emissions/
+      2026-02-05.jsonl
+    snapshots/
+      system_state__2026-02-05T120000Z.json
+```
+
+## Release
+```bash
+python -m pip install -e ".[dev]"
+pytest -q
+python -m build
+python -m twine check dist/*
+python -m twine upload dist/*
+```

metaspn_store-0.1.0/src/metaspn_store.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+LICENSE
+README.md
+pyproject.toml
+src/metaspn_store/__init__.py
+src/metaspn_store/store.py
+src/metaspn_store.egg-info/PKG-INFO
+src/metaspn_store.egg-info/SOURCES.txt
+src/metaspn_store.egg-info/dependency_links.txt
+src/metaspn_store.egg-info/requires.txt
+src/metaspn_store.egg-info/top_level.txt
+test/test_store.py

metaspn_store-0.1.0/src/metaspn_store.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

metaspn_store-0.1.0/src/metaspn_store.egg-info/requires.txt

@@ -0,0 +1,7 @@
+metaspn-schemas>=0.1.0
+
+[dev]
+pytest>=8
+build>=1.2.0
+twine>=5.0.0
+wheel>=0.43.0

metaspn_store-0.1.0/src/metaspn_store.egg-info/top_level.txt

@@ -0,0 +1 @@
+metaspn_store

metaspn_store-0.1.0/test/test_store.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from metaspn_schemas import EmissionEnvelope, EntityRef, SignalEnvelope
+
+from metaspn_store import FileSystemStore
+
+
+def _ts(day: int, hour: int = 0, minute: int = 0) -> datetime:
+    return datetime(2026, 2, day, hour, minute, tzinfo=timezone.utc)
+
+
+def test_round_trip_signal_and_emission(tmp_path: Path) -> None:
+    store = FileSystemStore(tmp_path)
+    ref = EntityRef(ref_type="entity_id", value="ent-1")
+    signal = SignalEnvelope(
+        signal_id="s-1",
+        timestamp=_ts(5, 10),
+        source="ingestor.a",
+        payload_type="SocialPostSeen",
+        payload={"post_id": "p1"},
+        entity_refs=(ref,),
+    )
+    emission = EmissionEnvelope(
+        emission_id="e-1",
+        timestamp=_ts(5, 11),
+        emission_type="ScoresComputed",
+        payload={"score": 0.8},
+        caused_by="s-1",
+        entity_refs=(ref,),
+    )
+
+    store.write_signal(signal)
+    store.write_emission(emission)
+
+    read_signals = list(store.iter_signals(_ts(5, 0), _ts(5, 23, 59)))
+    read_emissions = list(store.iter_emissions(_ts(5, 0), _ts(5, 23, 59)))
+
+    assert read_signals == [signal]
+    assert read_emissions == [emission]
+
+
+def test_replay_ordering_across_partitions(tmp_path: Path) -> None:
+    store = FileSystemStore(tmp_path)
+
+    signal_ids: list[str] = []
+    for idx, ts in enumerate([_ts(5, 23, 59), _ts(6, 0, 1), _ts(6, 12, 0)], start=1):
+        signal = SignalEnvelope(
+            signal_id=f"s-{idx}",
+            timestamp=ts,
+            source="ingestor.a",
+            payload_type="Synthetic",
+            payload={"idx": idx},
+        )
+        store.write_signal(signal)
+        signal_ids.append(signal.signal_id)
+
+    replayed_ids = [s.signal_id for s in store.iter_signals(_ts(5, 0), _ts(6, 23, 59))]
+    assert replayed_ids == signal_ids
+
+
+def test_time_window_entity_ref_and_source_filtering(tmp_path: Path) -> None:
+    store = FileSystemStore(tmp_path)
+    ref_a = EntityRef(ref_type="entity_id", value="ent-a")
+    ref_b = EntityRef(ref_type="entity_id", value="ent-b")
+
+    store.write_signal(
+        SignalEnvelope(
+            signal_id="s-1",
+            timestamp=_ts(5, 10, 0),
+            source="ingestor.a",
+            payload_type="Synthetic",
+            payload={},
+            entity_refs=(ref_a,),
+        )
+    )
+    store.write_signal(
+        SignalEnvelope(
+            signal_id="s-2",
+            timestamp=_ts(5, 10, 5),
+            source="ingestor.b",
+            payload_type="Synthetic",
+            payload={},
+            entity_refs=(ref_b,),
+        )
+    )
+    store.write_signal(
+        SignalEnvelope(
+            signal_id="s-3",
+            timestamp=_ts(5, 11, 0),
+            source="ingestor.a",
+            payload_type="Synthetic",
+            payload={},
+            entity_refs=(ref_a,),
+        )
+    )
+
+    windowed = list(
+        store.iter_signals(
+            start=_ts(5, 10, 1),
+            end=_ts(5, 11, 0),
+            entity_ref=ref_a,
+            sources=["ingestor.a"],
+        )
+    )
+    assert [s.signal_id for s in windowed] == ["s-3"]
+
+
+def test_large_file_streaming_replay(tmp_path: Path) -> None:
+    store = FileSystemStore(tmp_path)
+    start = _ts(5, 0, 0)
+    total = 12000
+
+    for idx in range(total):
+        store.write_signal(
+            SignalEnvelope(
+                signal_id=f"s-{idx}",
+                timestamp=start + timedelta(seconds=idx),
+                source="ingestor.bulk",
+                payload_type="Bulk",
+                payload={"i": idx},
+            )
+        )
+
+    stream = store.iter_signals(start=start, end=start + timedelta(days=1))
+    first = next(stream)
+    second = next(stream)
+    assert first.signal_id == "s-0"
+    assert second.signal_id == "s-1"
+
+    count = 2 + sum(1 for _ in stream)
+    assert count == total
+
+
+def test_snapshot_write_creates_expected_file(tmp_path: Path) -> None:
+    store = FileSystemStore(tmp_path)
+    path = store.write_snapshot(
+        name="system_state",
+        snapshot_state={"entity_count": 3, "version": "v1"},
+        snapshot_time=_ts(5, 12, 0),
+    )
+    assert path.name == "system_state__2026-02-05T120000Z.json"
+    assert path.exists()