sequentialist-core 0.1.0__py3-none-any.whl

sequentialist/core/__init__.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright (C) 2026 Novikov Laboratories LLC
+# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
+"""sequentialist.core: what the sequentialist libraries share.
+
+Statistics live in sequential-analysis.  This package adds what turns its
+procedures into applications:
+
+spec       declarative specs with schemas, normal forms and ids (`Spec`, `load_spec`)
+engine     tasks that take records idempotently, feed runs, decide and survive crashes (`Engine`, `Handler`)
+storage    memory, a directory of files, SQLite (`open_storage`)
+audit      the hash-chained log of decisions and actions (`verify_audit`)
+report     JSON and Markdown reports in which every finding names its guarantee
+simulate   error rates of a whole pipeline by simulation
+replay     recorded data through the live code, on the data's clock
+sinks      notifications: log, JSON lines, webhooks
+privacy    pseudonyms for identifiers of people
+"""
+from .audit import GUARANTEES, AuditEntry, AuditError, Evidence, verify_audit
+from .engine import Command, Context, Engine, Handler, IngestResult, Record, Stream, Task, shard_of
+from .errors import (
+    ConflictError,
+    InvalidRecord,
+    SequentialistError,
+    SpecError,
+    StorageError,
+    TaskError,
+    TaskLocked,
+)
+from .privacy import pseudonymize
+from .replay import replay
+from .report import Finding, Report, Section, Table
+from .simulate import Proportion, proportion, run_simulations
+from .sinks import JsonlSink, LogSink, WebhookSink
+from .spec import Spec, json_schema, load_spec, parse_spec, spec_id
+from .storage import FileStorage, MemoryStorage, SQLiteStorage, Storage, open_storage
+from .version import __version__
+
+__all__ = [
+    "GUARANTEES",
+    "AuditEntry",
+    "AuditError",
+    "Command",
+    "ConflictError",
+    "Context",
+    "Engine",
+    "Evidence",
+    "FileStorage",
+    "Finding",
+    "Handler",
+    "IngestResult",
+    "InvalidRecord",
+    "JsonlSink",
+    "LogSink",
+    "MemoryStorage",
+    "Proportion",
+    "Record",
+    "Report",
+    "SQLiteStorage",
+    "Section",
+    "SequentialistError",
+    "Spec",
+    "SpecError",
+    "Storage",
+    "StorageError",
+    "Stream",
+    "Table",
+    "Task",
+    "TaskError",
+    "TaskLocked",
+    "WebhookSink",
+    "__version__",
+    "json_schema",
+    "load_spec",
+    "open_storage",
+    "parse_spec",
+    "proportion",
+    "pseudonymize",
+    "replay",
+    "run_simulations",
+    "shard_of",
+    "spec_id",
+    "verify_audit",
+]

sequentialist/core/audit.py

@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright (C) 2026 Novikov Laboratories LLC
+# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
+"""The audit log: every decision and action of a task, chained by hashes.
+
+An entry records what was decided (``decision``), on what grounds
+(``basis``: the rule and its numbers), from which evidence (``evidence``:
+for every stream consulted, the procedure, its guarantee, the step and
+the history digest of sequential-analysis), and, for
+actions of people, who did it and why (``actor``, ``reason``).
+
+Each entry holds the hash of the one before it::
+
+    hash_n = SHA-256(hash_{n-1} || canonical JSON of entry n without its hash)
+
+so an entry changed, removed or inserted afterwards breaks the chain at
+that point, which `verify_audit` reports.  The chain shows tampering; it
+does not prevent it, and it does not by itself make a system compliant
+with any regulation.
+
+The history digest in the evidence identifies the exact sequence of
+inputs a procedure had seen; given the archived observations, anyone can
+recompute it with sequential-analysis (in Python or TypeScript) and
+check that the decision followed from the data.
+"""
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any, Sequence
+
+from .canonical import canonical_json
+from .errors import SequentialistError
+
+FORMAT = "sequentialist.audit/1"
+GENESIS = hashlib.sha256(FORMAT.encode("utf-8")).hexdigest()
+#: What a conclusion can promise, as sequential-analysis states it on every procedure.
+GUARANTEES = ("exact", "asymptotic", "model", "approximate", "none")
+
+
+class AuditError(SequentialistError):
+    """The audit log is broken: the message names the first entry that does not verify."""
+
+
+@dataclass(frozen=True)
+class Evidence:
+    """One stream's state behind a decision."""
+
+    stream: str
+    run_id: str
+    t: int
+    procedure: dict[str, Any]  # the procedure's identity(): name, version, config
+    guarantee: str
+    history_digest: str
+    output: Any = None  # the stream's last output, encoded
+
+    @classmethod
+    def of(cls, stream: str, run: Any, output: Any = None) -> Evidence:
+        """Evidence from a sequential_analysis Run (between transitions)."""
+        return cls(stream, run.run_id, run.t, run.procedure.identity(), str(run.procedure.guarantee),
+                   run.history_digest, output)
+
+    def to_json(self) -> dict[str, Any]:
+        return {"stream": self.stream, "run_id": self.run_id, "t": self.t, "procedure": self.procedure,
+                "guarantee": self.guarantee, "history_digest": self.history_digest, "output": self.output}
+
+
+@dataclass(frozen=True)
+class AuditEntry:
+    n: int  # position in the task's log, from 0
+    task_id: str
+    spec_id: str
+    kind: str  # "created", "decision", "action", "segment", ...
+    decision: str | None
+    at: float  # seconds since the epoch, by the engine's clock
+    basis: dict[str, Any] = field(default_factory=dict)
+    evidence: tuple[dict[str, Any], ...] = ()
+    actor: str | None = None
+    reason: str | None = None
+    prev: str = GENESIS
+    hash: str = ""
+
+    def body(self) -> dict[str, Any]:
+        return {"format": FORMAT, "n": self.n, "task_id": self.task_id, "spec_id": self.spec_id, "kind": self.kind,
+                "decision": self.decision, "at": self.at, "basis": self.basis, "evidence": list(self.evidence),
+                "actor": self.actor, "reason": self.reason, "prev": self.prev}
+
+    def to_json(self) -> dict[str, Any]:
+        return {**self.body(), "hash": self.hash}
+
+    @classmethod
+    def from_json(cls, data: dict[str, Any]) -> AuditEntry:
+        return cls(n=data["n"], task_id=data["task_id"], spec_id=data["spec_id"], kind=data["kind"],
+                   decision=data["decision"], at=data["at"], basis=data["basis"], evidence=tuple(data["evidence"]),
+                   actor=data["actor"], reason=data["reason"], prev=data["prev"], hash=data["hash"])
+
+    def sealed(self) -> AuditEntry:
+        """The entry with its hash computed."""
+        body = self.body()
+        digest = hashlib.sha256((self.prev + canonical_json(body)).encode("utf-8")).hexdigest()
+        return AuditEntry(**{**self.__dict__, "hash": digest})
+
+    def same_event(self, other: AuditEntry) -> bool:
+        """The same decision or action, whenever it was written (`at` and the hashes aside)."""
+        mine, theirs = self.body(), other.body()
+        for name in ("at", "prev"):
+            mine.pop(name)
+            theirs.pop(name)
+        return canonical_json(mine) == canonical_json(theirs)
+
+
+def verify_audit(entries: Sequence[dict[str, Any]], task_id: str | None = None) -> int:
+    """Check the chain of a task's audit log; return the number of entries, AuditError at the first bad one."""
+    prev = GENESIS
+    for i, data in enumerate(entries):
+        try:
+            entry = AuditEntry.from_json(data)
+        except (KeyError, TypeError) as error:
+            raise AuditError(f"entry {i}: malformed ({error!r})") from None
+        if data.get("format") != FORMAT:
+            raise AuditError(f"entry {i}: format {data.get('format')!r}, expected {FORMAT!r}")
+        if entry.n != i:
+            raise AuditError(f"entry {i}: numbered {entry.n}; an entry is missing or out of place")
+        if task_id is not None and entry.task_id != task_id:
+            raise AuditError(f"entry {i}: belongs to task {entry.task_id!r}")
+        if entry.prev != prev:
+            raise AuditError(f"entry {i}: does not follow the entry before it")
+        if entry.sealed().hash != entry.hash:
+            raise AuditError(f"entry {i}: its content does not match its hash; it was changed")
+        for evidence in entry.evidence:
+            if evidence.get("guarantee") not in GUARANTEES:
+                raise AuditError(f"entry {i}: evidence without a known guarantee")
+        prev = entry.hash
+    return len(entries)

sequentialist/core/canonical.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright (C) 2026 Novikov Laboratories LLC
+# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
+"""JSON as the stored state needs it: canonical bytes for hashes, exact numbers when read back.
+
+These are the functions sequential-analysis uses for its checkpoints and
+history digests, so that a spec id or an audit hash computed here is
+computed the same way as the library's own digests.
+"""
+from __future__ import annotations
+
+import hashlib
+import math
+from typing import Any
+
+from sequential_analysis.canonical import canonical_json, loads
+
+__all__ = ["canonical_json", "digest", "is_plain", "loads"]
+
+
+def digest(value: Any) -> str:
+    """SHA-256 of the canonical JSON of `value`, as hex."""
+    return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
+
+
+def _text(value: Any) -> bool:
+    if type(value) is not str:
+        return False
+    try:
+        value.encode("utf-8")
+    except UnicodeEncodeError:
+        return False
+    return True
+
+
+def is_plain(value: Any) -> bool:
+    """True for JSON that survives storage exactly: finite floats, integers within 2**53, UTF-8 text."""
+    if value is None or type(value) is bool:
+        return True
+    if type(value) is int:
+        return abs(value) <= 2**53
+    if type(value) is float:
+        return math.isfinite(value)
+    if type(value) is str:
+        return _text(value)
+    if type(value) is list:
+        return all(is_plain(v) for v in value)
+    if type(value) is dict:
+        return all(_text(k) and is_plain(v) for k, v in value.items())
+    return False

sequentialist/core/cli.py

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright (C) 2026 Novikov Laboratories LLC
+# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
+"""The `sequentialist` command.  Every installed library adds its subcommands.
+
+    sequentialist tasks STORAGE                 list the tasks in a storage
+    sequentialist audit STORAGE TASK            verify a task's audit log and print it
+    sequentialist <library> ...                 e.g. `sequentialist ab --help`
+
+STORAGE is ``files:<directory>`` or ``sqlite:<file>``.
+
+A library registers a subcommand with an entry point in the group
+``sequentialist.commands``: a function that receives the subparsers object
+and adds its parser, with ``set_defaults(handler=...)``.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from typing import Any, Callable
+
+from .audit import AuditError, verify_audit
+from .errors import SequentialistError
+from .storage import open_storage
+from .version import __version__
+
+
+def _entry_points() -> list[Any]:
+    from importlib.metadata import entry_points
+
+    found = entry_points()
+    if hasattr(found, "select"):  # Python 3.10 and later
+        return list(found.select(group="sequentialist.commands"))
+    return list(found.get("sequentialist.commands", []))
+
+
+def tasks_command(commands: Any) -> None:
+    parser = commands.add_parser("tasks", help="list the tasks in a storage")
+    parser.add_argument("storage")
+
+    def handler(args: argparse.Namespace) -> int:
+        with open_storage(args.storage) as storage:
+            for task_id in storage.tasks():
+                print(task_id)
+        return 0
+
+    parser.set_defaults(handler=handler)
+
+
+def audit_command(commands: Any) -> None:
+    parser = commands.add_parser("audit", help="verify a task's audit log and print it")
+    parser.add_argument("storage")
+    parser.add_argument("task")
+    parser.add_argument("--json", action="store_true", help="print the entries as JSON lines")
+
+    def handler(args: argparse.Namespace) -> int:
+        with open_storage(args.storage) as storage:
+            entries = storage.audit_read(args.task)
+        try:
+            count = verify_audit(entries, args.task)
+        except AuditError as error:
+            print(f"audit log of {args.task}: BROKEN: {error}", file=sys.stderr)
+            return 1
+        for e in entries:
+            if args.json:
+                print(json.dumps(e, sort_keys=True, ensure_ascii=False))
+            else:
+                who = f" by {e['actor']} ({e['reason']})" if e.get("actor") else ""
+                print(f"{e['n']:>5}  {e['kind']:<9} {e['decision'] or '':<12}{who}  {json.dumps(e['basis'])}")
+        print(f"audit log of {args.task}: {count} entries, chain verified", file=sys.stderr)
+        return 0
+
+    parser.set_defaults(handler=handler)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="sequentialist", description="The sequentialist libraries.")
+    parser.add_argument("--version", action="version", version=f"sequentialist-core {__version__}")
+    parser.add_argument("--log-level", default="WARNING")
+    commands = parser.add_subparsers(dest="command")
+    for point in sorted(_entry_points(), key=lambda p: p.name):
+        register: Callable[[Any], None] = point.load()
+        register(commands)
+    args = parser.parse_args(argv)
+    logging.basicConfig(level=args.log_level.upper(), format="%(levelname)s %(name)s: %(message)s")
+    if not getattr(args, "handler", None):
+        parser.print_help()
+        return 2
+    try:
+        return int(args.handler(args))
+    except (SequentialistError, ValueError, OSError) as error:
+        print(f"sequentialist: {error}", file=sys.stderr)
+        return 2