deeprem 0.3.0__py3-none-any.whl

deeprem/__init__.py

@@ -0,0 +1,20 @@
+from .crypto import SigningKey, generate_encryption_key, sign_approval
+from .errors import (ApprovalRequired, ConflictError, DeepRemError, EvidenceError,
+                     IntegrityError, KeyRequired, NotFoundError, ValidationError)
+from .memory import Memory
+from .models import Hit, Record, Verification
+
+__version__ = "0.3.0"
+__all__ = ["Memory", "Record", "Hit", "Verification", "SigningKey", "generate_encryption_key", "sign_approval",
+           "DeepRemError", "ValidationError", "IntegrityError", "EvidenceError", "ApprovalRequired", "ConflictError",
+           "NotFoundError", "KeyRequired"]
+
+from .dynamics import Policy
+from .engine import Engine
+
+__all__ += ["Engine", "Policy"]
+
+from .subconscious import Subconscious
+from .subconscious_rules import StreamWeights, SubconsciousPolicy
+
+__all__ += ["Subconscious", "StreamWeights", "SubconsciousPolicy"]

deeprem/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

deeprem/_schema.py

@@ -0,0 +1,26 @@
+"""Versioned schemas are shipped as package resources and used at runtime."""
+from functools import lru_cache
+from importlib.resources import files
+import json
+
+from jsonschema import Draft202012Validator, FormatChecker
+from jsonschema.exceptions import ValidationError as SchemaError
+
+from ._util import canonical
+from .errors import ValidationError
+
+
+@lru_cache(maxsize=None)
+def validator(name: str) -> Draft202012Validator:
+    schema = json.loads(files("deeprem").joinpath("schemas", name + "-v1.schema.json").read_text(encoding="utf-8"))
+    return Draft202012Validator(schema, format_checker=FormatChecker())
+
+
+def validate(name: str, value) -> None:
+    canonical(value)
+    try:
+        validator(name).validate(value)
+    except SchemaError as exc:
+        location = ".".join(str(p) for p in exc.absolute_path) or "root"
+        # Do not include secret journal prose in an exception message.
+        raise ValidationError(f"invalid {name} at {location}: failed {exc.validator}") from exc

deeprem/_util.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import re
+import tempfile
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+from .errors import IntegrityError, ValidationError
+
+MAX_BYTES = 16 * 1024 * 1024
+ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$"
+
+
+def json_value(value: Any, *, _seen: set[int] | None = None, _depth: int = 0) -> None:
+    if _depth > 64:
+        raise ValidationError("JSON nesting exceeds the v1 limit of 64")
+    if value is None or type(value) in (str, int, bool):
+        return
+    if type(value) is float and math.isfinite(value):
+        return
+    if type(value) not in (list, dict) or (type(value) is dict and not all(type(key) is str for key in value)):
+        raise ValidationError("values must be finite JSON types with string object keys")
+    seen = set() if _seen is None else _seen
+    identity = id(value)
+    if identity in seen:
+        raise ValidationError("cyclic objects are not JSON")
+    seen.add(identity)
+    try:
+        for item in (value.values() if isinstance(value, dict) else value):
+            json_value(item, _seen=seen, _depth=_depth + 1)
+    finally:
+        seen.remove(identity)
+
+
+def canonical(value: Any) -> bytes:
+    json_value(value)
+    try:
+        return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("ascii")
+    except (ValueError, TypeError, RecursionError) as exc:
+        raise ValidationError("invalid JSON value") from exc
+
+
+def sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _pairs(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValidationError("duplicate JSON object key")
+        result[key] = value
+    return result
+
+
+def loads(raw: bytes | str) -> Any:
+    if len(raw) > MAX_BYTES:
+        raise ValidationError("JSON document exceeds the 16 MiB v1 limit")
+    try:
+        value = json.loads(raw, object_pairs_hook=_pairs)
+        json_value(value)
+        return value
+    except (UnicodeError, ValueError, TypeError, RecursionError) as exc:
+        raise ValidationError("invalid JSON document") from exc
+
+
+def now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def iso(value: datetime) -> str:
+    return value.astimezone(timezone.utc).isoformat(timespec="microseconds")
+
+
+def timestamp(value: str) -> datetime:
+    if not isinstance(value, str):
+        raise ValidationError("timestamp must be an ISO8601 string with a timezone")
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        if parsed.tzinfo is None or parsed.utcoffset() is None:
+            raise ValueError("missing timezone")
+        return parsed.astimezone(timezone.utc)
+    except (ValueError, OverflowError) as exc:
+        raise ValidationError("timestamp must be ISO8601 with a timezone") from exc
+
+
+def boundary(value: str | None, *, relative_to: datetime) -> datetime | None:
+    if value is None:
+        return None
+    if isinstance(value, str) and re.fullmatch(r"[1-9][0-9]{0,4}d", value):
+        return relative_to - timedelta(days=int(value[:-1]))
+    return timestamp(value)
+
+
+def safe_id(value: str) -> str:
+    if not isinstance(value, str) or re.fullmatch(ID_PATTERN, value) is None:
+        raise ValidationError("unsafe or invalid entry id")
+    return value
+
+
+def text(value: str, name: str, maximum: int = 2048) -> str:
+    if not isinstance(value, str) or not value.strip() or len(value) > maximum:
+        raise ValidationError(f"{name} must be nonblank text of at most {maximum} characters")
+    return value
+
+
+def integer(value: int, name: str, minimum: int = 0, maximum: int = 10000) -> int:
+    if type(value) is not int or not minimum <= value <= maximum:
+        raise ValidationError(f"{name} must be an integer in [{minimum}, {maximum}]")
+    return value
+
+
+def safe_path(path: Path, root: Path) -> Path:
+    if not path.is_relative_to(root):
+        raise IntegrityError("path escapes the configured root")
+    for item in (path, *path.parents):
+        if item == root:
+            break
+        if item.is_symlink():
+            raise IntegrityError("symlinks are not allowed within canonical storage")
+    if not path.resolve().is_relative_to(root):
+        raise IntegrityError("path escapes the configured root")
+    return path
+
+
+def fsync_dir(path: Path) -> None:
+    if os.name == "posix":
+        fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+
+
+def atomic_write(path: Path, data: bytes, *, replace: bool = False) -> None:
+    """Publish a fully synced file; never replace canonical history by default.
+
+    All canonical calls are serialized by an OS-held lock. Hard-link publication
+    also prevents overwriting a pre-existing destination after a crash.
+    """
+    fd, name = tempfile.mkstemp(prefix=".pending-", dir=path.parent)
+    temp = Path(name)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(data)
+            handle.flush()
+            os.fsync(handle.fileno())
+        if replace:
+            os.replace(temp, path)
+        else:
+            os.link(temp, path)
+            temp.unlink()
+        fsync_dir(path.parent)
+    finally:
+        temp.unlink(missing_ok=True)
+
+
+def read_bytes(path: Path) -> bytes:
+    if path.stat().st_size > MAX_BYTES:
+        raise IntegrityError("canonical file exceeds the 16 MiB v1 limit")
+    return path.read_bytes()

deeprem/cli.py

@@ -0,0 +1,210 @@
+"""JSON-first CLI; private keys are accepted only by file path, never inline."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from filelock import Timeout
+
+from ._util import atomic_write, canonical, loads
+from .crypto import SigningKey, generate_encryption_key, sign_approval
+from .errors import DeepRemError, ValidationError
+from .memory import Memory
+
+
+def _read_json(path: str):
+    return loads(sys.stdin.read() if path == "-" else Path(path).read_bytes())
+
+
+def _emit(value):
+    if hasattr(value, "to_dict"):
+        value = value.to_dict()
+    elif isinstance(value, list):
+        value = [item.to_dict() if hasattr(item, "to_dict") else item for item in value]
+    print(json.dumps(value, ensure_ascii=True, sort_keys=True, indent=2, allow_nan=False))
+
+
+def parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="deeprem", description="Local memory. Use subconscious for autobiographical output hooks, runtime for the older Engine, or the legacy Memory commands below.")
+    p.add_argument("--root", default="memory")
+    p.add_argument("--journal", help="ghostjournal root; not needed for keygen, approve or verify --log-only")
+    p.add_argument("--agent", help="required on first initialization")
+    p.add_argument("--actor", help="audit label, not an authentication identity")
+    p.add_argument("--signing-key-file", help="external writer Ed25519 PEM file")
+    p.add_argument("--encryption-key-file", help="external Fernet key file")
+    p.add_argument("--reviewer-public-key-file", help="base64 Ed25519 public key, read at store creation")
+    p.add_argument("--checkpoint", help="trusted checkpoint kept outside the memory root")
+    sub = p.add_subparsers(dest="command", required=True)
+    sub.add_parser("init")
+    kg = sub.add_parser("keygen")
+    kg.add_argument("kind", choices=["signing", "encryption"])
+    kg.add_argument("--out", required=True, help="new external key file; signing also writes OUT.pub")
+    ap = sub.add_parser("approve", help="operator-only signing of a reviewed approval request")
+    ap.add_argument("--request", required=True)
+    ap.add_argument("--reviewer-key-file", required=True)
+    ap.add_argument("--out", required=True)
+    cp = sub.add_parser("consolidate")
+    cp.add_argument("--since", default="30d")
+    cp.add_argument("--until")
+    cp.add_argument("--min-entries", type=int, default=3)
+    cp.add_argument("--min-days", type=int, default=3)
+    cp.add_argument("--dimensions", nargs="+", choices=["theme", "focus", "tag"], default=["theme", "focus"])
+    pp = sub.add_parser("propose", help="read a proposal argument object, including supporting entry ids")
+    pp.add_argument("--json", default="-", help="file or '-' for stdin")
+    for command in ["commit", "reject", "deactivate", "resolve"]:
+        sp = sub.add_parser(command)
+        sp.add_argument("id")
+        sp.add_argument("--reason", required=True)
+        sp.add_argument("--approval", help="reviewer-signed approval JSON")
+        sp.add_argument("--client-key")
+    ev = sub.add_parser("add-evidence")
+    ev.add_argument("id")
+    ev.add_argument("--support", action="append", default=[])
+    ev.add_argument("--contradict", action="append", default=[])
+    ev.add_argument("--reason", required=True)
+    ev.add_argument("--client-key")
+    pa = sub.add_parser("prepare-approval")
+    pa.add_argument("action", choices=["commit", "reject", "deactivate", "resolve"])
+    pa.add_argument("id")
+    pa.add_argument("--reason", required=True)
+    pa.add_argument("--ttl-seconds", type=int, default=3600)
+    gp = sub.add_parser("get")
+    gp.add_argument("id")
+    lp = sub.add_parser("list")
+    lp.add_argument("--status")
+    lp.add_argument("--kind")
+    lp.add_argument("--limit", type=int, default=100)
+    rp = sub.add_parser("recall")
+    rp.add_argument("query")
+    rp.add_argument("-k", type=int, default=8)
+    rp.add_argument("--include-inactive", action="store_true")
+    pc = sub.add_parser("prompt-context")
+    pc.add_argument("query")
+    pc.add_argument("-k", type=int, default=6)
+    pc.add_argument("--max-chars", type=int, default=6000)
+    for cmd in ("identity", "open-threads", "seal-journal", "reindex"):
+        sub.add_parser(cmd)
+    ch = sub.add_parser("changes")
+    ch.add_argument("--since", default="90d")
+    ch.add_argument("--until")
+    dp = sub.add_parser("decay")
+    dp.add_argument("--as-of")
+    dp.add_argument("--threshold", type=float, default=0.1)
+    vp = sub.add_parser("verify")
+    vp.add_argument("--log-only", action="store_true", help="explicitly skip journal evidence verification")
+    ck = sub.add_parser("checkpoint")
+    ck.add_argument("--out", required=True, help="new path outside memory root; protect it with independent permissions")
+    return p
+
+
+def _outside(path: str, root: Path) -> Path:
+    result = Path(path).expanduser().resolve()
+    if result.is_relative_to(root):
+        raise ValidationError("private keys and trusted checkpoints must be stored outside the memory root")
+    return result
+
+
+def _run(args) -> int:
+    root = Path(args.root).expanduser().resolve()
+    if args.command == "keygen":
+        out = _outside(args.out, root)
+        if args.kind == "signing":
+            pub_path = Path(str(out) + ".pub")
+            if out.exists() or pub_path.exists():
+                raise ValidationError("key output already exists")
+            key = SigningKey.generate()
+            key.save(out)
+            atomic_write(pub_path, (key.public_key + "\n").encode("ascii"))
+            _emit({"private_key_file": str(out), "public_key_file": str(pub_path), "public_key": key.public_key})
+        else:
+            out.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+            atomic_write(out, generate_encryption_key() + b"\n")
+            _emit({"encryption_key_file": str(out)})
+        return 0
+    if args.command == "approve":
+        key = SigningKey.load(_outside(args.reviewer_key_file, root))
+        approval = sign_approval(_read_json(args.request), key)
+        out = Path(args.out)
+        out.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        atomic_write(out, canonical(approval) + b"\n")
+        _emit({"approval_file": str(out)})
+        return 0
+    log_only = args.command == "verify" and args.log_only
+    if args.journal is None and not log_only:
+        raise ValidationError("--journal is required for memory operations")
+    signer = SigningKey.load(_outside(args.signing_key_file, root)) if args.signing_key_file else None
+    key = _outside(args.encryption_key_file, root).read_bytes().strip() if args.encryption_key_file else None
+    reviewer = Path(args.reviewer_public_key_file).read_text(encoding="ascii").strip() if args.reviewer_public_key_file else None
+    checkpoint = _read_json(str(_outside(args.checkpoint, root))) if args.checkpoint else None
+    m = Memory(args.root, journal=None if log_only else args.journal, agent=args.agent, signing_key=signer,
+               encryption_key=key, reviewer_public_key=reviewer, checkpoint=checkpoint)
+    command = args.command
+    if command == "init":
+        result = {"root": str(m.root), "agent": m.agent, "verification": m.verify().to_dict()}
+    elif command == "propose":
+        data = _read_json(args.json)
+        allowed = {"statement", "supporting_entries", "contradicting_entries", "kind", "theme", "tags", "protected", "supersedes", "half_life_days", "client_key"}
+        if not isinstance(data, dict) or set(data) - allowed or "statement" not in data:
+            raise ValidationError("proposal JSON has missing or unknown argument fields")
+        result = m.propose(**data, actor=args.actor)
+    elif command == "consolidate":
+        result = m.consolidate(since=args.since, until=args.until, min_entries=args.min_entries, min_days=args.min_days,
+                               dimensions=tuple(args.dimensions), actor=args.actor)
+    elif command in {"commit", "reject", "deactivate", "resolve"}:
+        result = getattr(m, command)(args.id, reason=args.reason, approval=_read_json(args.approval) if args.approval else None,
+                                     client_key=args.client_key, actor=args.actor)
+    elif command == "add-evidence":
+        result = m.add_evidence(args.id, supporting_entries=args.support, contradicting_entries=args.contradict,
+                                reason=args.reason, client_key=args.client_key, actor=args.actor)
+    elif command == "prepare-approval":
+        result = m.prepare_approval(args.action, args.id, reason=args.reason, ttl_seconds=args.ttl_seconds)
+    elif command == "get":
+        result = m.get(args.id)
+    elif command == "list":
+        result = m.list(status=args.status, kind=args.kind, limit=args.limit)
+    elif command == "recall":
+        result = m.recall(args.query, k=args.k, include_inactive=args.include_inactive)
+    elif command == "prompt-context":
+        print(m.prompt_context(args.query, k=args.k, max_chars=args.max_chars))
+        return 0
+    elif command == "identity":
+        result = m.identity()
+    elif command == "open-threads":
+        result = m.open_threads()
+    elif command == "changes":
+        result = m.changes(since=args.since, until=args.until)
+    elif command == "decay":
+        result = m.decay(as_of=args.as_of, threshold=args.threshold, actor=args.actor)
+    elif command == "seal-journal":
+        result = {"newly_sealed": m.seal_journal(actor=args.actor)}
+    elif command == "verify":
+        result = m.verify(check_journal=not log_only)
+    elif command == "reindex":
+        result = m.reindex()
+    elif command == "checkpoint":
+        out = _outside(args.out, root)
+        out.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        result = m.checkpoint()
+        atomic_write(out, canonical(result) + b"\n")
+    else:
+        raise ValidationError("unknown command")
+    _emit(result)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv[:1] == ["subconscious"]:
+        from .subconscious_cli import main as subconscious_main
+        return subconscious_main(argv[1:])
+    if argv[:1] == ["runtime"]:
+        from .runtime_cli import main as runtime_main
+        return runtime_main(argv[1:])
+    args = parser().parse_args(argv)
+    try:
+        return _run(args)
+    except (DeepRemError, OSError, Timeout) as exc:
+        print(json.dumps({"error": type(exc).__name__, "message": str(exc)}, ensure_ascii=True), file=sys.stderr)
+        return 2

deeprem/crypto.py

@@ -0,0 +1,151 @@
+"""Optional cryptography adapter. No custom ciphers, key derivation, or nonce code."""
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+from typing import Any
+
+from ._schema import validate
+from ._util import atomic_write, canonical
+from .errors import ApprovalRequired, IntegrityError, KeyRequired, ValidationError
+
+APPROVAL_DOMAIN = b"deeprem:review:v1\x00"
+EVENT_DOMAIN = b"deeprem:event:v1\x00"
+
+
+def _imports():
+    try:
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
+        from cryptography.hazmat.primitives import serialization
+        return Ed25519PrivateKey, Ed25519PublicKey, serialization
+    except ImportError as exc:
+        raise KeyRequired("install deeprem[crypto] to use signing, review gates or encryption") from exc
+
+
+def b64(value: bytes) -> str:
+    return base64.urlsafe_b64encode(value).decode("ascii")
+
+
+def unb64(value: str) -> bytes:
+    try:
+        return base64.b64decode(value.encode("ascii"), altchars=b"-_", validate=True)
+    except (ValueError, UnicodeError, AttributeError) as exc:
+        raise ValidationError("invalid base64 key or signature") from exc
+
+
+class SigningKey:
+    """Ed25519 private key. Keep reviewer keys outside the agent's OS identity."""
+
+    def __init__(self, raw: bytes):
+        private, _, _ = _imports()
+        try:
+            self._key = private.from_private_bytes(raw)
+        except (TypeError, ValueError) as exc:
+            raise ValidationError("Ed25519 private keys must be 32 bytes") from exc
+
+    @classmethod
+    def generate(cls) -> SigningKey:
+        private, _, serialization = _imports()
+        raw = private.generate().private_bytes(serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption())
+        return cls(raw)
+
+    @classmethod
+    def load(cls, path: str | Path) -> SigningKey:
+        private, _, serialization = _imports()
+        try:
+            key = serialization.load_pem_private_key(Path(path).read_bytes(), password=None)
+            if not isinstance(key, private):
+                raise ValueError("not Ed25519")
+            return cls(key.private_bytes(serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption()))
+        except (TypeError, ValueError) as exc:
+            raise ValidationError("expected an unencrypted Ed25519 PKCS8 PEM key") from exc
+
+    @property
+    def public_key(self) -> str:
+        _, _, serialization = _imports()
+        return b64(self._key.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw))
+
+    def sign(self, data: bytes) -> str:
+        return b64(self._key.sign(data))
+
+    def save(self, path: str | Path) -> None:
+        """Write a NEW 0600 PEM key file; never print private material or overwrite."""
+        _, _, serialization = _imports()
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        atomic_write(path, self._key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()))
+
+
+def verify_signature(public_key: str, signature: str, data: bytes) -> None:
+    _, public, _ = _imports()
+    try:
+        public.from_public_bytes(unb64(public_key)).verify(unb64(signature), data)
+    except Exception as exc:
+        # Import errors are handled before this block, invalid signatures fail closed.
+        raise IntegrityError("signature verification failed") from exc
+
+
+def validate_public_key(value: str) -> None:
+    _, public, _ = _imports()
+    try:
+        public.from_public_bytes(unb64(value))
+    except (ValueError, TypeError) as exc:
+        raise ValidationError("invalid Ed25519 public key") from exc
+
+
+def sign_approval(request: dict[str, Any], reviewer: SigningKey) -> dict[str, Any]:
+    """Run in an operator-controlled process, after reviewing the request/evidence."""
+    validate("approval-request", request)
+    return {"request": request, "signature": reviewer.sign(APPROVAL_DOMAIN + canonical(request))}
+
+
+def verify_approval_signature(approval: dict, public_key: str | None) -> None:
+    if public_key is None:
+        raise ApprovalRequired("this store has no reviewer; protected transitions are disabled")
+    try:
+        validate("approval", approval)
+        verify_signature(public_key, approval["signature"], APPROVAL_DOMAIN + canonical(approval["request"]))
+    except (IntegrityError, ValidationError) as exc:
+        raise ApprovalRequired("independent reviewer signature is missing or invalid") from exc
+
+
+def generate_encryption_key() -> bytes:
+    try:
+        from cryptography.fernet import Fernet
+        return Fernet.generate_key()
+    except ImportError as exc:
+        raise KeyRequired("install deeprem[crypto] for encryption") from exc
+
+
+class Codec:
+    def __init__(self, encrypted: bool, key: bytes | None):
+        self.encrypted = encrypted
+        self._cipher = None
+        if encrypted:
+            if key is None:
+                raise KeyRequired("this store requires its external Fernet encryption key")
+            try:
+                from cryptography.fernet import Fernet
+                self._cipher = Fernet(key)
+            except ImportError as exc:
+                raise KeyRequired("install deeprem[crypto] for encryption") from exc
+            except (TypeError, ValueError) as exc:
+                raise ValidationError("invalid Fernet key; use generate_encryption_key()") from exc
+        elif key is not None:
+            raise ValidationError("cannot silently change an existing store's encryption mode")
+
+    @property
+    def suffix(self) -> str:
+        return ".fernet" if self.encrypted else ".json"
+
+    def encode(self, data: bytes) -> bytes:
+        return self._cipher.encrypt(data) if self._cipher else data + b"\n"
+
+    def decode(self, data: bytes) -> bytes:
+        if self._cipher is None:
+            return data
+        from cryptography.fernet import InvalidToken
+        try:
+            return self._cipher.decrypt(data)
+        except InvalidToken as exc:
+            raise IntegrityError("encrypted event authentication failed (wrong key or changed ciphertext)") from exc