mnemox 0.2.0__py3-none-any.whl

ember/__init__.py

@@ -0,0 +1,3 @@
+"""Ember — local-first memory system."""
+
+__version__ = "0.2.0"

ember/__main__.py

@@ -0,0 +1,129 @@
+"""Entry point for ``python -m ember``.
+
+Usage::
+
+    python -m ember mirror --project-id <id> [--output-dir memory] [--db-path /path/to.db]
+    python -m ember serve
+    python -m ember init [--mode project|global|server]
+    python -m ember --version
+"""
+
+import argparse
+import sys
+
+from ember.cli import cmd_backfill, cmd_drain, cmd_init, get_version
+
+
+def _cmd_mirror(args: argparse.Namespace) -> None:
+    from ember.storage import store as _store_factory
+    from ember.mirror import render_to_md
+
+    s = _store_factory(args.db_path or None)
+    try:
+        result = render_to_md(args.project_id, args.output_dir, store=s)
+    finally:
+        s.close()
+
+    total_atoms = sum(result.values())
+    total_files = len(result)
+    print(f"mirrored {total_atoms} atoms across {total_files} files")
+
+
+def _cmd_serve(args: argparse.Namespace) -> None:
+    import asyncio
+    from ember.server import run_stdio
+
+    asyncio.run(run_stdio())
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="ember",
+        description="Ember local-first memory system.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"ember {get_version()}",
+    )
+    sub = parser.add_subparsers(dest="command", metavar="<command>")
+
+    mirror_p = sub.add_parser("mirror", help="Render Ember records to memory/*.md files.")
+    mirror_p.add_argument("--project-id", required=True, help="Ember project identifier.")
+    mirror_p.add_argument(
+        "--output-dir",
+        default="memory",
+        help="Output directory for .md files (default: memory).",
+    )
+    mirror_p.add_argument(
+        "--db-path",
+        default=None,
+        help="Path to the Ember SQLite DB (default: resolved via EMBER_DB_PATH or ~/.ember/ember.db).",
+    )
+
+    sub.add_parser("serve", help="Start the Ember MCP server over stdio.")
+
+    init_p = sub.add_parser("init", help="Initialise the Ember store and write config.json.")
+    init_p.add_argument(
+        "--mode",
+        choices=["project", "global", "server"],
+        default=None,
+        help="Initialisation mode (default: project).",
+    )
+
+    drain_p = sub.add_parser("drain", help="Drain the capture queue and write records to the store.")
+    drain_p.add_argument(
+        "--queue-path",
+        default=None,
+        dest="queue_path",
+        help="Path to capture-queue.jsonl (default: <project-root>/.ember/capture-queue.jsonl).",
+    )
+    drain_p.add_argument(
+        "--db-path",
+        default=None,
+        dest="db_path",
+        help="Path to the Ember SQLite DB (default: resolved via EMBER_DB_PATH or mode).",
+    )
+    drain_p.add_argument(
+        "--project-id",
+        default=None,
+        dest="project_id",
+        help="Bound project scope; only records with this project_id are written.",
+    )
+
+    backfill_p = sub.add_parser(
+        "backfill",
+        help="Re-emit pre-existing atoms into the oplog so they sync to the hub.",
+    )
+    backfill_p.add_argument(
+        "--db-path",
+        default=None,
+        dest="db_path",
+        help="Path to the Ember SQLite DB (default: resolved via EMBER_DB_PATH or mode).",
+    )
+    backfill_p.add_argument(
+        "--project-id",
+        default=None,
+        dest="project_id",
+        help="Project scope to backfill (default: derived from the project root).",
+    )
+
+    args = parser.parse_args()
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    if args.command == "mirror":
+        _cmd_mirror(args)
+    elif args.command == "serve":
+        _cmd_serve(args)
+    elif args.command == "init":
+        sys.exit(cmd_init(args.mode))
+    elif args.command == "drain":
+        sys.exit(cmd_drain(args.queue_path, args.db_path, args.project_id))
+    elif args.command == "backfill":
+        sys.exit(cmd_backfill(args.db_path, args.project_id))
+
+
+if __name__ == "__main__":
+    main()

ember/capture/__init__.py

@@ -0,0 +1,6 @@
+"""Ember capture: drain queue consumer + secret scanner."""
+
+from .drain import drain
+from .secrets import looks_like_secret
+
+__all__ = ["drain", "looks_like_secret"]

ember/capture/drain.py

@@ -0,0 +1,341 @@
+"""Drain the ember capture queue into the write pipeline.
+
+Reads .ember/capture-queue.jsonl, validates each record, filters secrets and
+foreign project_ids, then calls ember.write.write() for each valid record.
+
+Durability model (atomic rename):
+  1. queue absent/empty → zero-counts no-op.
+  2. os.replace(queue, queue+".processing") — atomic rename; producer keeps
+     appending to the original path without data loss.
+  3. Read .processing line-by-line, write() each valid record.
+  4. Clean completion → os.remove(.processing) (== rotation/truncation).
+  5. Stale .processing at entry → drain it first.
+  If write() raises mid-drain: leave .processing in place and re-raise after
+  logging; next run reprocesses the whole batch (may double-write succeeded
+  records; accepted — cosine invalidate_superseded mitigates duplicates).
+  Skips (malformed/foreign/secret) never leave .processing.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ember.storage.store import Store
+
+# ---------------------------------------------------------------------------
+# Timestamp validation
+# ---------------------------------------------------------------------------
+
+# Accepts second-precision (ember-capture.sh) and millisecond-precision (DB DEFAULT).
+# Pattern: YYYY-MM-DDTHH:MM:SS[.fraction]Z
+_TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$")
+
+
+def _validate_ts(ts: object) -> "str | None":
+    """Validate an ISO-8601 UTC timestamp string.
+
+    Returns the original string if valid, else None.
+    Accepts second-precision (YYYY-MM-DDTHH:MM:SSZ) and millisecond-precision
+    (YYYY-MM-DDTHH:MM:SS.fffZ).
+    """
+    if not isinstance(ts, str):
+        return None
+    if not _TS_RE.match(ts):
+        return None
+    # Round-trip validation (catches out-of-range dates like month=13)
+    try:
+        # Normalise to a form datetime.strptime can parse.
+        ts_clean = ts.rstrip("Z")
+        if "." in ts_clean:
+            # Truncate/pad fractional to 6 digits for %f
+            parts = ts_clean.split(".")
+            frac = parts[1][:6].ljust(6, "0")
+            ts_clean = f"{parts[0]}.{frac}"
+            datetime.strptime(ts_clean, "%Y-%m-%dT%H:%M:%S.%f").replace(
+                tzinfo=timezone.utc
+            )
+        else:
+            datetime.strptime(ts_clean, "%Y-%m-%dT%H:%M:%S").replace(
+                tzinfo=timezone.utc
+            )
+    except ValueError:
+        return None
+    return ts
+
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_MAX_LINE_BYTES = 1 * 1024 * 1024  # 1 MB
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def drain(
+    queue_path: "str | Path | None" = None,
+    project_id: "str | None" = None,
+    *,
+    store: "Store | None" = None,
+) -> dict:
+    """Drain the capture queue and write valid records to the store.
+
+    Args:
+        queue_path: Path to capture-queue.jsonl. If None, resolves via
+                    EMBER_PROJECT_ROOT (or cwd) + /.ember/capture-queue.jsonl.
+        project_id: Bound project scope. If None, reads EMBER_PROJECT_ID from
+                    env; if still None, accepts any record project_id (unbound).
+        store: Optional injected Store. If None, opens one for the drain and
+               closes it when done.
+
+    Returns:
+        dict with keys: written, skipped_malformed, skipped_secret,
+        skipped_foreign.
+    """
+    from ember.capture.secrets import looks_like_secret
+    from ember.write import write
+    from ember.storage import store as open_store
+
+    counts: dict[str, int] = {
+        "written": 0,
+        "skipped_malformed": 0,
+        "skipped_secret": 0,
+        "skipped_foreign": 0,
+    }
+
+    # Resolve queue path
+    if queue_path is None:
+        from ember.config import get_project_root
+        root = get_project_root()
+        resolved_queue = Path(root) / ".ember" / "capture-queue.jsonl"
+    else:
+        resolved_queue = Path(queue_path)
+
+    processing_path = Path(str(resolved_queue) + ".processing")
+
+    # Resolve bound project_id
+    if project_id is None:
+        project_id = os.environ.get("EMBER_PROJECT_ID") or None
+
+    # Open store if not injected
+    own_store = store is None
+    if own_store:
+        store = open_store()
+
+    try:
+        _drain_inner(
+            resolved_queue,
+            processing_path,
+            project_id,
+            store,
+            counts,
+            looks_like_secret,
+            write,
+        )
+    finally:
+        if own_store:
+            store.close()
+
+    return counts
+
+
+def _drain_inner(
+    queue: Path,
+    processing: Path,
+    project_id: "str | None",
+    store: "Store",
+    counts: dict,
+    looks_like_secret_fn,
+    write_fn,
+) -> None:
+    """Core drain logic (separated for testability)."""
+
+    # Handle stale .processing from a prior crash — drain it first.
+    if processing.exists():
+        _drain_file(
+            processing,
+            project_id,
+            store,
+            counts,
+            looks_like_secret_fn,
+            write_fn,
+        )
+        # If drain succeeds, remove stale processing file.  If removal fails,
+        # return early to avoid os.replace(queue, processing) overwriting the
+        # not-yet-removed file with a fresh batch.
+        try:
+            os.remove(processing)
+        except OSError as exc:
+            print(
+                f"ember drain: warning: could not remove stale .processing: {exc}",
+                file=sys.stderr,
+            )
+            return
+
+    # Check if queue file exists and is non-empty.
+    if not queue.exists():
+        return
+    if queue.stat().st_size == 0:
+        return
+
+    # Atomic rename: producer can keep writing to the original path.
+    try:
+        os.replace(queue, processing)
+    except OSError as exc:
+        print(
+            f"ember drain: error: could not rename queue to processing: {exc}",
+            file=sys.stderr,
+        )
+        return
+
+    # Drain the .processing file.
+    _drain_file(
+        processing,
+        project_id,
+        store,
+        counts,
+        looks_like_secret_fn,
+        write_fn,
+    )
+
+    # Clean completion: remove .processing.
+    try:
+        os.remove(processing)
+    except OSError as exc:
+        print(
+            f"ember drain: warning: could not remove .processing after drain: {exc}",
+            file=sys.stderr,
+        )
+
+
+def _drain_file(
+    path: Path,
+    project_id: "str | None",
+    store: "Store",
+    counts: dict,
+    looks_like_secret_fn,
+    write_fn,
+) -> None:
+    """Read path line-by-line and process each record.
+
+    Raises if write_fn raises (caller handles cleanup). Skips do NOT raise.
+    """
+    try:
+        data = path.read_bytes()
+    except OSError as exc:
+        print(
+            f"ember drain: error: could not read {path}: {exc}",
+            file=sys.stderr,
+        )
+        return
+
+    for raw_line in data.split(b"\n"):
+        line = raw_line.strip()
+        if not line:
+            continue
+
+        # 1. Size guard
+        if len(line) > _MAX_LINE_BYTES:
+            print(
+                "ember drain: skipped malformed record: line exceeds 1MB",
+                file=sys.stderr,
+            )
+            counts["skipped_malformed"] += 1
+            continue
+
+        # 2. JSON parse
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError as exc:
+            print(
+                f"ember drain: skipped malformed record: JSONDecodeError: {exc}",
+                file=sys.stderr,
+            )
+            counts["skipped_malformed"] += 1
+            continue
+
+        if not isinstance(record, dict):
+            print(
+                "ember drain: skipped malformed record: not a JSON object",
+                file=sys.stderr,
+            )
+            counts["skipped_malformed"] += 1
+            continue
+
+        # 3. Required fields: content (non-empty str), project_id (str)
+        content = record.get("content")
+        rec_project_id = record.get("project_id")
+
+        if not isinstance(content, str) or not content.strip():
+            print(
+                "ember drain: skipped malformed record: missing or empty content",
+                file=sys.stderr,
+            )
+            counts["skipped_malformed"] += 1
+            continue
+
+        if not isinstance(rec_project_id, str) or not rec_project_id:
+            print(
+                "ember drain: skipped malformed record: missing or invalid project_id",
+                file=sys.stderr,
+            )
+            counts["skipped_malformed"] += 1
+            continue
+
+        # 4. Bound-scope check
+        if project_id is not None and rec_project_id != project_id:
+            print(
+                f"ember drain: skipped foreign record: project_id={rec_project_id!r} "
+                f"(bound to {project_id!r})",
+                file=sys.stderr,
+            )
+            counts["skipped_foreign"] += 1
+            continue
+
+        # 5. Secret scanner — log WITHOUT echoing content
+        if looks_like_secret_fn(content):
+            print(
+                "ember drain: skipped secret record: content looks like a credential",
+                file=sys.stderr,
+            )
+            counts["skipped_secret"] += 1
+            continue
+
+        # 6. Validate metadata
+        raw_metadata = record.get("metadata")
+        metadata = raw_metadata if isinstance(raw_metadata, dict) else None
+
+        # 7. Validate type
+        raw_type = record.get("type")
+        rec_type = raw_type if isinstance(raw_type, str) and raw_type else "diary"
+
+        # 8. Validate event timestamp
+        valid_ts = _validate_ts(record.get("ts"))
+        created_at = valid_ts  # None if invalid → write() uses SQL DEFAULT now
+
+        if record.get("ts") and valid_ts is None:
+            print(
+                "ember drain: malformed ts for record — using drain time as created_at",
+                file=sys.stderr,
+            )
+
+        # 9. Write (may raise — caller handles cleanup)
+        write_fn(
+            content,
+            metadata=metadata,
+            project_id=rec_project_id,
+            store=store,
+            type=rec_type,
+            created_at=created_at,
+        )
+        counts["written"] += 1

ember/capture/secrets.py

@@ -0,0 +1,137 @@
+"""Secret scanner for ember capture drain.
+
+Detects credentials/secrets in content strings and prevents them from being
+written to the store. Conservative: over-skip is acceptable; under-skip is not.
+
+All patterns compiled at module level for performance.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+
+# ---------------------------------------------------------------------------
+# Compiled patterns
+# ---------------------------------------------------------------------------
+
+# PEM private key block
+_RE_PEM = re.compile(r"-----BEGIN\s+(?:\w+\s+)*PRIVATE KEY-----", re.IGNORECASE)
+
+# OpenAI API key: sk- followed by 20+ alphanumeric chars
+_RE_OPENAI = re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")
+
+# AWS Access Key ID
+_RE_AWS_KEY = re.compile(r"\bAKIA[0-9A-Z]{16}\b")
+
+# AWS secret access key assignment
+_RE_AWS_SECRET = re.compile(r"aws_secret_access_key\s*[=:]\s*\S{30,}", re.IGNORECASE)
+
+# Bearer token (Authorization header style)
+_RE_BEARER = re.compile(r"\bBearer\s+[A-Za-z0-9._\-]{20,}\b")
+
+# JWT: eyJ...eyJ....  (header.payload.signature pattern)
+_RE_JWT = re.compile(r"\beyJ[\w-]+\.eyJ[\w-]+\.[\w-]+\b")
+
+# .env-style assignment with keyword gate: KEY=value (8+ char value)
+# Matches KEY=VALUE assignments anywhere on a line — after 'export ', after
+# leading whitespace, or at column 0. The keyword gate requires one of the
+# sensitive keywords inside the variable name. No start-of-line anchor so
+# "export API_KEY=value" and " SECRET_TOKEN=value" are caught.
+# Keywords: KEY|TOKEN|SECRET|PASSWORD|PASSWD|PWD|API|CREDENTIAL|PRIVATE
+_RE_DOTENV = re.compile(
+    r"(?<![A-Za-z0-9_])(?:[A-Z][A-Z0-9_]*)?(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|PWD|API|CREDENTIAL|PRIVATE)"
+    r"[A-Z0-9_]*\s*=\s*\S{8,}",
+    re.IGNORECASE,
+)
+
+# Generic high-entropy hex string: 32+ hex chars
+_RE_HEX32 = re.compile(r"\b[0-9a-fA-F]{32,}\b")
+
+# Generic high-entropy base64 string: 40+ base64 chars
+_RE_B64_40 = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}")
+
+# Entropy threshold (bits per character)
+_ENTROPY_THRESHOLD = 3.5
+
+
+def _shannon_entropy(s: str) -> float:
+    """Compute Shannon entropy of a string in bits per character."""
+    if not s:
+        return 0.0
+    freq: dict[str, int] = {}
+    for c in s:
+        freq[c] = freq.get(c, 0) + 1
+    n = len(s)
+    entropy = 0.0
+    for count in freq.values():
+        p = count / n
+        entropy -= p * math.log2(p)
+    return entropy
+
+
+def _has_high_entropy_hex(content: str) -> bool:
+    """Return True if content contains a 32+ char hex string with entropy > threshold."""
+    for m in _RE_HEX32.finditer(content):
+        if _shannon_entropy(m.group()) > _ENTROPY_THRESHOLD:
+            return True
+    return False
+
+
+def _has_high_entropy_b64(content: str) -> bool:
+    """Return True if content contains a 40+ char base64 string with entropy > threshold."""
+    for m in _RE_B64_40.finditer(content):
+        token = m.group().rstrip("=")
+        if len(token) >= 40 and _shannon_entropy(token) > _ENTROPY_THRESHOLD:
+            return True
+    return False
+
+
+def looks_like_secret(content: str) -> bool:
+    """Return True if content looks like it contains a credential/secret.
+
+    Conservative: false positives (over-skip) are acceptable.
+    Never logs or echoes the content — caller must NOT log content on True.
+
+    Detectors (in order):
+      1. PEM private key block
+      2. OpenAI API key (sk-...)
+      3. AWS AKIA access key ID
+      4. AWS secret access key assignment
+      5. Bearer token
+      6. JWT (eyJ...eyJ...signature)
+      7. .env-style KEY=value with secret keyword
+      8. High-entropy hex32+ string
+      9. High-entropy base64-40+ string
+    """
+    if not content:
+        return False
+
+    if _RE_PEM.search(content):
+        return True
+
+    if _RE_OPENAI.search(content):
+        return True
+
+    if _RE_AWS_KEY.search(content):
+        return True
+
+    if _RE_AWS_SECRET.search(content):
+        return True
+
+    if _RE_BEARER.search(content):
+        return True
+
+    if _RE_JWT.search(content):
+        return True
+
+    if _RE_DOTENV.search(content):
+        return True
+
+    if _has_high_entropy_hex(content):
+        return True
+
+    if _has_high_entropy_b64(content):
+        return True
+
+    return False