memdate 1.0.0__py3-none-any.whl

memdate/__init__.py

@@ -0,0 +1,78 @@
+"""memdate — file-based memory with a hard CAPTURE/DISTILL split.
+
+CAPTURE preserves raw records (append-only, timestamped, never
+interpreted). DISTILL consolidates raw into derived distilled.md files
+(never touching raw). INDEX regenerates the cross-domain connective
+tissue after multi-domain distills.
+
+Front-facing conversion of the ``memdate-v2`` skill (Ryan's
+skill-suite, MIT). The original skill's Ryan-specific domains and Google
+Drive root are configuration here, not defaults: domains come from
+``MemoryConfig`` (neutral defaults shipped), storage is a
+``StorageAdapter`` (local filesystem default, Drive an optional seam).
+"""
+
+from .adapters import (
+    DriveNotConfigured,
+    GoogleDriveAdapter,
+    LocalFilesystemAdapter,
+    StorageAdapter,
+    StorageError,
+    default_adapter,
+)
+from .capture import (
+    CaptureError,
+    CaptureResult,
+    InterpretationError,
+    capture,
+    read_raw,
+    route,
+    scan_interpretation,
+)
+from .config import (
+    DEFAULT_DOMAINS,
+    ConfigError,
+    MemoryConfig,
+)
+from .distill import (
+    DistillError,
+    DistillResult,
+    distill,
+    read_distilled,
+)
+from .drop import SweepItem, ensure_layout, sweep_new
+from .frontmatter import FrontmatterError
+from .index import IndexError, IndexResult, read_index, regenerate_index
+
+__all__ = [
+    "MemoryConfig",
+    "ConfigError",
+    "DEFAULT_DOMAINS",
+    "StorageAdapter",
+    "StorageError",
+    "LocalFilesystemAdapter",
+    "GoogleDriveAdapter",
+    "DriveNotConfigured",
+    "default_adapter",
+    "CaptureError",
+    "CaptureResult",
+    "InterpretationError",
+    "capture",
+    "read_raw",
+    "route",
+    "scan_interpretation",
+    "DistillError",
+    "DistillResult",
+    "distill",
+    "read_distilled",
+    "SweepItem",
+    "ensure_layout",
+    "sweep_new",
+    "FrontmatterError",
+    "IndexError",
+    "IndexResult",
+    "read_index",
+    "regenerate_index",
+]
+
+__version__ = "1.0.0"

memdate/adapters.py

@@ -0,0 +1,174 @@
+"""Storage adapters.
+
+The core library talks only to ``StorageAdapter``. The local filesystem
+is the default. Google Drive — the original skill's primary location —
+is an *optional* seam: ``GoogleDriveAdapter`` documents the interface and
+fails loudly with setup instructions instead of pretending to work.
+
+To wire a real backend, subclass ``StorageAdapter`` and pass it to the
+operations. All paths are relative to the memory root, using "/" as the
+separator.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+from abc import ABC, abstractmethod
+
+
+class StorageError(OSError):
+    """Raised when the storage layer fails."""
+
+
+class DriveNotConfigured(StorageError):
+    """Raised by the Google Drive adapter seam (see below)."""
+
+
+class StorageAdapter(ABC):
+    """Minimal surface the memory operations need. Implement for a backend."""
+
+    @abstractmethod
+    def read(self, path: str) -> str | None:
+        """Return file text, or None if the file does not exist."""
+
+    @abstractmethod
+    def write(self, path: str, text: str) -> None:
+        """Create or overwrite a file."""
+
+    @abstractmethod
+    def append(self, path: str, text: str) -> None:
+        """Append text to a file, creating it if needed."""
+
+    @abstractmethod
+    def exists(self, path: str) -> bool:
+        """True if path exists (file or directory)."""
+
+    @abstractmethod
+    def makedirs(self, path: str) -> None:
+        """Create a directory and any missing parents."""
+
+    @abstractmethod
+    def move(self, src: str, dst: str) -> None:
+        """Move/rename a file, creating destination parents as needed."""
+
+    @abstractmethod
+    def list_files(self, path: str) -> list[str]:
+        """Names of files directly inside a directory (no recursion)."""
+
+    @abstractmethod
+    def digest(self, path: str) -> str:
+        """SHA-256 hex of a file's bytes. Used for the append-only check."""
+
+
+class LocalFilesystemAdapter(StorageAdapter):
+    """Default adapter. A plain directory on the local filesystem."""
+
+    def __init__(self, root: str) -> None:
+        self.root = os.path.abspath(os.path.expanduser(root))
+
+    def _full(self, path: str) -> str:
+        full = os.path.abspath(os.path.join(self.root, *path.split("/")))
+        if full != self.root and not full.startswith(self.root + os.sep):
+            raise StorageError(f"Path escapes the memory root: {path!r}.")
+        return full
+
+    def read(self, path: str) -> str | None:
+        full = self._full(path)
+        if not os.path.isfile(full):
+            return None
+        with open(full, encoding="utf-8") as f:
+            return f.read()
+
+    def write(self, path: str, text: str) -> None:
+        full = self._full(path)
+        os.makedirs(os.path.dirname(full), exist_ok=True)
+        with open(full, "w", encoding="utf-8") as f:
+            f.write(text)
+
+    def append(self, path: str, text: str) -> None:
+        full = self._full(path)
+        os.makedirs(os.path.dirname(full), exist_ok=True)
+        with open(full, "a", encoding="utf-8") as f:
+            f.write(text)
+
+    def exists(self, path: str) -> bool:
+        return os.path.exists(self._full(path))
+
+    def makedirs(self, path: str) -> None:
+        os.makedirs(self._full(path), exist_ok=True)
+
+    def move(self, src: str, dst: str) -> None:
+        src_full, dst_full = self._full(src), self._full(dst)
+        os.makedirs(os.path.dirname(dst_full), exist_ok=True)
+        shutil.move(src_full, dst_full)
+
+    def list_files(self, path: str) -> list[str]:
+        full = self._full(path)
+        if not os.path.isdir(full):
+            return []
+        return sorted(
+            name for name in os.listdir(full)
+            if os.path.isfile(os.path.join(full, name))
+        )
+
+    def digest(self, path: str) -> str:
+        full = self._full(path)
+        h = hashlib.sha256()
+        with open(full, "rb") as f:
+            for chunk in iter(lambda: f.read(65536), b""):
+                h.update(chunk)
+        return h.hexdigest()
+
+
+class GoogleDriveAdapter(StorageAdapter):
+    """Optional seam for a Google Drive backend. Not a real implementation.
+
+    The original skill stored everything in a Drive folder; this package
+    defaults to the local filesystem instead. If you want Drive, subclass
+    ``StorageAdapter`` and implement the seven methods — for example by
+    shelling out to the ``gws`` CLI or using the Drive API with your own
+    credentials. Instantiating this class and calling any method raises
+    ``DriveNotConfigured`` with a pointer, so a half-wired Drive setup
+    fails loudly instead of silently writing nowhere.
+    """
+
+    _MSG = (
+        "GoogleDriveAdapter is a seam, not an implementation: it has no "
+        "credentials and performs no I/O. To use Google Drive as the "
+        "memory root, subclass memdate.StorageAdapter and implement "
+        "read/write/append/exists/makedirs/move/list_files/digest against "
+        "the Drive API (or the gws CLI), then pass your adapter to the "
+        "capture/distill/index functions. Until then, LocalFilesystemAdapter "
+        "is the default and the only working backend."
+    )
+
+    def read(self, path: str) -> str | None:
+        raise DriveNotConfigured(self._MSG)
+
+    def write(self, path: str, text: str) -> None:
+        raise DriveNotConfigured(self._MSG)
+
+    def append(self, path: str, text: str) -> None:
+        raise DriveNotConfigured(self._MSG)
+
+    def exists(self, path: str) -> bool:
+        raise DriveNotConfigured(self._MSG)
+
+    def makedirs(self, path: str) -> None:
+        raise DriveNotConfigured(self._MSG)
+
+    def move(self, src: str, dst: str) -> None:
+        raise DriveNotConfigured(self._MSG)
+
+    def list_files(self, path: str) -> list[str]:
+        raise DriveNotConfigured(self._MSG)
+
+    def digest(self, path: str) -> str:
+        raise DriveNotConfigured(self._MSG)
+
+
+def default_adapter(root: str) -> LocalFilesystemAdapter:
+    """The out-of-the-box backend: a local directory."""
+    return LocalFilesystemAdapter(root)

memdate/capture.py

@@ -0,0 +1,136 @@
+"""CAPTURE: append-only, timestamped raw records. Never interprets.
+
+The isolation rule is enforced two ways:
+1. Structural: ``capture()`` only ever appends. It has no code path that
+   modifies, reorders, or deletes prior entries.
+2. Heuristic: the entry text is scanned for interpretation markers
+   (configurable; see ``config.MemoryConfig.interpretation_markers``).
+   A hit raises ``InterpretationError`` unless the caller passes
+   ``force=True``, in which case the bypass is recorded in the entry
+   header — the exception is visible, never silent.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from .adapters import StorageAdapter, default_adapter
+from .config import MemoryConfig
+
+
+class CaptureError(ValueError):
+    """Base for capture failures."""
+
+
+class InterpretationError(CaptureError):
+    """The entry looks like synthesis, not capture. Move it to DISTILL."""
+
+
+ENTRY_HEADER_PREFIX = "## "
+
+
+def scan_interpretation(text: str, markers: tuple[str, ...]) -> list[str]:
+    """Return the interpretation markers found in text (case-insensitive)."""
+    lowered = text.lower()
+    return [m for m in markers if m.lower() in lowered]
+
+
+def stamp(now: datetime | None = None) -> str:
+    dt = now or datetime.now().astimezone()
+    return dt.strftime("%Y-%m-%d %H:%M %Z").strip()
+
+
+@dataclass
+class CaptureResult:
+    domain: str
+    path: str
+    timestamp: str
+    bytes_written: int
+    bypassed_check: bool
+
+
+def capture(
+    config: MemoryConfig,
+    domain: str,
+    text: str,
+    *,
+    source: str = "",
+    now: datetime | None = None,
+    force: bool = False,
+    adapter: StorageAdapter | None = None,
+) -> CaptureResult:
+    """Append one raw record to ``<domain>/raw.md``.
+
+    Raises ``InterpretationError`` if the text trips an interpretation
+    marker (unless ``force=True``), ``CaptureError`` on empty text or an
+    unknown domain. Never touches any other file.
+    """
+    name = config.require_domain(domain)
+    body = text.strip()
+    if not body:
+        raise CaptureError("Capture text must not be empty.")
+
+    hits = scan_interpretation(body, config.interpretation_markers)
+    if hits and not force:
+        raise InterpretationError(
+            f"Entry looks like interpretation, not capture — markers found: "
+            f"{', '.join(hits)}. Rewrite as raw record, or pass force=True "
+            f"to record the bypass explicitly."
+        )
+
+    ts = stamp(now)
+    header = f"{ENTRY_HEADER_PREFIX}{ts}"
+    if source.strip():
+        header += f" — source: {source.strip()}"
+    if hits:
+        header += " — [interpretation-check bypassed: " + ", ".join(hits) + "]"
+    entry = f"{header}\n\n{body}\n\n"
+
+    store = adapter or default_adapter(config.root)
+    path = f"{name}/raw.md"
+    existed = store.exists(path)
+    store.append(path, ("\n" if existed and not _ends_blank(store, path) else "") + entry)
+    return CaptureResult(
+        domain=name,
+        path=path,
+        timestamp=ts,
+        bytes_written=len(entry.encode("utf-8")),
+        bypassed_check=bool(hits),
+    )
+
+
+def _ends_blank(store: StorageAdapter, path: str) -> bool:
+    text = store.read(path) or ""
+    return text.endswith("\n\n") or not text
+
+
+def read_raw(
+    config: MemoryConfig,
+    domain: str,
+    *,
+    adapter: StorageAdapter | None = None,
+) -> str | None:
+    """Return a domain's raw.md, or None if it has no captures yet."""
+    name = config.require_domain(domain)
+    store = adapter or default_adapter(config.root)
+    return store.read(f"{name}/raw.md")
+
+
+def route(kind: str) -> str:
+    """The decision rule as a dispatch: recording→capture, consolidating→distill.
+
+    ``kind`` must be exactly "capture" or "distill". Anything else —
+    including "both" — raises, because the skill forbids performing both
+    implicitly in one operation.
+    """
+    normalized = kind.strip().lower()
+    if normalized == "capture":
+        return "capture"
+    if normalized == "distill":
+        return "distill"
+    raise CaptureError(
+        f"Unknown operation {kind!r}: use 'capture' for recording new "
+        f"information, 'distill' for consolidating existing information. "
+        f"Never both in one operation."
+    )

memdate/config.py

@@ -0,0 +1,148 @@
+"""Configuration: memory root, domain list, and capture discipline.
+
+The domain list is the main decoupling point of the whole package.
+Ryan's original skill shipped with his domains (personal, fhk, tribunal,
+memory-system, misc). This package ships with neutral defaults and lets
+the caller define their own — via constructor, JSON, or a small YAML
+subset (top-level keys with scalar or list values only).
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+# Neutral defaults. Nothing Ryan-specific ships here.
+DEFAULT_DOMAINS = ("personal", "work", "projects", "reference", "misc")
+
+# Heuristic markers of interpretation inside a capture entry.
+# CAPTURE preserves; these phrases suggest the entry is synthesizing.
+DEFAULT_INTERPRETATION_MARKERS = (
+    "in summary",
+    "in conclusion",
+    "to summarize",
+    "overall,",
+    "the takeaway",
+    "the bottom line",
+    "what this means is",
+    "in other words,",
+    "therefore we should",
+    "we should therefore",
+)
+
+DROP_SUBDIRS = ("NEW", "PROCESSED", "PENDING", "QUARANTINE")
+
+
+class ConfigError(ValueError):
+    """Raised for invalid configuration."""
+
+
+@dataclass
+class MemoryConfig:
+    """Everything the library needs to find and govern a memory store."""
+
+    root: str
+    domains: tuple[str, ...] = DEFAULT_DOMAINS
+    interpretation_markers: tuple[str, ...] = DEFAULT_INTERPRETATION_MARKERS
+    drop_subdirs: tuple[str, ...] = DROP_SUBDIRS
+
+    def __post_init__(self) -> None:
+        if not self.root:
+            raise ConfigError("Memory root must not be empty.")
+        if not self.domains:
+            raise ConfigError("Domain list must not be empty.")
+        seen: set[str] = set()
+        for d in self.domains:
+            name = d.strip()
+            if not name:
+                raise ConfigError("Domain names must not be blank.")
+            if "/" in name or "\\" in name or name in (".", ".."):
+                raise ConfigError(f"Invalid domain name: {d!r}.")
+            if name.lower() in seen:
+                raise ConfigError(f"Duplicate domain: {name!r}.")
+            seen.add(name.lower())
+        self.domains = tuple(d.strip() for d in self.domains)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "MemoryConfig":
+        root = data.get("root")
+        if not root:
+            raise ConfigError("Config needs a 'root' key.")
+        return cls(
+            root=root,
+            domains=tuple(data.get("domains", DEFAULT_DOMAINS)),
+            interpretation_markers=tuple(
+                data.get("interpretation_markers", DEFAULT_INTERPRETATION_MARKERS)
+            ),
+        )
+
+    @classmethod
+    def from_json(cls, path: str) -> "MemoryConfig":
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if not isinstance(data, dict):
+            raise ConfigError(f"{path}: top-level JSON value must be an object.")
+        return cls.from_dict(data)
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "MemoryConfig":
+        """Parse a strict, tiny YAML subset.
+
+        Supported: top-level ``key: value`` pairs and ``key:`` followed by
+        ``- item`` list lines. Comments (#) and blank lines ignored.
+        Anything else raises ConfigError. This is deliberately not a
+        general YAML parser — stdlib has none, and a memory config
+        doesn't need one.
+        """
+        data: dict[str, object] = {}
+        current_key: str | None = None
+        with open(path, encoding="utf-8") as f:
+            for lineno, raw in enumerate(f, 1):
+                line = raw.split("#", 1)[0].rstrip()
+                if not line.strip():
+                    continue
+                if line != line.lstrip():
+                    stripped = line.strip()
+                    if stripped.startswith("- ") and current_key is not None:
+                        item = stripped[2:].strip().strip("'\"")
+                        lst = data.get(current_key)
+                        if not isinstance(lst, list):
+                            raise ConfigError(
+                                f"{path}:{lineno}: list item without a list key."
+                            )
+                        lst.append(item)
+                        continue
+                    raise ConfigError(
+                        f"{path}:{lineno}: unsupported indented line: {raw.rstrip()!r}."
+                    )
+                if ":" not in line:
+                    raise ConfigError(
+                        f"{path}:{lineno}: expected 'key: value': {raw.rstrip()!r}."
+                    )
+                key, _, value = line.partition(":")
+                key = key.strip()
+                value = value.strip().strip("'\"")
+                if not key:
+                    raise ConfigError(f"{path}:{lineno}: blank key.")
+                if value:
+                    data[key] = value
+                    current_key = None
+                else:
+                    data[key] = []
+                    current_key = key
+        return cls.from_dict(data)
+
+    def domain_dir(self, domain: str) -> str:
+        """Relative path of a domain's directory inside the root."""
+        self.require_domain(domain)
+        return domain
+
+    def require_domain(self, domain: str) -> str:
+        name = domain.strip()
+        known = {d.lower(): d for d in self.domains}
+        if name.lower() not in known:
+            raise ConfigError(
+                f"Unknown domain {domain!r}. Known domains: "
+                f"{', '.join(self.domains)}."
+            )
+        return known[name.lower()]

memdate/distill.py

@@ -0,0 +1,142 @@
+"""DISTILL: consolidate raw into derived distilled.md. Never touches raw.
+
+The caller (human or model) supplies the consolidated body; the library
+writes it with provenance frontmatter and enforces the isolation rule by
+checksumming raw.md before and after the write. If raw changed in
+between, the run fails — DISTILL never alters raw, and it refuses to
+certify a run where something else did.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import date
+
+from . import frontmatter as fm
+from .adapters import StorageAdapter, default_adapter
+from .config import MemoryConfig
+
+ARTIFACT = "memdate-distilled"
+PROTOCOL_VERSION = "memdate/1.0.0"
+
+LIFECYCLES = ("INITIAL", "ITERATIVE", "FINAL")
+# FINAL only if the domain's raw record is fully consolidated with no open threads.
+
+
+class DistillError(ValueError):
+    """Raised when a distill run breaks the isolation rules."""
+
+
+@dataclass
+class DistillResult:
+    domain: str
+    path: str
+    lifecycle: str
+    raw_untouched: bool
+    entities: tuple[str, ...] = ()
+    open_questions: tuple[str, ...] = ()
+
+
+def distill(
+    config: MemoryConfig,
+    domain: str,
+    body: str,
+    *,
+    entities: list[str] | None = None,
+    entity_notes: dict[str, str] | None = None,
+    informs: list[str] | None = None,
+    open_questions: list[str] | None = None,
+    lifecycle: str = "ITERATIVE",
+    today: str | None = None,
+    adapter: StorageAdapter | None = None,
+) -> DistillResult:
+    """Write ``<domain>/distilled.md`` with provenance frontmatter.
+
+    - ``body`` is the consolidated synthesis (supplied by the caller).
+    - ``entities`` names people/places/concepts this distillation
+      references; the cross-domain index uses them. ``entity_notes`` maps
+      an entity to a one-line status note.
+    - ``informs`` lists other domains this domain's material informs
+      (dependency direction).
+    - ``open_questions`` are contradictions/unknowns visible in this domain.
+    - Raises ``DistillError`` if raw.md is missing (nothing to distill),
+      if lifecycle is invalid, or if raw.md's bytes changed during the run.
+    """
+    name = config.require_domain(domain)
+    text = body.strip()
+    if not text:
+        raise DistillError("Distilled body must not be empty.")
+    cycle = lifecycle.strip().upper()
+    if cycle not in LIFECYCLES:
+        raise DistillError(
+            f"Invalid lifecycle {lifecycle!r}. Use one of: {', '.join(LIFECYCLES)}."
+        )
+    if informs:
+        for other in informs:
+            config.require_domain(other)
+
+    store = adapter or default_adapter(config.root)
+    raw_path = f"{name}/raw.md"
+    raw_text = store.read(raw_path)
+    if raw_text is None:
+        raise DistillError(
+            f"Cannot distill domain {name!r}: no raw.md exists. "
+            f"Capture something first."
+        )
+    raw_before = store.digest(raw_path)
+
+    fields: dict[str, object] = {
+        "artifact": ARTIFACT,
+        "domain": name,
+        "date": today or date.today().isoformat(),
+        "protocol": PROTOCOL_VERSION,
+        "lifecycle": cycle,
+        "entities": [e.strip() for e in (entities or []) if e.strip()],
+        "informs": [config.require_domain(d) for d in (informs or [])],
+        "open_questions": [q.strip() for q in (open_questions or []) if q.strip()],
+    }
+    if entity_notes:
+        fields["entity_notes"] = {k.strip(): v.strip() for k, v in entity_notes.items()}
+
+    document = fm.render(fields) + text.rstrip() + "\n"
+    distilled_path = f"{name}/distilled.md"
+    store.write(distilled_path, document)
+
+    raw_after = store.digest(raw_path)
+    if raw_after != raw_before:
+        raise DistillError(
+            f"Isolation violation: {raw_path} changed during the distill run. "
+            f"DISTILL never alters raw — the run is void; inspect what wrote to raw."
+        )
+
+    return DistillResult(
+        domain=name,
+        path=distilled_path,
+        lifecycle=cycle,
+        raw_untouched=True,
+        entities=tuple(fields["entities"]),  # type: ignore[arg-type]
+        open_questions=tuple(fields["open_questions"]),  # type: ignore[arg-type]
+    )
+
+
+def read_distilled(
+    config: MemoryConfig,
+    domain: str,
+    *,
+    adapter: StorageAdapter | None = None,
+) -> tuple[dict[str, object], str] | None:
+    """Return ``(frontmatter, body)`` for a domain's distilled.md, or None."""
+    name = config.require_domain(domain)
+    store = adapter or default_adapter(config.root)
+    text = store.read(f"{name}/distilled.md")
+    if text is None:
+        return None
+    fields, body = fm.split(text)
+    fm.require(fields, "artifact", "domain", "date", "protocol", "lifecycle",
+               what=f"distilled.md for {name!r}")
+    if fields.get("artifact") != ARTIFACT:
+        raise DistillError(
+            f"distilled.md for {name!r} has artifact {fields.get('artifact')!r}, "
+            f"expected {ARTIFACT!r}."
+        )
+    return fields, body