memoryledger 0.1.0__py3-none-any.whl

memoryledger/errors.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class MemoryledgerError(Exception):
+    code: str
+    message: str
+    details: dict[str, object] | None = None
+
+    def __str__(self) -> str:
+        return f"{self.code}: {self.message}"
+
+
+def require(condition: bool, code: str, message: str) -> None:
+    if not condition:
+        raise MemoryledgerError(code, message)

memoryledger/evidence_scan.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+
+from .models import EvidenceRef
+from .storage import Store
+
+
+@dataclass(frozen=True)
+class ScanProposal:
+    scanner: str
+    path: str
+    fact_key: str
+    title: str
+    kind: str
+    observed: str
+    suggested: str
+    line: int
+    source_hash: str
+    render_target: str = "root_agents"
+
+    @property
+    def origin(self) -> str:
+        return f"scan:{self.scanner}:{self.path}:{self.fact_key}"
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "origin": self.origin,
+            "path": self.path,
+            "fact_key": self.fact_key,
+            "title": self.title,
+            "kind": self.kind,
+            "observed": self.observed,
+            "suggested": self.suggested,
+            "line": self.line,
+            "source_hash": self.source_hash,
+            "render_target": self.render_target,
+        }
+
+
+def scan(root: Path) -> list[ScanProposal]:
+    proposals: list[ScanProposal] = []
+    pyproject = root / "pyproject.toml"
+    if pyproject.exists():
+        lines = pyproject.read_text().splitlines()
+        for number, line in enumerate(lines, 1):
+            stripped = line.strip()
+            if stripped.startswith("requires-python"):
+                observed = stripped
+                proposals.append(
+                    _proposal(
+                        "pyproject",
+                        "pyproject.toml",
+                        "requires-python",
+                        "Python requirement",
+                        "learning",
+                        observed,
+                        f"Project Python requirement is `{observed.split('=', 1)[-1].strip()}`.",
+                        number,
+                    )
+                )
+                break
+        section_headers = {
+            "[tool.mypy]": (
+                "tool.mypy",
+                "Python type checking",
+                "procedure",
+                "Run `mypy memoryledger` when changing typed public APIs or core logic.",
+                "linked_doc",
+            ),
+            "[tool.ruff]": (
+                "tool.ruff",
+                "Ruff validation",
+                "procedure",
+                "Run `ruff check .` when Python code or lint configuration changes.",
+                "linked_doc",
+            ),
+            "[tool.ruff.lint]": (
+                "tool.ruff.lint",
+                "Ruff validation",
+                "procedure",
+                "Run `ruff check .` when Python code or lint configuration changes.",
+                "linked_doc",
+            ),
+            "[tool.pytest.ini_options]": (
+                "tool.pytest.ini_options",
+                "Pytest validation",
+                "procedure",
+                "Run `pytest -q` and any focused pytest selection that covers the changed behavior.",
+                "linked_doc",
+            ),
+        }
+        for number, line in enumerate(lines, 1):
+            stripped = line.strip()
+            if stripped not in section_headers:
+                continue
+            key, title, kind, suggested, render_target = section_headers[stripped]
+            proposals.append(
+                _proposal(
+                    "pyproject",
+                    "pyproject.toml",
+                    key,
+                    title,
+                    kind,
+                    stripped,
+                    suggested,
+                    number,
+                    render_target=render_target,
+                )
+            )
+    for name in (".pre-commit-config.yaml", ".pre-commit-config.yml"):
+        pre_commit = root / name
+        if not pre_commit.exists():
+            continue
+        line = 1
+        for number, raw in enumerate(pre_commit.read_text().splitlines(), 1):
+            if raw.strip():
+                line = number
+                break
+        proposals.append(
+            _proposal(
+                "pre-commit",
+                name,
+                "pre-commit-hooks",
+                "Pre-commit validation",
+                "procedure",
+                "configured hooks",
+                "Run `pre-commit run --all-files` when changes affect repository-wide formatting, linting, or configured hooks.",
+                line,
+                render_target="linked_doc",
+            )
+        )
+        break
+    return sorted(proposals, key=lambda item: item.origin)
+
+
+def _proposal(
+    scanner: str,
+    path: str,
+    key: str,
+    title: str,
+    kind: str,
+    observed: str,
+    suggested: str,
+    line: int,
+    *,
+    render_target: str = "root_agents",
+) -> ScanProposal:
+    digest = hashlib.sha256(observed.encode()).hexdigest()
+    return ScanProposal(
+        scanner,
+        path,
+        key,
+        title,
+        kind,
+        observed,
+        suggested,
+        line,
+        digest,
+        render_target,
+    )
+
+
+def apply(store: Store, proposals: list[ScanProposal]) -> list[dict[str, str]]:
+    results: list[dict[str, str]] = []
+    by_origin = {memory.origin: memory for memory in store.all_memories()}
+    for proposal in proposals:
+        old = by_origin.get(proposal.origin)
+        if old and old.origin_hash == proposal.source_hash:
+            results.append({"action": "unchanged", "memory_id": old.id})
+            continue
+        evidence = EvidenceRef(
+            kind="file",
+            title="Observed configuration",
+            uri=proposal.path,
+            excerpt=proposal.observed,
+            line_start=proposal.line,
+            line_end=proposal.line,
+        )
+        if old:
+            from dataclasses import replace
+
+            from ledgercore.time import utc_now_iso
+
+            updated = replace(
+                old,
+                kind=proposal.kind,
+                title=proposal.title,
+                status="candidate",
+                scope_path=proposal.path,
+                render_target=proposal.render_target,
+                origin_hash=proposal.source_hash,
+                evidence_refs=[evidence],
+                updated_at=utc_now_iso(),
+                version=old.version + 1,
+            )
+            store.write(
+                updated, proposal.suggested, "Scanner fact changed.", "scan update"
+            )
+            results.append({"action": "updated", "memory_id": old.id})
+        else:
+            memory = store.create(
+                proposal.kind,
+                proposal.title,
+                proposal.suggested,
+                "Detected from repository configuration.",
+                "repo",
+                proposal.path,
+                proposal.render_target,
+                "scan",
+                origin=proposal.origin,
+                origin_hash=proposal.source_hash,
+                evidence_refs=[evidence],
+            )
+            results.append({"action": "created", "memory_id": memory.id})
+    return results

memoryledger/guardrails.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from .errors import MemoryledgerError
+from .models import (
+    EVIDENCE_KINDS,
+    GENERATED_MARKER,
+    KINDS,
+    RENDER_TARGETS,
+    SCOPES,
+    STATUSES,
+    EvidenceRef,
+    Memory,
+)
+
+SECRET_RE = re.compile(
+    r"(?i)(api[_-]?key|secret|password|token)\s*[=:]\s*['\"]?[A-Za-z0-9_./+-]{12,}"
+)
+PLACEHOLDER_RE = re.compile(r"\b(TBD|TODO:|<fill>)\b")
+
+
+def confined_path(
+    root: Path,
+    value: str | Path,
+    *,
+    code: str = "INVALID_PATH",
+    label: str = "path",
+    must_exist: bool = False,
+) -> Path:
+    raw = Path(value)
+    if raw.is_absolute() or ".." in raw.parts:
+        raise MemoryledgerError(code, f"{label} must be relative to its owning root")
+    target = (root.resolve() / raw).resolve()
+    try:
+        target.relative_to(root.resolve())
+    except ValueError as exc:
+        raise MemoryledgerError(
+            code, f"{label} must not escape its owning root"
+        ) from exc
+    if must_exist and not target.exists():
+        raise MemoryledgerError(code, f"{label} does not exist: {value}")
+    return target
+
+
+def validate_scope_path(root: Path, scope_path: str) -> None:
+    if scope_path:
+        confined_path(root, scope_path, code="INVALID_SCOPE_PATH", label="scope_path")
+
+
+def validate_content(content: str) -> None:
+    if not content.strip():
+        raise MemoryledgerError("EMPTY_CONTENT", "content.md must be non-empty")
+    if SECRET_RE.search(content):
+        raise MemoryledgerError(
+            "SECRET_LIKE_CONTENT", "content contains secret-like text"
+        )
+    if PLACEHOLDER_RE.search(content):
+        raise MemoryledgerError(
+            "PLACEHOLDER_CONTENT", "content contains unresolved placeholders"
+        )
+    if len(content) > 200_000 or content.lower().count("assistant") > 100:
+        raise MemoryledgerError(
+            "TRANSCRIPT_LIKE_CONTENT", "content looks like a huge pasted transcript"
+        )
+
+
+def validate_evidence_ref(root: Path, evidence: EvidenceRef) -> None:
+    if evidence.kind not in EVIDENCE_KINDS:
+        raise MemoryledgerError("INVALID_EVIDENCE_KIND", evidence.kind)
+    if not evidence.title.strip() or not evidence.uri.strip():
+        raise MemoryledgerError(
+            "INVALID_EVIDENCE", "evidence title and URI are required"
+        )
+    if len(evidence.excerpt) > 2000:
+        raise MemoryledgerError("EVIDENCE_TOO_LARGE", "evidence excerpt is too large")
+    if SECRET_RE.search(evidence.excerpt):
+        raise MemoryledgerError(
+            "SECRET_LIKE_EVIDENCE", "evidence contains secret-like text"
+        )
+    if evidence.line_start is not None and evidence.line_start < 1:
+        raise MemoryledgerError("INVALID_LINE_RANGE", "line_start must be positive")
+    if evidence.line_end is not None and (
+        evidence.line_start is None or evidence.line_end < evidence.line_start
+    ):
+        raise MemoryledgerError("INVALID_LINE_RANGE", "invalid evidence line range")
+    if evidence.kind in {"file", "config"}:
+        confined_path(
+            root, evidence.uri, code="INVALID_EVIDENCE_URI", label="evidence URI"
+        )
+    if evidence.kind == "run" and not re.fullmatch(
+        r"run:[A-Za-z0-9._-]+#[A-Za-z0-9._-]+", evidence.uri
+    ):
+        raise MemoryledgerError("INVALID_EVIDENCE_URI", "invalid internal run URI")
+
+
+def validate_memory(
+    memory: Memory, content: str, evidence: str = "", root: Path | None = None
+) -> None:
+    if memory.kind not in KINDS:
+        raise MemoryledgerError("INVALID_KIND", f"Invalid kind: {memory.kind}")
+    if memory.status not in STATUSES:
+        raise MemoryledgerError("INVALID_STATUS", f"Invalid status: {memory.status}")
+    if memory.scope not in SCOPES:
+        raise MemoryledgerError("INVALID_SCOPE", f"Invalid scope: {memory.scope}")
+    if memory.render_target not in RENDER_TARGETS:
+        raise MemoryledgerError(
+            "INVALID_RENDER_TARGET", f"Invalid render target: {memory.render_target}"
+        )
+    if not memory.title.strip():
+        raise MemoryledgerError("EMPTY_TITLE", "title must be non-empty")
+    if memory.section and (
+        len(memory.section) > 80
+        or "\n" in memory.section
+        or not re.fullmatch(r"[\w .&/+()-]+", memory.section)
+    ):
+        raise MemoryledgerError("INVALID_SECTION", "section name is invalid")
+    validate_content(content)
+    if evidence and SECRET_RE.search(evidence):
+        raise MemoryledgerError(
+            "SECRET_LIKE_EVIDENCE", "evidence contains secret-like text"
+        )
+    for ref in memory.evidence_refs:
+        validate_evidence_ref(root or Path.cwd(), ref)
+    if memory.status == "accepted" and not (evidence.strip() or memory.evidence_refs):
+        raise MemoryledgerError(
+            "MISSING_EVIDENCE", "accepted memory must have evidence or approval reason"
+        )
+
+
+def validate_generated_text(
+    text: str, max_chars: int, require_marker: bool = True
+) -> None:
+    if require_marker and GENERATED_MARKER not in text:
+        raise MemoryledgerError(
+            "MISSING_MARKER", "generated output must contain marker"
+        )
+    if len(text) > max_chars:
+        raise MemoryledgerError(
+            "OUTPUT_TOO_LARGE", "generated output exceeds configured limit"
+        )
+    if "run.html" in text.lower():
+        raise MemoryledgerError(
+            "RAW_RUN_HTML", "generated output must not contain raw run.html"
+        )
+    if SECRET_RE.search(text):
+        raise MemoryledgerError(
+            "SECRET_LIKE_OUTPUT", "generated output contains secret-like text"
+        )
+
+
+def safe_to_replace(path: Path) -> None:
+    if path.exists() and GENERATED_MARKER not in path.read_text():
+        raise MemoryledgerError(
+            "MANUAL_FILE", f"Refusing to overwrite manual file: {path}"
+        )

memoryledger/intake.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from html import unescape
+from pathlib import Path
+
+from ledgercore.atomic import atomic_write_text
+
+from .models import EvidenceRef
+from .run_import import decode_session
+from .storage import Store
+
+
+def import_text(store: Store, text: str, source: str = "import") -> list[str]:
+    body = text.strip()
+    title = next(
+        (line.strip("# ") for line in body.splitlines() if line.strip()),
+        "Imported memory",
+    )[:80]
+    memory = store.create(
+        "learning",
+        title,
+        body,
+        "Imported from text.",
+        "global",
+        "",
+        "root_agents",
+        source,
+    )
+    return [memory.id]
+
+
+def import_run_html(store: Store, path: Path) -> list[str]:
+    raw = path.read_text(errors="replace")
+    proposals = decode_session(raw)
+    if proposals is not None:
+        import_key = hashlib.sha256(raw.encode()).hexdigest()[:16]
+        existing = {memory.origin: memory for memory in store.all_memories()}
+        for proposal in proposals:
+            origin = f"run:{import_key}:{proposal.entry_id}"
+            if origin in existing:
+                continue
+            store.validate_new(
+                proposal.kind,
+                proposal.title,
+                proposal.text,
+                "Imported from allowlisted structured session data.",
+                "global",
+                "",
+                proposal.render_target,
+                "run_html",
+                origin=origin,
+                origin_hash=hashlib.sha256(proposal.text.encode()).hexdigest(),
+                evidence_refs=[
+                    EvidenceRef(
+                        kind="run",
+                        title="Structured run entry",
+                        uri=f"run:{import_key}#{proposal.entry_id}",
+                        timestamp=proposal.timestamp,
+                    )
+                ],
+            )
+        ids: list[str] = []
+        for proposal in proposals:
+            origin = f"run:{import_key}:{proposal.entry_id}"
+            if origin in existing:
+                ids.append(existing[origin].id)
+                continue
+            memory = store.create(
+                proposal.kind,
+                proposal.title,
+                proposal.text,
+                "Imported from allowlisted structured session data.",
+                "global",
+                "",
+                proposal.render_target,
+                "run_html",
+                origin=origin,
+                origin_hash=hashlib.sha256(proposal.text.encode()).hexdigest(),
+                evidence_refs=[
+                    EvidenceRef(
+                        kind="run",
+                        title="Structured run entry",
+                        uri=f"run:{import_key}#{proposal.entry_id}",
+                        timestamp=proposal.timestamp,
+                    )
+                ],
+            )
+            ids.append(memory.id)
+        return ids
+    text = re.sub(r"<[^>]+>", " ", raw)
+    text = unescape(re.sub(r"\s+", " ", text)).strip()
+    excerpt = text[:2000]
+    memory = store.create(
+        "episode",
+        "Imported run memory",
+        excerpt,
+        f"Imported from {path.name}.",
+        "global",
+        "",
+        "linked_doc",
+        "run_html",
+        origin=f"run-legacy:{hashlib.sha256(raw.encode()).hexdigest()}",
+        origin_hash=hashlib.sha256(excerpt.encode()).hexdigest(),
+    )
+    import_id = store.next_import_id()
+    idir = store.config.storage_dir / "imports" / import_id
+    idir.mkdir(parents=True, exist_ok=True)
+    atomic_write_text(idir / "raw_excerpt.md", excerpt + "\n")
+    return [memory.id]

memoryledger/launcher.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from .cli import app
+
+
+def main() -> None:
+    app()

memoryledger/models.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+KINDS = ("rule", "learning", "episode", "procedure", "semantic", "document", "local")
+STATUSES = ("candidate", "accepted", "rejected", "archived")
+SCOPES = ("global", "repo", "directory", "file", "command", "workflow", "local")
+RENDER_TARGETS = ("root_agents", "linked_doc", "nested_agents", "none")
+EVIDENCE_KINDS = (
+    "file",
+    "config",
+    "run",
+    "command",
+    "user_approval",
+    "external",
+)
+GENERATED_MARKER = "<!-- Generated by memoryledger. Do not edit directly. -->"
+
+
+@dataclass(frozen=True)
+class EvidenceRef:
+    kind: str
+    title: str
+    uri: str
+    excerpt: str = ""
+    content_hash: str = ""
+    line_start: int | None = None
+    line_end: int | None = None
+    timestamp: str = ""
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, object]) -> EvidenceRef:
+        return cls(
+            kind=str(data["kind"]),
+            title=str(data["title"]),
+            uri=str(data["uri"]),
+            excerpt=str(data.get("excerpt", "")),
+            content_hash=str(data.get("content_hash", "")),
+            line_start=(
+                int(str(data["line_start"])) if data.get("line_start") else None
+            ),
+            line_end=int(str(data["line_end"])) if data.get("line_end") else None,
+            timestamp=str(data.get("timestamp", "")),
+        )
+
+
+@dataclass(frozen=True)
+class Memory:
+    id: str
+    kind: str
+    title: str
+    status: str
+    priority: int
+    scope: str
+    scope_path: str
+    render_target: str
+    source: str
+    created_at: str
+    updated_at: str
+    version: int
+    tags: list[str] = field(default_factory=list)
+    origin: str = ""
+    origin_hash: str = ""
+    section: str = ""
+    evidence_refs: list[EvidenceRef] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, object]) -> Memory:
+        raw_tags = data.get("tags", [])
+        tags = raw_tags if isinstance(raw_tags, list) else []
+        raw_refs = data.get("evidence_refs", [])
+        refs = raw_refs if isinstance(raw_refs, list) else []
+        return cls(
+            id=str(data["id"]),
+            kind=str(data["kind"]),
+            title=str(data["title"]),
+            status=str(data["status"]),
+            priority=int(str(data.get("priority", 100))),
+            scope=str(data.get("scope", "global")),
+            scope_path=str(data.get("scope_path", "")),
+            render_target=str(data.get("render_target", "root_agents")),
+            source=str(data.get("source", "cli")),
+            created_at=str(data["created_at"]),
+            updated_at=str(data["updated_at"]),
+            version=int(str(data.get("version", 1))),
+            tags=[str(tag) for tag in tags],
+            origin=str(data.get("origin", "")),
+            origin_hash=str(data.get("origin_hash", "")),
+            section=str(data.get("section", "")),
+            evidence_refs=[
+                EvidenceRef.from_dict(ref) for ref in refs if isinstance(ref, dict)
+            ],
+        )
+
+
+@dataclass(frozen=True)
+class RenderConfig:
+    root_agents_path: str = "AGENTS.md"
+    linked_docs_dir: str = "docs/agents"
+    nested_agents_enabled: bool = False
+    linked_docs_enabled: bool = True
+    max_root_agents_chars: int = 12000
+    max_linked_doc_chars: int = 50000
+    include_local: bool = False
+    include_rejected: bool = False
+    include_evidence: bool = False
+    evidence_index_path: str = ""
+    sort_order: list[str] = field(
+        default_factory=lambda: [
+            "rule",
+            "procedure",
+            "semantic",
+            "learning",
+            "episode",
+            "document",
+        ]
+    )
+
+
+@dataclass(frozen=True)
+class Template:
+    id: str
+    version: str
+    kind: str
+    title: str
+    content: str = ""
+    content_file: str = ""
+    scope: str = "global"
+    scope_path: str = ""
+    render_target: str = "root_agents"
+    section: str = ""
+    source_root: Path = Path(".")
+
+
+@dataclass(frozen=True)
+class TemplatePolicy:
+    enabled: bool = False
+    ids: list[str] = field(default_factory=list)
+    auto_accept: bool = False
+
+
+@dataclass(frozen=True)
+class GlobalConfig:
+    path: Path
+    templates: list[Template] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class Config:
+    root: Path
+    config_path: Path
+    ledger_code: str = "ml"
+    ledger_name: str = "memoryledger"
+    project_name: str = "my-project"
+    project_uuid: str = ""
+    memoryledger_dir: str = ".memoryledger"
+    render: RenderConfig = field(default_factory=RenderConfig)
+    allow_run_html: bool = True
+    allow_current_run: bool = True
+    default_review_status: str = "candidate"
+    template_policy: TemplatePolicy = field(default_factory=TemplatePolicy)
+
+    @property
+    def storage_dir(self) -> Path:
+        return self.root / self.memoryledger_dir