memorymaster 3.16.0__tar.gz → 3.17.1__tar.gz

{memorymaster-3.16.0/memorymaster.egg-info → memorymaster-3.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memorymaster
-Version: 3.16.0
+Version: 3.17.1
 Summary: Production-grade memory reliability system for AI coding agents. Lifecycle-managed claims with citations, conflict detection, steward governance, and MCP integration.
 Author: wolverin0
 License: MIT

{memorymaster-3.16.0 → memorymaster-3.17.1}/memorymaster/cli.py

@@ -23,6 +23,7 @@ from memorymaster.cli_handlers_curation import COMMAND_HANDLERS
 from memorymaster.cli_handlers_basic import (
     _handle_decay,
     _handle_entity_graph_export,
+    _handle_ingest_daydream,
     _handle_recompute_confidence_priors,
     _handle_wiki_suggest_links,
     handle_mcp_usage_report,
@@ -33,6 +34,7 @@ COMMAND_HANDLERS["entity-graph-export"] = _handle_entity_graph_export
 COMMAND_HANDLERS["recompute-confidence-priors"] = _handle_recompute_confidence_priors
 COMMAND_HANDLERS["wiki-suggest-links"] = _handle_wiki_suggest_links
 COMMAND_HANDLERS["decay"] = _handle_decay
+COMMAND_HANDLERS["ingest-daydream"] = _handle_ingest_daydream
 COMMAND_HANDLERS["mcp-usage-report"] = (
     lambda args, service, parser, effective_db: handle_mcp_usage_report(args, effective_db)
 )
@@ -66,6 +68,12 @@ def build_parser() -> argparse.ArgumentParser:
     ingest.add_argument("--valid-from", default=None, help="ISO-8601 timestamp: start of the claim validity window")
     ingest.add_argument("--valid-until", default=None, help="ISO-8601 timestamp: end of the validity window (omit if still current)")
 
+    ingest_daydream = sub.add_parser("ingest-daydream", help="Ingest daydream insights as candidate hypothesis claims")
+    ingest_daydream.add_argument("insights_dir", help="Daydreams directory containing insight JSON or Markdown files")
+    ingest_daydream.add_argument("--min-score", type=float, default=7.0, help="Minimum average score to ingest")
+    ingest_daydream.add_argument("--scope", default="user", help="Claim scope (default: user)")
+    ingest_daydream.add_argument("--dry-run", action="store_true", help="Preview ingest without writing claims")
+
     import_whatsapp = sub.add_parser("import-whatsapp", help="Import WhatsApp messages from a wacli JSON/JSONL export")
     import_whatsapp.add_argument("--input", required=True, help="Path to wacli JSON or JSONL export")
     import_whatsapp.add_argument("--display-name", default="WhatsApp", help="External source display name")

{memorymaster-3.16.0 → memorymaster-3.17.1}/memorymaster/cli_handlers_basic.py

@@ -324,6 +324,29 @@ def _handle_ingest(args: argparse.Namespace, service, parser: argparse.ArgumentP
     return 0
 
 
+def _handle_ingest_daydream(args: argparse.Namespace, service, parser: argparse.ArgumentParser, effective_db: str) -> int:
+    from memorymaster.jobs.daydream_ingest import ingest_insights
+
+    t0 = time.perf_counter()
+    result = ingest_insights(
+        service,
+        Path(args.insights_dir),
+        min_score=args.min_score,
+        scope=args.scope,
+        dry_run=args.dry_run,
+    )
+    elapsed_ms = (time.perf_counter() - t0) * 1000
+    if args.json_output:
+        print(_json_envelope(result, query_ms=elapsed_ms))
+    else:
+        tag = " [DRY RUN]" if args.dry_run else ""
+        print(
+            f"ingest-daydream{tag}: ingested={result['ingested']} "
+            f"skipped={result['skipped']} errors={len(result['errors'])}"
+        )
+    return 0
+
+
 def _handle_import_whatsapp(args: argparse.Namespace, service, parser: argparse.ArgumentParser, effective_db: str) -> int:
     from memorymaster.atlas_contract import atlas_meta
     from memorymaster.connectors.whatsapp import import_wacli_json

memorymaster-3.17.1/memorymaster/jobs/daydream_ingest.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from memorymaster.models import CitationInput
+from memorymaster.service import MemoryService
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(slots=True)
+class _DaydreamInsight:
+    title: str
+    connection: str
+    synthesis: str
+    implication: str
+    average: float | None
+    date: str
+    references: list[dict[str, str]]
+    path: Path
+
+
+def ingest_insights(
+    service: MemoryService,
+    insights_dir: Path,
+    *,
+    min_score: float = 7.0,
+    scope: str = "user",
+    dry_run: bool = False,
+) -> dict:
+    """Ingest accepted daydream insights as candidate hypothesis claims."""
+    root = Path(insights_dir)
+    result: dict[str, Any] = {"ingested": 0, "skipped": 0, "errors": []}
+    if not root.exists() or not root.is_dir():
+        raise ValueError(f"insights_dir does not exist or is not a directory: {root}")
+
+    for path in _iter_insight_files(root):
+        insight = _load_insight(path)
+        if insight is None:
+            result["skipped"] += 1
+            result["errors"].append(f"{path}: could not parse daydream insight")
+            continue
+
+        if insight.average is None or insight.average < min_score:
+            result["skipped"] += 1
+            continue
+
+        idempotency_key = _idempotency_key(insight)
+        if _claim_exists(service, idempotency_key):
+            result["skipped"] += 1
+            continue
+
+        if dry_run:
+            result["ingested"] += 1
+            continue
+
+        claim = service.ingest(
+            text=insight.synthesis,
+            citations=_citations_for(insight),
+            idempotency_key=idempotency_key,
+            claim_type="hypothesis",
+            subject=insight.title or _subject_from_connection(insight.connection),
+            scope=scope,
+            volatility="medium",
+            confidence=0.5,
+            source_agent="daydream",
+        )
+        if claim.idempotency_key == idempotency_key:
+            result["ingested"] += 1
+        else:
+            result["skipped"] += 1
+
+    return result
+
+
+def _iter_insight_files(root: Path) -> list[Path]:
+    files = [path for path in root.rglob("*.json") if path.is_file()]
+    files.extend(
+        path
+        for path in root.rglob("*.md")
+        if path.is_file() and "digests" not in {part.lower() for part in path.parts}
+    )
+    return sorted(files)
+
+
+def _load_insight(path: Path) -> _DaydreamInsight | None:
+    try:
+        if path.suffix.lower() == ".json":
+            return _load_json_insight(path)
+        if path.suffix.lower() == ".md":
+            return _load_markdown_insight(path)
+    except Exception as exc:
+        logger.warning("Skipping daydream insight %s: %s", path, exc)
+    return None
+
+
+def _load_json_insight(path: Path) -> _DaydreamInsight | None:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, list):
+        logger.warning("Skipping daydream manifest-like JSON %s", path)
+        return None
+    if not isinstance(data, dict):
+        return None
+
+    metadata = _dict_value(data.get("metadata"))
+    scores = _dict_value(metadata.get("scores"))
+    average = _float_value(
+        data.get("avg_score")
+        or data.get("average")
+        or scores.get("average")
+        or scores.get("avg_score")
+    )
+    title = _text_value(data.get("title") or data.get("suggested_title"))
+    connection = _text_value(data.get("connection"))
+    synthesis = _text_value(data.get("synthesis"))
+    implication = _text_value(data.get("implication"))
+    if not synthesis:
+        return None
+
+    references = _references_from_json(data, metadata)
+    date = _normalize_date(
+        data.get("date")
+        or data.get("created_date")
+        or metadata.get("date")
+        or metadata.get("created_date")
+        or _date_from_filename(path)
+    )
+    return _DaydreamInsight(
+        title=title or _subject_from_connection(connection),
+        connection=connection,
+        synthesis=synthesis,
+        implication=implication,
+        average=average,
+        date=date,
+        references=references,
+        path=path,
+    )
+
+
+def _load_markdown_insight(path: Path) -> _DaydreamInsight | None:
+    raw = path.read_text(encoding="utf-8")
+    frontmatter, body = _split_frontmatter(raw)
+    if frontmatter.get("type") != "daydream":
+        return None
+    title = _first_heading(body) or path.stem
+    average = _float_value(frontmatter.get("scores.average"))
+    if average is None:
+        average = _float_value(frontmatter.get("average"))
+    connection = _blockquote_connection(body)
+    synthesis = _section_before(body, "## Implication").strip()
+    if title:
+        synthesis = re.sub(rf"^\s*#\s+{re.escape(title)}\s*", "", synthesis).strip()
+    synthesis = re.sub(r"^>\s*Connection.*$", "", synthesis, flags=re.MULTILINE).strip()
+    implication = _section_between(body, "## Implication", "## Critic")
+    if not synthesis:
+        return None
+    date = _normalize_date(frontmatter.get("created_date") or _date_from_filename(path))
+    return _DaydreamInsight(
+        title=title,
+        connection=connection,
+        synthesis=synthesis,
+        implication=implication,
+        average=average,
+        date=date,
+        references=_references_from_frontmatter(frontmatter),
+        path=path,
+    )
+
+
+def _references_from_json(data: dict[str, Any], metadata: dict[str, Any]) -> list[dict[str, str]]:
+    raw_refs = metadata.get("references") or data.get("references") or []
+    references: list[dict[str, str]] = []
+    if isinstance(raw_refs, list):
+        for item in raw_refs:
+            if isinstance(item, dict):
+                locator = _text_value(item.get("path") or item.get("locator") or item.get("file"))
+                title = _text_value(item.get("title") or item.get("name"))
+            else:
+                locator = _text_value(item)
+                title = ""
+            if locator:
+                references.append({"path": locator, "title": title})
+    for side in ("a", "b"):
+        locator = _text_value(data.get(f"path_{side}"))
+        title = _text_value(data.get(f"title_{side}"))
+        if locator and not any(ref["path"] == locator for ref in references):
+            references.append({"path": locator, "title": title})
+    return references
+
+
+def _references_from_frontmatter(frontmatter: dict[str, str]) -> list[dict[str, str]]:
+    references: list[dict[str, str]] = []
+    for value in frontmatter.get("source_notes", "").splitlines():
+        cleaned = _clean_wikilink(value.strip().lstrip("-").strip().strip("'\""))
+        if cleaned:
+            references.append({"path": cleaned, "title": cleaned})
+    return references
+
+
+def _citations_for(insight: _DaydreamInsight) -> list[CitationInput]:
+    citations = [
+        CitationInput(source="daydream", locator=ref["path"], excerpt=ref.get("title") or None)
+        for ref in insight.references
+        if ref.get("path")
+    ]
+    if citations:
+        return citations
+    return [
+        CitationInput(
+            source="daydream",
+            locator=str(insight.path),
+            excerpt=insight.connection or insight.title,
+        )
+    ]
+
+
+def _claim_exists(service: MemoryService, idempotency_key: str) -> bool:
+    getter = getattr(service.store, "get_claim_by_idempotency_key", None)
+    return bool(getter and getter(idempotency_key, include_citations=False))
+
+
+def _idempotency_key(insight: _DaydreamInsight) -> str:
+    title = re.sub(r"\s+", " ", insight.title.strip().lower())
+    return f"daydream:{insight.date}:{title}"
+
+
+def _subject_from_connection(connection: str) -> str:
+    words = re.findall(r"[A-Za-z0-9_]+", connection)
+    return " ".join(words[:3]) if words else "daydream insight"
+
+
+def _dict_value(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def _text_value(value: Any) -> str:
+    return str(value).strip() if value is not None else ""
+
+
+def _float_value(value: Any) -> float | None:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _normalize_date(value: Any) -> str:
+    text = _clean_wikilink(_text_value(value))
+    match = re.search(r"(\d{4})-?(\d{2})-?(\d{2})", text)
+    if match:
+        return f"{match.group(1)}-{match.group(2)}-{match.group(3)}"
+    return "unknown-date"
+
+
+def _date_from_filename(path: Path) -> str:
+    return _normalize_date(path.stem)
+
+
+def _clean_wikilink(value: str) -> str:
+    value = value.strip()
+    match = re.fullmatch(r"\[\[([^|\]]+)(?:\|[^\]]+)?\]\]", value)
+    return match.group(1).strip() if match else value
+
+
+def _split_frontmatter(raw: str) -> tuple[dict[str, str], str]:
+    if not raw.startswith("---"):
+        return {}, raw
+    parts = raw.split("---", 2)
+    if len(parts) < 3:
+        return {}, raw
+    return _parse_frontmatter(parts[1]), parts[2]
+
+
+def _parse_frontmatter(raw: str) -> dict[str, str]:
+    data: dict[str, str] = {}
+    current_list: str | None = None
+    current_map: str | None = None
+    for line in raw.splitlines():
+        if not line.strip():
+            continue
+        if line.startswith("  -") and current_list:
+            existing = data.get(current_list, "")
+            data[current_list] = f"{existing}\n{line.strip()}".strip()
+            continue
+        if line.startswith("  ") and current_map and ":" in line:
+            key, value = line.strip().split(":", 1)
+            data[f"{current_map}.{key.strip()}"] = value.strip().strip("'\"")
+            continue
+        current_map = None
+        current_list = None
+        if ":" not in line:
+            continue
+        key, value = line.split(":", 1)
+        key = key.strip()
+        value = value.strip().strip("'\"")
+        data[key] = value
+        if not value:
+            current_list = key
+            current_map = key
+    return data
+
+
+def _first_heading(body: str) -> str:
+    match = re.search(r"^#\s+(.+)$", body, flags=re.MULTILINE)
+    return match.group(1).strip() if match else ""
+
+
+def _blockquote_connection(body: str) -> str:
+    match = re.search(r"^>\s*(.+)$", body, flags=re.MULTILINE)
+    return match.group(1).strip() if match else ""
+
+
+def _section_before(body: str, heading: str) -> str:
+    index = body.find(heading)
+    return body if index < 0 else body[:index]
+
+
+def _section_between(body: str, start: str, end_prefix: str) -> str:
+    start_index = body.find(start)
+    if start_index < 0:
+        return ""
+    content_start = start_index + len(start)
+    end_match = re.search(rf"^{re.escape(end_prefix)}.*$", body[content_start:], flags=re.MULTILINE)
+    if not end_match:
+        return body[content_start:].strip()
+    return body[content_start:content_start + end_match.start()].strip()

{memorymaster-3.16.0 → memorymaster-3.17.1}/memorymaster/steward.py

@@ -1625,5 +1625,27 @@ def run_steward(
         },
         "artifact_path": str(Path(artifact_path)),
     }
+    import os
+
+    daydream_dir = os.environ.get("MEMORYMASTER_DAYDREAM_INGEST_DIR")
+    if daydream_dir:
+        try:
+            import logging
+
+            from memorymaster.jobs.daydream_ingest import ingest_insights
+
+            verbose = os.environ.get("MEMORYMASTER_DAYDREAM_VERBOSE") == "1"
+            path = Path(daydream_dir)
+            if path.is_dir():
+                summary = ingest_insights(service, path, dry_run=False)
+                report["daydream"] = summary
+                if verbose:
+                    logging.getLogger(__name__).info("daydream auto-ingest: %s", summary)
+            else:
+                report["daydream"] = {"skipped": "dir-not-found", "path": str(path)}
+        except Exception as exc:
+            report["daydream"] = {"error": str(exc)[:200]}
+            if os.environ.get("MEMORYMASTER_DAYDREAM_VERBOSE") == "1":
+                logging.getLogger(__name__).warning("daydream auto-ingest failed: %s", exc)
     _write_artifact(Path(artifact_path), report)
     return report

{memorymaster-3.16.0 → memorymaster-3.17.1/memorymaster.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memorymaster
-Version: 3.16.0
+Version: 3.17.1
 Summary: Production-grade memory reliability system for AI coding agents. Lifecycle-managed claims with citations, conflict detection, steward governance, and MCP integration.
 Author: wolverin0
 License: MIT

{memorymaster-3.16.0 → memorymaster-3.17.1}/memorymaster.egg-info/SOURCES.txt

@@ -127,6 +127,7 @@ memorymaster/jobs/__init__.py
 memorymaster/jobs/calibration.py
 memorymaster/jobs/compact_summaries.py
 memorymaster/jobs/compactor.py
+memorymaster/jobs/daydream_ingest.py
 memorymaster/jobs/decay.py
 memorymaster/jobs/dedup.py
 memorymaster/jobs/deterministic.py
@@ -236,6 +237,7 @@ tests/test_dashboard_coverage.py
 tests/test_dashboard_latency.py
 tests/test_dashboard_lineage.py
 tests/test_dashboard_review_queue.py
+tests/test_daydream_ingest.py
 tests/test_db_merge_confidence_conflict.py
 tests/test_db_merge_coverage_v2.py
 tests/test_decay_coverage.py
@@ -329,6 +331,7 @@ tests/test_staleness.py
 tests/test_stealth_mode.py
 tests/test_steward.py
 tests/test_steward_classifier.py
+tests/test_steward_daydream_hook.py
 tests/test_steward_features.py
 tests/test_steward_features_v3.py
 tests/test_steward_resolution_parity.py

{memorymaster-3.16.0 → memorymaster-3.17.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "memorymaster"
-version = "3.16.0"
+version = "3.17.1"
 description = "Production-grade memory reliability system for AI coding agents. Lifecycle-managed claims with citations, conflict detection, steward governance, and MCP integration."
 license = {text = "MIT"}
 authors = [{name = "wolverin0"}]

memorymaster-3.17.1/tests/test_daydream_ingest.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from memorymaster.jobs.daydream_ingest import ingest_insights
+from memorymaster.service import MemoryService
+
+
+@pytest.fixture()
+def service(tmp_path: Path) -> MemoryService:
+    svc = MemoryService(str(tmp_path / "memorymaster.db"))
+    svc.init_db()
+    return svc
+
+
+def _write_insight(
+    directory: Path,
+    name: str,
+    *,
+    score: float,
+    title: str,
+    synthesis: str | None = None,
+) -> Path:
+    path = directory / name
+    path.write_text(
+        json.dumps(
+            {
+                "connection": f"{title} connection",
+                "synthesis": synthesis or f"{title} synthesis",
+                "implication": f"{title} implication",
+                "suggested_title": title,
+                "metadata": {
+                    "date": "2026-05-16",
+                    "references": [
+                        {"title": "Source A", "path": "notes/source-a.md"},
+                        {"title": "Source B", "path": "notes/source-b.md"},
+                    ],
+                    "scores": {"average": score},
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def _claims(service: MemoryService):
+    return service.list_claims(status="candidate", limit=20, allow_sensitive=True)
+
+
+def test_ingests_above_threshold(service: MemoryService, tmp_path: Path):
+    _write_insight(tmp_path, "one.json", score=8.0, title="One")
+    _write_insight(tmp_path, "two.json", score=6.0, title="Two")
+    _write_insight(tmp_path, "three.json", score=9.5, title="Three")
+
+    result = ingest_insights(service, tmp_path, min_score=7.0)
+
+    assert result["ingested"] == 2
+    assert result["skipped"] == 1
+    assert len(_claims(service)) == 2
+
+
+def test_dry_run_no_writes(service: MemoryService, tmp_path: Path):
+    _write_insight(tmp_path, "one.json", score=8.0, title="One")
+    _write_insight(tmp_path, "two.json", score=6.0, title="Two")
+    _write_insight(tmp_path, "three.json", score=9.5, title="Three")
+
+    result = ingest_insights(service, tmp_path, min_score=7.0, dry_run=True)
+
+    assert result["ingested"] == 2
+    assert result["skipped"] == 1
+    assert _claims(service) == []
+
+
+def test_idempotent(service: MemoryService, tmp_path: Path):
+    _write_insight(tmp_path, "one.json", score=8.0, title="One")
+    _write_insight(tmp_path, "two.json", score=9.5, title="Two")
+
+    first = ingest_insights(service, tmp_path, min_score=7.0)
+    second = ingest_insights(service, tmp_path, min_score=7.0)
+
+    assert first["ingested"] == 2
+    assert second["ingested"] == 0
+    assert second["skipped"] == 2
+    assert len(_claims(service)) == 2
+
+
+def test_handles_malformed_json(service: MemoryService, tmp_path: Path):
+    _write_insight(tmp_path, "one.json", score=8.0, title="One")
+    (tmp_path / "bad.json").write_text("{not json", encoding="utf-8")
+
+    result = ingest_insights(service, tmp_path, min_score=7.0)
+
+    assert result["ingested"] == 1
+    assert result["skipped"] == 1
+    assert len(result["errors"]) == 1
+    assert "bad.json" in result["errors"][0]
+
+
+def test_uses_correct_claim_type_and_confidence(service: MemoryService, tmp_path: Path):
+    _write_insight(tmp_path, "one.json", score=8.0, title="One")
+
+    ingest_insights(service, tmp_path, min_score=7.0)
+
+    [claim] = _claims(service)
+    assert claim.claim_type == "hypothesis"
+    assert claim.confidence == 0.5
+    assert claim.source_agent == "daydream"
+
+
+def test_citations_link_to_source_notes(service: MemoryService, tmp_path: Path):
+    _write_insight(tmp_path, "one.json", score=8.0, title="One")
+
+    ingest_insights(service, tmp_path, min_score=7.0)
+
+    [claim] = _claims(service)
+    assert [citation.locator for citation in claim.citations] == [
+        "notes/source-a.md",
+        "notes/source-b.md",
+    ]