etchmem 0.1.4__tar.gz → 0.1.6__tar.gz

{etchmem-0.1.4 → etchmem-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etchmem
-Version: 0.1.4
+Version: 0.1.6
 Summary: A pluggable, embeddable reconsolidating skill memory engine for AI agents
 Author: Andrey Olishchuk
 License: MIT
@@ -46,7 +46,7 @@ pip install etchmem
 
 `etchmem` gives your agent a persistent memory that matures through living. You deposit raw observations — facts, outcomes, feedback — scoped to a skill or left general. You retrieve with natural-language queries. Periodically you consolidate, and the engine synthesizes scattered signal into compact knowledge articles, rewriting anything that has been recalled against fresh context.
 
-It is not a vector database and not an agent runtime. It sits above [ChromaDB](https://www.trychroma.com/), which handles all storage and embedding, and exposes exactly **three methods**.
+It is not a vector database and not an agent runtime. It sits above [ChromaDB](https://www.trychroma.com/), which handles all storage and embedding, and exposes a small, deliberate API: remember, recall, consolidate, export, and stats.
 
 ---
 
@@ -88,6 +88,18 @@ Retrieves relevant knowledge and emits a recall-event for future reconsolidation
 
 Runs the consolidation worker: clusters the buffer, forms new injected articles, reconsolidates recalled articles against fresh context, hard-deletes superseded ones. Returns a summary dict with counts (`formed`, `reconsolidated`, `dropped`, `kept`, `flushed`, `superseded`). Requires an LLM API key.
 
+### `export() → dict`
+
+Serializes the entire injected knowledge store to disk. Writes one `<id>.json` file per synthesized article into `.etchmem/export/<UTC-timestamp>/` next to the data directory. No LLM call — cheap.
+
+Returns `{"export_dir": "...", "count": N, "documents": [...]}`.
+
+Use this to transfer institutional memory to another agent, or to produce fine-tuning data shaped by real task outcomes rather than hand-authored examples. Experience becomes a portable artifact.
+
+### `stats() → dict`
+
+Returns live collection sizes for all three tiers: `{"relational": N, "buffer": N, "injected": N}`. Useful for monitoring and debugging.
+
 ---
 
 ## Hello world
@@ -120,6 +132,12 @@ print(summary)
 # will be blended in during reconsolidation.
 results = engine.recall("Eiffel Tower visitors")
 print(results[0].content)
+
+# Export the full knowledge store — one JSON file per synthesized article.
+# Transfer to another agent, or use as fine-tuning data.
+result = engine.export()
+print(result["export_dir"])   # e.g. /your/project/.etchmem/export/20260525T120000Z
+print(result["count"])        # 1
 ```
 
 ---

{etchmem-0.1.4 → etchmem-0.1.6}/README.md

@@ -18,7 +18,7 @@ pip install etchmem
 
 `etchmem` gives your agent a persistent memory that matures through living. You deposit raw observations — facts, outcomes, feedback — scoped to a skill or left general. You retrieve with natural-language queries. Periodically you consolidate, and the engine synthesizes scattered signal into compact knowledge articles, rewriting anything that has been recalled against fresh context.
 
-It is not a vector database and not an agent runtime. It sits above [ChromaDB](https://www.trychroma.com/), which handles all storage and embedding, and exposes exactly **three methods**.
+It is not a vector database and not an agent runtime. It sits above [ChromaDB](https://www.trychroma.com/), which handles all storage and embedding, and exposes a small, deliberate API: remember, recall, consolidate, export, and stats.
 
 ---
 
@@ -60,6 +60,18 @@ Retrieves relevant knowledge and emits a recall-event for future reconsolidation
 
 Runs the consolidation worker: clusters the buffer, forms new injected articles, reconsolidates recalled articles against fresh context, hard-deletes superseded ones. Returns a summary dict with counts (`formed`, `reconsolidated`, `dropped`, `kept`, `flushed`, `superseded`). Requires an LLM API key.
 
+### `export() → dict`
+
+Serializes the entire injected knowledge store to disk. Writes one `<id>.json` file per synthesized article into `.etchmem/export/<UTC-timestamp>/` next to the data directory. No LLM call — cheap.
+
+Returns `{"export_dir": "...", "count": N, "documents": [...]}`.
+
+Use this to transfer institutional memory to another agent, or to produce fine-tuning data shaped by real task outcomes rather than hand-authored examples. Experience becomes a portable artifact.
+
+### `stats() → dict`
+
+Returns live collection sizes for all three tiers: `{"relational": N, "buffer": N, "injected": N}`. Useful for monitoring and debugging.
+
 ---
 
 ## Hello world
@@ -92,6 +104,12 @@ print(summary)
 # will be blended in during reconsolidation.
 results = engine.recall("Eiffel Tower visitors")
 print(results[0].content)
+
+# Export the full knowledge store — one JSON file per synthesized article.
+# Transfer to another agent, or use as fine-tuning data.
+result = engine.export()
+print(result["export_dir"])   # e.g. /your/project/.etchmem/export/20260525T120000Z
+print(result["count"])        # 1
 ```
 
 ---

{etchmem-0.1.4 → etchmem-0.1.6}/etchmem/consolidate/worker.py

@@ -52,8 +52,14 @@ accurate knowledge article from a set of raw signal fragments.
 {fragments}
 
 ## Instructions
+- Summarize all stored information properly: compress each point to its main
+  idea while keeping the same words and phrasing from the fragments wherever
+  possible.
 - Write a single, cohesive knowledge article that captures all key facts.
 - Do NOT invent facts not present in the fragments.
+- Opinions, preferences, and subjective claims must NOT be stated as facts.
+  Attribute them explicitly, e.g. "There is an opinion that grapefruits are
+  the best" — never "Grapefruits are the best."
 - Each fragment may be prefixed with a timestamp like `[21 Jun 2025, 15:25 UTC]`.
   Preserve these timestamps verbatim inline, anchoring the facts they precede,
   e.g. "As of 21 Jun 2025, 15:25 UTC, the deployment was rolled back."
@@ -75,9 +81,16 @@ accurate and incorporates the new information.
 {fresh}
 
 ## Instructions
+- Write ONLY what appears in the existing article and fresh context. Do NOT
+  add new facts, inferences, or knowledge from your own training.
+- Summarize properly: compress to the main idea of each point while keeping
+  the same words and phrasing from the source material wherever possible.
 - Preserve ALL load-bearing facts from the existing article.
 - Incorporate relevant new information from the fresh context.
 - Remove outdated claims only when the fresh context explicitly supersedes them.
+- Opinions, preferences, and subjective claims must NOT be stated as facts.
+  Attribute them explicitly, e.g. "There is an opinion that grapefruits are
+  the best" — never "Grapefruits are the best."
 - Fresh context fragments may be prefixed with a timestamp like `[21 Jun 2025, 15:25 UTC]`.
   Preserve these timestamps verbatim inline, anchoring the facts they precede.
   Also carry forward any timestamps already present in the existing article.

{etchmem-0.1.4 → etchmem-0.1.6}/etchmem/engine.py

@@ -176,6 +176,77 @@ class Engine:
         self._connector = build_connector(self._cfg)
         self._connector_built = True
 
+    def export(self) -> dict[str, Any]:
+        """
+        Serialize the entire injected knowledge store as JSON.
+
+        Writes every synthesized article, tag, and metadata from the
+        *injected* collection into individual ``.json`` files under::
+
+            .etchmem/export/<UTC-timestamp>/   (one file per document)
+
+        The export root is a sibling of ``config.data_dir`` named
+        ``.etchmem``, so it lives next to the Chroma data directory in
+        the working tree rather than inside it.
+
+        Use the export to:
+          - Transfer institutional memory to another agent.
+          - Use accumulated experience directly as LLM fine-tuning data:
+            knowledge shaped by real task outcomes, not hand-authored
+            examples.  Experience becomes a transferable artifact.
+
+        Returns:
+            dict with keys:
+
+            ``export_dir``
+                Absolute path of the timestamped directory that was
+                created.
+            ``count``
+                Number of documents written.
+            ``documents``
+                List of dicts, one per exported article, each containing
+                ``id``, ``content``, ``source_hashes``, ``skill``,
+                ``tags``, ``metadata``, and ``created_at``.
+        """
+        import datetime
+        import json as _json
+        import os
+
+        # Derive export root: <parent-of-data_dir>/.etchmem/export/<ts>
+        data_dir_abs = os.path.abspath(self._cfg.data_dir)
+        parent = os.path.dirname(data_dir_abs)
+        timestamp = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
+        export_dir = os.path.join(parent, ".etchmem", "export", timestamp)
+        os.makedirs(export_dir, exist_ok=True)
+
+        # Fetch all InjectedArticle documents
+        from etchmem.models import InjectedArticle
+        raw = self._injected.get_all(filters={"doc_type": "injected"})
+
+        documents: list[dict[str, Any]] = []
+        for item in raw:
+            if not isinstance(item, InjectedArticle):
+                continue
+            doc_dict: dict[str, Any] = {
+                "id": item.id,
+                "content": item.content,
+                "source_hashes": sorted(item.source_hashes),
+                "skill": item.skill,
+                "tags": item.tags,
+                "metadata": item.metadata,
+                "created_at": item.created_at,
+            }
+            file_path = os.path.join(export_dir, f"{item.id}.json")
+            with open(file_path, "w", encoding="utf-8") as fh:
+                _json.dump(doc_dict, fh, indent=2, ensure_ascii=False)
+            documents.append(doc_dict)
+
+        return {
+            "export_dir": export_dir,
+            "count": len(documents),
+            "documents": documents,
+        }
+
     # ── Convenience properties ────────────────────────────────────────────
 
     @property

{etchmem-0.1.4 → etchmem-0.1.6}/etchmem.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etchmem
-Version: 0.1.4
+Version: 0.1.6
 Summary: A pluggable, embeddable reconsolidating skill memory engine for AI agents
 Author: Andrey Olishchuk
 License: MIT
@@ -46,7 +46,7 @@ pip install etchmem
 
 `etchmem` gives your agent a persistent memory that matures through living. You deposit raw observations — facts, outcomes, feedback — scoped to a skill or left general. You retrieve with natural-language queries. Periodically you consolidate, and the engine synthesizes scattered signal into compact knowledge articles, rewriting anything that has been recalled against fresh context.
 
-It is not a vector database and not an agent runtime. It sits above [ChromaDB](https://www.trychroma.com/), which handles all storage and embedding, and exposes exactly **three methods**.
+It is not a vector database and not an agent runtime. It sits above [ChromaDB](https://www.trychroma.com/), which handles all storage and embedding, and exposes a small, deliberate API: remember, recall, consolidate, export, and stats.
 
 ---
 
@@ -88,6 +88,18 @@ Retrieves relevant knowledge and emits a recall-event for future reconsolidation
 
 Runs the consolidation worker: clusters the buffer, forms new injected articles, reconsolidates recalled articles against fresh context, hard-deletes superseded ones. Returns a summary dict with counts (`formed`, `reconsolidated`, `dropped`, `kept`, `flushed`, `superseded`). Requires an LLM API key.
 
+### `export() → dict`
+
+Serializes the entire injected knowledge store to disk. Writes one `<id>.json` file per synthesized article into `.etchmem/export/<UTC-timestamp>/` next to the data directory. No LLM call — cheap.
+
+Returns `{"export_dir": "...", "count": N, "documents": [...]}`.
+
+Use this to transfer institutional memory to another agent, or to produce fine-tuning data shaped by real task outcomes rather than hand-authored examples. Experience becomes a portable artifact.
+
+### `stats() → dict`
+
+Returns live collection sizes for all three tiers: `{"relational": N, "buffer": N, "injected": N}`. Useful for monitoring and debugging.
+
 ---
 
 ## Hello world
@@ -120,6 +132,12 @@ print(summary)
 # will be blended in during reconsolidation.
 results = engine.recall("Eiffel Tower visitors")
 print(results[0].content)
+
+# Export the full knowledge store — one JSON file per synthesized article.
+# Transfer to another agent, or use as fine-tuning data.
+result = engine.export()
+print(result["export_dir"])   # e.g. /your/project/.etchmem/export/20260525T120000Z
+print(result["count"])        # 1
 ```
 
 ---

{etchmem-0.1.4 → etchmem-0.1.6}/etchmem.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ etchmem/util/__init__.py
 etchmem/util/affectability.py
 etchmem/util/hashing.py
 tests/test_consolidate_idempotent.py
+tests/test_export.py
 tests/test_formation_buckets.py
 tests/test_recall.py
 tests/test_reconsolidation.py

{etchmem-0.1.4 → etchmem-0.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "etchmem"
-version = "0.1.4"
+version = "0.1.6"
 description = "A pluggable, embeddable reconsolidating skill memory engine for AI agents"
 readme = "README.md"
 license = { text = "MIT" }

etchmem-0.1.6/tests/test_export.py

@@ -0,0 +1,182 @@
+"""
+Tests for engine.export().
+
+Covers:
+  - Empty injected store produces export_dir, count=0, documents=[].
+  - Export directory is created under .etchmem/export/<timestamp>/.
+  - Each injected article is written as a separate JSON file.
+  - JSON file content matches the InjectedArticle fields.
+  - Multiple articles all appear in the export.
+  - Return dict has the expected keys and shapes.
+  - Repeated export calls create distinct timestamped directories.
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+
+import pytest
+
+from etchmem import Config, Engine
+from etchmem.models import InjectedArticle
+
+
+# ── Fixture ───────────────────────────────────────────────────────────────────
+
+@pytest.fixture
+def engine(tmp_path):
+    """Engine backed by a fresh temporary Chroma directory, using offline
+    hash embeddings (no model download required)."""
+    cfg = Config(
+        data_dir=str(tmp_path / "skillmem"),
+        llm_provider="auto",
+        openai_api_key=None,
+        anthropic_api_key=None,
+        embedding_function="hash",
+    )
+    return Engine(config=cfg)
+
+
+def _inject_article(engine: Engine, article_id: str, content: str, **kwargs) -> InjectedArticle:
+    """Helper: directly upsert an InjectedArticle into the injected store."""
+    article = InjectedArticle(
+        id=article_id,
+        content=content,
+        source_hashes=frozenset(kwargs.get("source_hashes", [article_id + "_src"])),
+        skill=kwargs.get("skill", None),
+        tags=kwargs.get("tags", {}),
+        metadata=kwargs.get("metadata", {}),
+        created_at=kwargs.get("created_at", time.time()),
+    )
+    engine._injected.upsert([article])
+    return article
+
+
+# ── Tests ─────────────────────────────────────────────────────────────────────
+
+def test_export_empty_store_returns_zero_count(engine):
+    """export() on an empty injected store should return count=0."""
+    result = engine.export()
+    assert result["count"] == 0
+    assert result["documents"] == []
+
+
+def test_export_creates_directory(engine, tmp_path):
+    """export() must create the .etchmem/export/<timestamp> directory."""
+    result = engine.export()
+    assert os.path.isdir(result["export_dir"])
+
+
+def test_export_dir_is_under_etchmem(engine, tmp_path):
+    """The export directory must live under .etchmem/export/ next to the data_dir."""
+    result = engine.export()
+    # data_dir is tmp_path/"skillmem"; its parent is tmp_path
+    expected_root = os.path.join(str(tmp_path), ".etchmem", "export")
+    assert result["export_dir"].startswith(expected_root)
+
+
+def test_export_single_article_creates_one_file(engine):
+    """One InjectedArticle → one .json file in the export directory."""
+    _inject_article(engine, "abc123", "Python uses reference counting.")
+    result = engine.export()
+
+    assert result["count"] == 1
+    export_dir = result["export_dir"]
+    files = [f for f in os.listdir(export_dir) if f.endswith(".json")]
+    assert len(files) == 1
+
+
+def test_export_file_named_after_article_id(engine):
+    """Each exported file must be named <article_id>.json."""
+    art_id = "myarticleid"
+    _inject_article(engine, art_id, "Some synthesized content.")
+    result = engine.export()
+
+    export_dir = result["export_dir"]
+    assert os.path.isfile(os.path.join(export_dir, f"{art_id}.json"))
+
+
+def test_export_json_content_matches_article(engine):
+    """The JSON file content must faithfully reproduce all article fields."""
+    art = _inject_article(
+        engine,
+        "detailed_id",
+        "ChromaDB stores embeddings on disk.",
+        source_hashes=["hash_a", "hash_b"],
+        skill="code-reviewer",
+        tags={"topic": "databases", "access_level": "public"},
+        metadata={"source": "https://docs.trychroma.com"},
+    )
+    result = engine.export()
+
+    file_path = os.path.join(result["export_dir"], f"{art.id}.json")
+    with open(file_path, encoding="utf-8") as fh:
+        data = json.load(fh)
+
+    assert data["id"] == art.id
+    assert data["content"] == art.content
+    assert set(data["source_hashes"]) == art.source_hashes
+    assert data["skill"] == art.skill
+    assert data["tags"] == art.tags
+    assert data["metadata"] == art.metadata
+    assert isinstance(data["created_at"], float)
+
+
+def test_export_multiple_articles(engine):
+    """All InjectedArticles in the store must be exported."""
+    ids = ["art_one", "art_two", "art_three"]
+    for aid in ids:
+        _inject_article(engine, aid, f"Content for {aid}.")
+
+    result = engine.export()
+
+    assert result["count"] == len(ids)
+    assert len(result["documents"]) == len(ids)
+    exported_ids = {doc["id"] for doc in result["documents"]}
+    assert exported_ids == set(ids)
+
+
+def test_export_return_dict_has_required_keys(engine):
+    """export() must return a dict with export_dir, count, and documents."""
+    result = engine.export()
+    assert "export_dir" in result
+    assert "count" in result
+    assert "documents" in result
+    assert isinstance(result["export_dir"], str)
+    assert isinstance(result["count"], int)
+    assert isinstance(result["documents"], list)
+
+
+def test_export_document_dict_has_required_fields(engine):
+    """Each document dict in the return value must have all expected fields."""
+    _inject_article(engine, "field_check", "Testing field presence.")
+    result = engine.export()
+
+    doc = result["documents"][0]
+    for field in ("id", "content", "source_hashes", "skill", "tags", "metadata", "created_at"):
+        assert field in doc, f"Missing field: {field}"
+
+
+def test_repeated_exports_use_distinct_directories(engine):
+    """Two export() calls must produce different timestamped directories."""
+    _inject_article(engine, "repeat_art", "Some knowledge.")
+    result_a = engine.export()
+    time.sleep(1.1)  # ensure the UTC second ticks over
+    result_b = engine.export()
+
+    assert result_a["export_dir"] != result_b["export_dir"]
+    assert os.path.isdir(result_a["export_dir"])
+    assert os.path.isdir(result_b["export_dir"])
+
+
+def test_export_source_hashes_is_sorted_list(engine):
+    """source_hashes in the JSON output must be a sorted list (JSON-serialisable)."""
+    _inject_article(
+        engine, "hash_order", "Hashes sorted.",
+        source_hashes=["zzz", "aaa", "mmm"],
+    )
+    result = engine.export()
+    doc = result["documents"][0]
+    assert doc["source_hashes"] == sorted(doc["source_hashes"])
+    assert isinstance(doc["source_hashes"], list)