zettelforge 2.7.0__tar.gz → 2.8.0__tar.gz

{zettelforge-2.7.0 → zettelforge-2.8.0}/.github/workflows/ci.yml

@@ -13,10 +13,10 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
 
     - name: Set up Python
-      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
       with:
         python-version: '3.12'
 
@@ -35,13 +35,13 @@ jobs:
   pip-audit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
     - name: Set up Python
-      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
       with:
         python-version: '3.12'
     - name: Install pip-audit
-      run: pip install pip-audit
+      run: pip install pip-audit==2.9.0
     - name: Audit dependencies (any reported vuln blocks)
       run: |
         pip install -e ".[dev]" || pip install -e "."
@@ -49,7 +49,7 @@ jobs:
         # --ignore-vuln=CVE-... with a citation when the finding is
         # explicitly accepted per GOV-009 §"Vulnerability Response".
         #
-        # CVE-2026-3219: vulnerability in `pip` itself (the package
+        # CVE-2026-3219 / PYSEC-2026-196: vulnerability in `pip` itself (the package
         # manager), not a project dependency. The runner's pip is
         # supplied by GitHub's setup-python image and is not something
         # ZettelForge's pyproject can pin or upgrade. Risk-accepted
@@ -57,25 +57,59 @@ jobs:
         # install, not at runtime; CI builds in ephemeral runners with
         # no persistent state. Re-evaluate when GitHub's images ship a
         # patched pip.
-        pip-audit --strict --vulnerability-service=osv \
-          --ignore-vuln=CVE-2026-3219
+        #
+        # CVE-2023-36464 / GHSA-4vvm-4w3v-6mr8: medium-severity
+        # infinite-loop DoS in PyPDF2 3.0.1, introduced transitively by
+        # Maigret. PyPDF2 has no patched release under that package name
+        # (upstream recommends migrating to pypdf>=3.9.0), and ZettelForge's
+        # AGE-120 username collector does not parse attacker-supplied PDFs or
+        # invoke Maigret report generation. Accepted for AGE-120 because the
+        # GOV-009 blocking threshold is HIGH/CRITICAL and the collector
+        # lazy-imports/fails closed.
+        #
+        # --skip-editable: do not audit the editable local zettelforge
+        # install. It is the project itself, not a third-party dependency,
+        # and on a release-prep PR its version is bumped ahead of PyPI
+        # (e.g. 2.8.0 before publish), so pip-audit cannot resolve it.
+        #
+        # --strict is intentionally NOT used: it fails the run whenever a
+        # distribution can't be audited, which includes the editable
+        # package we deliberately skip ("distribution marked as editable").
+        # pip-audit still exits non-zero on any real reported vulnerability,
+        # so the "any reported vuln blocks" gate is preserved; every
+        # third-party dependency (and the runner's pip) is still audited.
+        pip-audit --skip-editable \
+          --ignore-vuln=CVE-2026-3219 \
+          --ignore-vuln=PYSEC-2026-196 \
+          --ignore-vuln=CVE-2023-36464
 
   test:
     runs-on: ubuntu-latest
     needs: lint
     strategy:
       fail-fast: false
+      # The fastembed model download is shared across Python versions. Running
+      # these jobs in parallel can double-hit HuggingFace and trigger 429s.
+      max-parallel: 1
       matrix:
         python-version: ['3.12', '3.13']
 
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Cache fastembed model
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
+      with:
+        path: |
+          ~/.cache/fastembed
+          ~/.cache/huggingface
+        key: fastembed-nomic-embed-text-v1.5-Q-${{ runner.os }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -102,19 +136,21 @@ jobs:
 
     - name: Upload coverage
       if: matrix.python-version == '3.12'
-      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354
-      with:
-        file: ./coverage.xml
-        fail_ci_if_error: false
+      continue-on-error: true
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      run: |
+        python -m pip install codecov-cli
+        codecovcli upload-process -f ./coverage.xml
 
   governance:
     runs-on: ubuntu-latest
     needs: lint
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
 
     - name: Set up Python
-      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
       with:
         python-version: '3.12'
 
@@ -139,10 +175,10 @@ jobs:
     needs: [test, governance]
 
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
 
     - name: Set up Python
-      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
       with:
         python-version: '3.12'
 

{zettelforge-2.7.0 → zettelforge-2.8.0}/.github/workflows/docs.yml

@@ -14,8 +14,8 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
         with:
           python-version: '3.12'
       - run: pip install mkdocs-material

{zettelforge-2.7.0 → zettelforge-2.8.0}/.github/workflows/publish.yml

@@ -6,15 +6,19 @@ on:
 
 jobs:
   publish:
+    # Package-only releases (for example packages-v0.1.0) are not
+    # Python package releases and must not attempt a PyPI upload.
+    if: startsWith(github.event.release.tag_name, 'v')
     runs-on: ubuntu-latest
     permissions:
+      contents: read
       id-token: write  # trusted publishing
 
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
 
     - name: Set up Python
-      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
       with:
         python-version: '3.12'
 

{zettelforge-2.7.0 → zettelforge-2.8.0}/.github/workflows/snyk-security.yml

@@ -17,10 +17,10 @@ jobs:
       actions: read
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
 
       - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
         with:
           python-version: '3.12'
 
@@ -65,7 +65,7 @@ jobs:
 
       - name: Upload SARIF to GitHub
         if: steps.check_token.outputs.has_token == 'true'
-        uses: github/codeql-action/upload-sarif@b25d0ebf40e5b63ee81e1bd6e5d2a12b7c2aeb61
+        uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e
         with:
           sarif_file: snyk-code.sarif
         continue-on-error: true

{zettelforge-2.7.0 → zettelforge-2.8.0}/ARCHITECTURE.md

@@ -3,6 +3,19 @@
 Visual diagram: [`docs/architecture-diagram.mmd`](docs/architecture-diagram.mmd)
 Deep explanation: [`docs/explanation/architecture.md`](docs/explanation/architecture.md)
 
+## Foundational Model: Two Hemispheres
+
+ZettelForge is modeled on the brain. A symbolic left hemisphere (the typed,
+STIX-aligned knowledge graph with schema enforcement and rule-based inference) is
+the precise, explainable authority of record. An associative right hemisphere
+(blended vector and graph retrieval with GraphRAG and agentic synthesis) explores
+and proposes. Underneath both, knowledge is atomic facts connected by directed,
+time-stamped, pairwise associations, with time and provenance as first-class
+primitives (Cognitive Data Model, Pieris 2025). The components below implement
+this model. See [design philosophy](docs/explanation/design-philosophy-dual-hemisphere.md)
+for the rationale and research basis, and [GOVERNANCE.md](GOVERNANCE.md) for the
+binding design commitment.
+
 ## Storage
 
 ZettelForge uses a `StorageBackend` ABC (33 methods) with pluggable

{zettelforge-2.7.0 → zettelforge-2.8.0}/CHANGELOG.md

@@ -6,6 +6,54 @@ Versioning follows [Semantic Versioning](https://semver.org/).
 
 ## [Unreleased]
 
+## [2.8.0] - 2026-06-26
+
+Feature release. Extends the RFC-016 OSINT layer with passive ingest and
+live AGE-120 enrichers, adds AGE-127 prompt-injection /
+retrieval-poisoning guardrails across the memory pipeline, and lays the
+RFC-018 Phase 0 enrichment job ledger foundation, alongside configurable
+LLM generation budgets and bulk detection-ingest hardening. No data
+migration is required.
+
+### Added
+
+- **Prompt-injection and retrieval-poisoning guardrails** (AGE-127). A
+  deterministic guard for untrusted memory and CTI text, wired into the
+  remember, recall, synthesis, fact-extraction, entity-indexing,
+  note-construction, intent-classification, memory-update, and evolution
+  boundaries. LLM prompts are hardened to treat query and context text as
+  untrusted evidence. (#166)
+- **Passive OSINT executor** (RFC-016 Phase 1.5). Validates collector
+  tuples and persists knowledge-graph writes; scopes OSINT alias caching to
+  each `KnowledgeGraph` instance; canonicalizes Organization values to
+  avoid WHOIS/RDAP duplicates; treats an explicit empty collector
+  allow-list as empty; adds user-facing passive OSINT docs and mkdocs
+  navigation. (#163)
+- **Live OSINT enrichers** (AGE-120). Native RFC-016 collectors that feed
+  the graph backend: WHOIS/DNS, maigret/sherlock username discovery, HIBP
+  breach lookup, and blockchain wallet transactions, with OSINT
+  ontology / executor / entity-resolution / graph-persistence extensions.
+  The network collectors are an opt-in `[osint]` extra, key-gated and
+  fail-closed, and are never auto-triggered by `remember()`. Known
+  fast-follow robustness items are tracked in #176. (#167)
+- **RFC-018 Phase 0 enrichment job ledger.** Durable SQLite job-metadata
+  foundation with `MemoryManager` queue-state recording and regression
+  tests, plus the RFC-018 spec for the next ingestion, enrichment,
+  parser-safety, metadata-policy, operator-health, and release-train
+  improvements. (#178)
+- Configurable LLM generation budgets for causal extraction, synthesis,
+  fact extraction, NER, and memory evolution, plus `reasoning_model` scaling
+  floors and `<think>` / `<thinking>` JSON parse stripping. (#153)
+- Bulk Sigma/YARA ingest can defer enrichment and drain once via
+  `MemoryManager.flush()`. (#153)
+
+### Fixed
+
+- Hardened CCCS YARA metadata validation against multiline regex injection
+  and overly permissive author values. (#153)
+- `MemoryManager.flush()` now waits for in-flight enrichment work, not only
+  queued work; cached `Plyara` parsing is serialized for concurrent callers. (#153)
+
 ## [2.7.0] - 2026-05-26
 
 Security and OSINT release. Adds the RFC-016 OSINT layer and the first

zettelforge-2.8.0/GOVERNANCE.md

@@ -0,0 +1,49 @@
+# Governance
+
+## Maintainer
+
+ZettelForge is maintained by Patrick Roland (@rolandpg).
+
+## Licensing Commitment
+
+ZettelForge is MIT licensed. This will not change.
+
+## Design Commitment: Brain-Inspired Memory Architecture
+
+ZettelForge is modeled on the brain by deliberate decision, not metaphor. This
+commitment is binding on the architecture and will not be quietly abandoned.
+
+The memory engine is dual-hemisphere. A symbolic left hemisphere (a typed,
+STIX/ATT&CK-aligned knowledge graph with schema enforcement and rule-based
+inference) is the authority of record: precise, constraint-checked, and
+explainable. An associative right hemisphere (blended vector and graph retrieval,
+GraphRAG, and agentic synthesis) explores and proposes, and writes back into the
+symbolic layer only through validated paths. The two layers stay distinct.
+
+Underneath both, knowledge is represented as atomic facts connected by directed,
+time-stamped, pairwise associations (the Cognitive Data Model, Pieris 2025). Time
+and provenance are first-class primitives: every note and claim carries its
+source, markings (TLP), confidence, and time validity, and every retrieval result
+carries provenance. Ingested content is untrusted and is sanitized or isolated
+before any model call. Quantitative quality or performance claims are reproduced
+on the project's own CTI benchmark suite before being stated as fact.
+
+Changes to storage, retrieval, the knowledge graph, or ingestion MUST preserve
+this model, and design decisions SHOULD cite the relevant hemisphere or layer.
+The full rationale and research basis are in
+[docs/explanation/design-philosophy-dual-hemisphere.md](docs/explanation/design-philosophy-dual-hemisphere.md).
+
+## Extension Boundary
+
+The extension package (`zettelforge-enterprise`) provides features
+that require external infrastructure (TypeDB, OpenCTI, multi-tenant
+OAuth). The open source project will never be degraded to create
+commercial incentive.
+
+Rule: if a feature works with JSONL + local embeddings, it belongs
+in this repo.
+
+## Contributions
+
+Community contributions are reviewed on their technical merits.
+All contributions to this repository remain MIT licensed.

{zettelforge-2.7.0 → zettelforge-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zettelforge
-Version: 2.7.0
+Version: 2.8.0
 Summary: ZettelForge: Agentic Memory System with vector search, knowledge graph, and synthesis
 Project-URL: Homepage, https://github.com/rolandpg/zettelforge
 Project-URL: Documentation, https://docs.threatrecall.ai
@@ -39,10 +39,11 @@ Provides-Extra: crewai
 Requires-Dist: crewai>=1.14.0; extra == 'crewai'
 Provides-Extra: dev
 Requires-Dist: dnspython>=2.4.0; extra == 'dev'
-Requires-Dist: fastapi>=0.100.0; extra == 'dev'
+Requires-Dist: fastapi<0.138.2,>=0.100.0; extra == 'dev'
 Requires-Dist: ipwhois>=1.2.0; extra == 'dev'
 Requires-Dist: jinja2>=3.0.0; extra == 'dev'
 Requires-Dist: langchain-core>=0.2.0; extra == 'dev'
+Requires-Dist: maigret>=0.4.0; extra == 'dev'
 Requires-Dist: mypy>=1.0.0; extra == 'dev'
 Requires-Dist: psutil>=5.9.0; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
@@ -50,6 +51,7 @@ Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
 Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: python-whois>=0.9.0; extra == 'dev'
 Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Requires-Dist: sherlock-project>=0.14.0; extra == 'dev'
 Requires-Dist: uvicorn>=0.20.0; extra == 'dev'
 Provides-Extra: extensions
 Requires-Dist: zettelforge-enterprise>=2.1.0; extra == 'extensions'
@@ -67,13 +69,15 @@ Requires-Dist: onnxruntime-genai>=0.4.0; extra == 'local-onnx'
 Provides-Extra: osint
 Requires-Dist: dnspython>=2.4.0; extra == 'osint'
 Requires-Dist: ipwhois>=1.2.0; extra == 'osint'
+Requires-Dist: maigret>=0.4.0; extra == 'osint'
 Requires-Dist: python-whois>=0.9.0; extra == 'osint'
+Requires-Dist: sherlock-project>=0.14.0; extra == 'osint'
 Provides-Extra: pii
 Requires-Dist: presidio-analyzer>=2.2.0; extra == 'pii'
 Requires-Dist: presidio-anonymizer>=2.2.0; extra == 'pii'
 Requires-Dist: spacy>=3.5.0; extra == 'pii'
 Provides-Extra: web
-Requires-Dist: fastapi>=0.100.0; extra == 'web'
+Requires-Dist: fastapi<0.138.2,>=0.100.0; extra == 'web'
 Requires-Dist: jinja2>=3.0.0; extra == 'web'
 Requires-Dist: psutil>=5.9.0; extra == 'web'
 Requires-Dist: uvicorn>=0.20.0; extra == 'web'

{zettelforge-2.7.0 → zettelforge-2.8.0}/benchmarks/BENCHMARK_REPORT.md

@@ -28,6 +28,79 @@ ZettelForge was evaluated across five benchmark suites. The system runs with zer
 
 ---
 
+## 0. Performance session 2026-06-09 (v2.8.0-dev, branch perf/cti-memory-40)
+
+All numbers below are same-machine (DGX Spark GB10), same-day, deterministic
+config: enrichment disabled (`ZETTELFORGE_ENRICHMENT_ENABLED=false`), keyword
+judge, heuristic answer extraction (no synthesis LLM installed). The clean
+baseline was measured first on unmodified v2.7.0 source after repairing the
+rotted harnesses (dead `disable_enrichment` kwarg, removed `remember_chunked`
+API). Raw logs: `benchmarks/results/session_2026-06-09/`.
+
+| Metric | v2.7.0 baseline | optimized | delta |
+|--------|-----------------|-----------|-------|
+| LoCoMo accuracy (keyword judge) | 7.0% | 11.0% | +57% relative |
+| LoCoMo p50 / p95 latency | 336ms / 387ms | 170ms / 193ms | -49% / -50% |
+| LoCoMo ingest (272 sessions) | 262.5s (1.0/s) | 33.8s (8.0/s) | 7.8x |
+| CTI retrieval accuracy | 75.0% | 75.0% | held |
+| CTI p50 latency (idle machine) | 79ms | 39ms | -51% |
+| recall p95 (profiled, 60 calls) | 258ms | 93ms | -64% |
+| recall mean (profiled) | 117.6ms | 54.8ms | -53% |
+
+Note on LoCoMo baselines: the published 22% (v2.1.1) used a local synthesis
+LLM (qwen2.5:3b) that is not installed on this host; both columns above use
+the same deterministic heuristic-extraction path, so the comparison is
+apples to apples. Latency includes harness overhead (keyword boost scan and
+synthesis fallback), not just `recall()`.
+
+### What changed
+
+1. **Scoped knowledge graph reads.** `_recall_inner` traversed the
+   process-global JSONL KG (109MB on this host, mixing every store) while
+   writes went to the per-store SQLite KG. Isolated stores saw up to ~2000
+   phantom note IDs per entity query and never saw their own graph. Recall
+   now reads the store's KG via `StoreGraphSource`.
+2. **MemSAD gate vectorized.** The write-time anomaly gate was 93% of
+   remember() latency at 50 references (~1.1s/ingest): O(n^2) pure-Python
+   cosines plus n^2 n-gram recounts per ingest. numpy pairwise scoring,
+   content-hash counter cache, and a bounded reference fetch
+   (`get_recent_notes_by_domain`) brought warm evaluate() to ~3.4ms with
+   scores pinned to the original math at 1e-9 by characterization tests.
+3. **Rerank policy.** Cross-encoder rerank is the dominant read cost and is
+   worth +15pp CTI accuracy (75% vs 60% without it). Grid-tuned bounds:
+   8 candidates, 256 chars/doc (accuracy holds from 50x512 down to 8x128;
+   collapses below 8 candidates). `rerank_model` is configurable; the
+   model grid kept ms-marco-MiniLM-L-6-v2.
+4. **ONNX thread pinning.** 20-core default oversubscribed small batches:
+   8 threads cut rerank 23.7ms to 11.5ms and query embedding 5.9ms to 4.5ms.
+5. **Embedding LRU cache** keyed by (model, sha256(text)) — first
+   integration of the dormant cache.py.
+6. **Entity fan-out gate.** Query entities whose KG out-degree exceeds
+   `retrieval.entity_max_fanout` (default 25) are skipped by graph and
+   entity-augmentation stages (conversational speaker names map to every
+   session and flood blended recall).
+7. **Enrichment off-switch** (`ZETTELFORGE_ENRICHMENT_ENABLED`) restoring
+   deterministic benchmark ingestion; `remember_chunked()` restored.
+
+### Chunked-ingestion configuration (recorded, not default)
+
+`LOCOMO_CHUNK_SIZE=800` stores each session as ~800-char chunks
+(MemPalace granularity, no 4000-char truncation): 13.0% accuracy at
+p50 347ms / p95 418ms on a ~1400-note store. Compared to the v2.7.0
+baseline at effectively the same latency (336ms), that is +86%
+relative accuracy; compared to the default optimized config it trades
+2x latency for +2pp. Default stays full-session (11.0% at 170ms).
+
+### Negative result (recorded)
+
+Free-text person extraction (capitalized tokens in running text) dropped
+LoCoMo from 11% to 5% by reshuffling supersession chains at ingest, with no
+single-hop or multi-hop gain. Reverted same day; regression-locked in
+`tests/test_conversational_entities.py`. Conversational NER should come via
+the RFC-001 LLM path, not regex.
+
+---
+
 ## 1. CTI Retrieval Benchmark (Domain Benchmark)
 
 **Date:** 2026-04-10 | **Corpus:** 8 real-world-style CTI reports | **Queries:** 20

{zettelforge-2.7.0 → zettelforge-2.8.0}/benchmarks/cti_retrieval_benchmark.py

@@ -22,6 +22,8 @@ from datetime import datetime
 from typing import List, Dict, Tuple
 
 os.environ["ZETTELFORGE_BACKEND"] = "jsonl"
+# Deterministic ingestion: no background LLM enrichment during benchmarks.
+os.environ.setdefault("ZETTELFORGE_ENRICHMENT_ENABLED", "false")
 
 from zettelforge import MemoryManager
 

{zettelforge-2.7.0 → zettelforge-2.8.0}/benchmarks/cti_retrieval_results.json

@@ -1,78 +1,78 @@
 {
   "meta": {
-    "date": "2026-04-10T08:05:55.405026",
+    "date": "2026-06-09T13:56:15.802128",
     "reports": 8,
     "queries": 20
   },
   "full_session": {
     "strategy": "full_session",
     "notes": 8,
-    "ingest_time_s": 69.1,
+    "ingest_time_s": 3.4,
     "accuracy": 75.0,
-    "avg_score": 0.875,
-    "p50_latency_ms": 620.0,
-    "p95_latency_ms": 2732.0,
+    "avg_score": 0.85,
+    "p50_latency_ms": 39.0,
+    "p95_latency_ms": 159.0,
     "by_category": {
       "tool-attribution": {
         "accuracy": 40.0,
         "avg_score": 0.7,
-        "p50_latency_ms": 1343.0
+        "p50_latency_ms": 42.0
       },
       "cve-linkage": {
         "accuracy": 75.0,
-        "avg_score": 0.875,
-        "p50_latency_ms": 794.0
+        "avg_score": 0.75,
+        "p50_latency_ms": 38.0
       },
       "attribution": {
         "accuracy": 100.0,
         "avg_score": 1.0,
-        "p50_latency_ms": 611.0
+        "p50_latency_ms": 59.0
       },
       "temporal": {
         "accuracy": 66.7,
         "avg_score": 0.833,
-        "p50_latency_ms": 569.0
+        "p50_latency_ms": 41.0
       },
       "multi-hop": {
         "accuracy": 100.0,
         "avg_score": 1.0,
-        "p50_latency_ms": 644.0
+        "p50_latency_ms": 38.0
       }
     }
   },
   "chunked_800": {
     "strategy": "chunked_800",
     "notes": 8,
-    "ingest_time_s": 56.5,
+    "ingest_time_s": 0.1,
     "accuracy": 75.0,
-    "avg_score": 0.875,
-    "p50_latency_ms": 706.0,
-    "p95_latency_ms": 2729.0,
+    "avg_score": 0.85,
+    "p50_latency_ms": 52.0,
+    "p95_latency_ms": 59.0,
     "by_category": {
       "tool-attribution": {
         "accuracy": 40.0,
         "avg_score": 0.7,
-        "p50_latency_ms": 1299.0
+        "p50_latency_ms": 50.0
       },
       "cve-linkage": {
         "accuracy": 75.0,
-        "avg_score": 0.875,
-        "p50_latency_ms": 795.0
+        "avg_score": 0.75,
+        "p50_latency_ms": 52.0
       },
       "attribution": {
         "accuracy": 100.0,
         "avg_score": 1.0,
-        "p50_latency_ms": 535.0
+        "p50_latency_ms": 52.0
       },
       "temporal": {
         "accuracy": 66.7,
         "avg_score": 0.833,
-        "p50_latency_ms": 772.0
+        "p50_latency_ms": 54.0
       },
       "multi-hop": {
         "accuracy": 100.0,
         "avg_score": 1.0,
-        "p50_latency_ms": 741.0
+        "p50_latency_ms": 33.0
       }
     }
   }

zettelforge-2.8.0/benchmarks/instrument_lookups.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Instrument note-lookup volume per recall stage.
+
+Counts store.get_note_by_id calls (total vs unique ids) and graph result
+sizes per query to locate the redundant-lookup source the profiler exposed
+(~476 lookups/query on an 8-note corpus).
+
+Usage:
+  python benchmarks/instrument_lookups.py
+"""
+import os
+import tempfile
+
+os.environ.setdefault('ZETTELFORGE_ENRICHMENT_ENABLED', 'false')
+
+from cti_retrieval_benchmark import CTI_QUERIES, CTI_REPORTS
+
+from zettelforge import MemoryManager
+from zettelforge.graph_retriever import GraphRetriever
+
+
+def main() -> None:
+    tmpdir = tempfile.mkdtemp(prefix='instr_lookups_')
+    mm = MemoryManager(jsonl_path=f'{tmpdir}/notes.jsonl', lance_path=f'{tmpdir}/vectordb')
+    for report in CTI_REPORTS:
+        mm.remember(report['content'], source_type='threat_report', source_ref=report['id'], domain='cti')
+
+    # Wrap get_note_by_id with a counter
+    calls = {'total': 0, 'ids': []}
+    orig = mm.store.get_note_by_id
+
+    def counting(nid):
+        calls['total'] += 1
+        calls['ids'].append(nid)
+        return orig(nid)
+
+    mm.store.get_note_by_id = counting
+
+    # Wrap graph retrieval to report result sizes
+    orig_retrieve = GraphRetriever.retrieve_note_ids
+    graph_sizes = []
+
+    def counting_retrieve(self, query_entities, max_depth=2):
+        res = orig_retrieve(self, query_entities, max_depth=max_depth)
+        graph_sizes.append(len(res))
+        return res
+
+    GraphRetriever.retrieve_note_ids = counting_retrieve
+
+    print(f'{"query":<48} {"lookups":>8} {"unique":>7} {"graph_n":>8}')
+    for qa in CTI_QUERIES:
+        calls['total'] = 0
+        calls['ids'] = []
+        graph_sizes.clear()
+        mm.recall(qa['question'], k=10, exclude_superseded=False)
+        uniq = len(set(calls['ids']))
+        gsz = graph_sizes[0] if graph_sizes else 0
+        print(f'{qa["question"][:46]:<48} {calls["total"]:>8} {uniq:>7} {gsz:>8}')
+
+
+if __name__ == '__main__':
+    main()