outmem 0.1.0__tar.gz

outmem-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,69 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build sdist and wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install ripgrep
+        run: sudo apt-get update && sudo apt-get install -y ripgrep
+
+      - name: Verify release tag matches package version
+        run: |
+          tag="${GITHUB_REF_NAME#v}"
+          pkg=$(python -c "import tomllib,pathlib; print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])")
+          if [ "$tag" != "$pkg" ]; then
+            echo "Release tag v$tag does not match pyproject.toml version $pkg" >&2
+            exit 1
+          fi
+
+      - name: Install build
+        run: python -m pip install --upgrade build
+
+      - name: Install package with dev extras (for tests)
+        run: python -m pip install -e ".[dev,agent,dashboard,semantic]"
+
+      - name: Run tests
+        run: python -m pytest -q
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/outmem
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

outmem-0.1.0/.gitignore

@@ -0,0 +1,67 @@
+# Virtualenvs
+.venv/
+venv/
+ENV/
+
+# Python build artefacts
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Python bytecode + extensions
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Tooling caches — each tool drops its own self-ignore inside the cache
+# dir, but only AFTER first run. Listing here is defensive.
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.pyright/
+.pytype/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.hypothesis/
+
+# Notebooks
+.ipynb_checkpoints/
+
+# Lockfiles — outmem is a library; consumers pick their own resolver.
+uv.lock
+poetry.lock
+Pipfile.lock
+
+# Pyenv pin — let contributors use any 3.12+ patch.
+.python-version
+
+# Secrets — never commit.
+.env
+.env.*
+.secrets.*
+
+# Outmem wiki state — only created if a contributor points
+# OUTMEM_PATH at this repo during dev.
+.outmem/
+.vectors.db
+
+# IDE / editor
+.idea/
+.vscode/
+.cursor/
+.zed/
+*.swp
+*.swo
+*~
+
+# Scratch
+tmp/
+
+# OS junk
+.DS_Store
+Thumbs.db
+Desktop.ini

outmem-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 phiweger
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

outmem-0.1.0/PKG-INFO

@@ -0,0 +1,458 @@
+Metadata-Version: 2.4
+Name: outmem
+Version: 0.1.0
+Summary: Agentic RAG memory: a git-versioned wiki an agent maintains for itself and a team.
+Project-URL: Repository, https://github.com/phiweger/outmem
+Author: phiweger
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent,git,memory,rag,wiki
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.12
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: skillfull>=0.2.0
+Provides-Extra: agent
+Requires-Dist: jinja2>=3.1; extra == 'agent'
+Requires-Dist: pydantic-ai>=0.1.0; extra == 'agent'
+Provides-Extra: all
+Requires-Dist: fastapi>=0.115; extra == 'all'
+Requires-Dist: jinja2>=3.1; extra == 'all'
+Requires-Dist: logfire>=3.0; extra == 'all'
+Requires-Dist: markdown-it-py>=3.0; extra == 'all'
+Requires-Dist: pydantic-ai>=0.1.0; extra == 'all'
+Requires-Dist: sqlite-vec>=0.1.7; extra == 'all'
+Requires-Dist: uvicorn[standard]>=0.30; extra == 'all'
+Provides-Extra: dashboard
+Requires-Dist: fastapi>=0.115; extra == 'dashboard'
+Requires-Dist: jinja2>=3.1; extra == 'dashboard'
+Requires-Dist: markdown-it-py>=3.0; extra == 'dashboard'
+Requires-Dist: uvicorn[standard]>=0.30; extra == 'dashboard'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
+Provides-Extra: evals
+Requires-Dist: jinja2>=3.1; extra == 'evals'
+Requires-Dist: pydantic-ai>=0.1.0; extra == 'evals'
+Requires-Dist: pytest>=8.3; extra == 'evals'
+Requires-Dist: sqlite-vec>=0.1.7; extra == 'evals'
+Provides-Extra: logfire
+Requires-Dist: logfire>=3.0; extra == 'logfire'
+Provides-Extra: pydantic-ai
+Requires-Dist: pydantic-ai>=0.1.0; extra == 'pydantic-ai'
+Provides-Extra: semantic
+Requires-Dist: pydantic-ai>=0.1.0; extra == 'semantic'
+Requires-Dist: sqlite-vec>=0.1.7; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# outmem
+
+**Agentic RAG memory over a git-versioned wiki.**
+
+A standalone Python library + CLI for maintaining a directory of
+plain-markdown notes that an LLM agent compiles, retrieves from, and
+writes back to. Retrieval is shell-tool based (`ripgrep`, `git log`,
+`cat`) — no vector index by default — and every agent turn is required
+to produce a git commit so identical future queries don't re-pay
+retrieval cost.
+
+This implements the pattern in [`specs/concept.md`](specs/concept.md):
+the LLM owns the wiki, the human curates sources and asks questions,
+the wiki is a compounding artifact that gets richer with every source
+ingested.
+
+---
+
+## Main idea
+
+outmem is a deliberate inversion of conventional RAG. Instead of
+pre-indexing raw sources into a vector store and reaching into them
+on every query, the agent **compiles** raw material into a wiki of
+small markdown pages, **retrieves** over the compiled material with
+shell tools (`ripgrep`, `git log`, `cat`), and is **required to
+commit** at the end of every turn — so identical future queries
+don't re-pay retrieval cost. The wiki compounds; the vector index
+that would otherwise grow stale doesn't exist.
+
+Three design choices anchor the rest of the system:
+
+1. **Compaction first.** The cheapest retrieval is reading a compiled
+   wiki page; raw sources are the fall-through, not the default. This
+   directly attacks the *relevance trap* — the gap between embedding
+   similarity and actual usefulness ([Raudaschl, "The Relevance
+   Trap"][rt]; [Fleck, "Divergence Engines"][de]).
+2. **Git as the substrate.** Every write produces a commit; `git log`
+   is both the audit trail and the agent's steering signal (recent
+   human commits become phase-1 planning context), and `git blame`
+   tracks line-level authorship. The same move Claude Code makes for
+   code search — agentic shell tools, no index — works here for
+   prose ([Nicolai, "Claude Code Doesn't Index Your
+   Codebase"][cc]; [SmartScope, "Settling the RAG Debate"][rd]).
+3. **Mandatory writeback.** Every agent turn ends with at least one
+   commit (`compact:` / `extend:` / `log:`), so the system records
+   not just what was retrieved but what it was retrieved *for*. The
+   pattern is closest to Karpathy's "LLM Wiki" sketch
+   ([gist][kw]). Adoption is measured via the **TARS** product
+   metric — Target / Adopted / Retained / Satisfied — rather than
+   recall@k or nDCG ([Raudaschl, "TARS"][tars]).
+
+A single divergence primitive ships in v0.1 — `topic_evolution`, a
+chronological `git log -p` over a topic — for the class of question
+convergent retrieval can't answer ("how has our thinking on X
+changed?"). The four other divergence primitives sketched in
+`concept.md` (contradiction surfacer, negative-space query,
+associative drift, cross-domain bridges) are queued for when the
+first one has earned its place.
+
+The full conceptual rationale and v0.1 implementation spec live in
+[`specs/concept.md`](specs/concept.md) and
+[`specs/spec.md`](specs/spec.md).
+
+[rt]: https://breakingproduct.substack.com/p/the-relevance-trap
+[tars]: https://uxdesign.cc/tars-a-product-metric-game-changer-c523f260306a
+[de]: https://medium.com/@j0lian/divergence-engines-escaping-the-relevance-trap-1bbdbee55ea6
+[cc]: https://vadim.blog/claude-code-no-indexing
+[rd]: https://smartscope.blog/en/ai-development/practices/rag-debate-agentic-search-code-exploration/
+[kw]: https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f
+
+---
+
+## Install
+
+```bash
+pip install outmem            # core: WikiStore + CLI
+pip install outmem[all]       # everything: agent runtime, semantic, dashboard, logfire
+```
+
+Or pick a subset:
+
+```bash
+pip install outmem[agent]      # + standalone PydanticAI agent runtime
+pip install outmem[semantic]   # + sqlite-vec index for find_similar
+pip install outmem[dashboard]  # + read-only FastAPI dashboard
+pip install outmem[logfire]    # + Pydantic Logfire instrumentation
+```
+
+System: Python 3.12+, `git`, and `ripgrep` (`rg`) on PATH. `outmem init`
+checks for these and refuses to proceed if either is missing.
+
+---
+
+## The 60-second mental model
+
+outmem maintains four directories under one wiki root:
+
+| Directory   | Tracked in git | Who writes | What lives there |
+|-------------|----------------|------------|------------------|
+| `raw/`      | no             | upstream ingestion pipeline | plain-text/markdown source material |
+| `wiki/`     | **yes**        | agent + humans (via Obsidian) | compiled knowledge, one concept per file, YAML frontmatter + `[[wikilinks]]` |
+| `log/`      | **yes**        | agent + humans | dated decision / observation trail |
+| `.outmem/`  | no (auto-gitignored) | outmem | non-git state (backlinks cache, last-run marker) |
+
+Plus two special wiki-root files:
+
+- **`wiki/AGENTS.md`** — user-editable conventions doc loaded into the
+  agent's system prompt every turn. Your customization layer for
+  domain, page structure, source-handling preferences.
+- **`wiki/index.md`** — auto-maintained slug list, regenerated on
+  every write.
+
+The agent's loop per turn:
+
+1. **Orient** — read recent human commits (steering signal); choose
+   *convergence* (look up a fact) or *expansion* (walk history).
+2. **Retrieve** — cheapest tool first: `rg wiki/`, then `rg raw/`,
+   then `git log -p --follow` for the expansion path.
+3. **Compact** — produce at least one commit before responding
+   (`compact:` for new pages, `extend:` for edits, `log:` for
+   observations). **Mandatory** — runs that skip it raise `WritebackError`.
+
+---
+
+## Quickstart
+
+```bash
+# Scaffold a fresh wiki.
+outmem init /srv/my-wiki
+export OUTMEM_PATH=/srv/my-wiki
+
+# Optional: tell the agent what this wiki is for.
+${EDITOR:-vi} /srv/my-wiki/wiki/AGENTS.md
+
+# Optional: ask the agent something (requires outmem[agent] + an API key).
+export OUTMEM_MODEL="anthropic:claude-sonnet-4-6"
+echo "ANTHROPIC_API_KEY=sk-ant-..." > .env
+outmem ask "what's our pricing policy?"
+```
+
+There's also a pre-populated example wiki at
+[`examples/starter-wiki/`](examples/starter-wiki/) if you want to
+poke at outmem before scaffolding your own.
+
+---
+
+## Common workflows
+
+### Ask the agent
+
+```bash
+outmem ask "what is our pricing formula and where does it come from?"
+```
+
+Searches the wiki first, falls back to `raw/` if needed, and produces
+at least one commit before responding — either extending a wiki page
+(`extend: <slug>`) or logging the observation (`log: <topic>`).
+
+### Ingest a source document
+
+```bash
+outmem ingest /path/to/some-paper.md \
+    --into research \
+    --prompt "extract methodology and headline results"
+```
+
+Copies the file under `wiki/sources/[<into>/]<sha256[:12]>/`,
+registers it in `wiki/sources/.sources.db`, then runs the agent to
+write/extend pages with `provenance:` pointing at the registered
+source. Parallel `outmem ingest` runs are safe (SQLite serialises
+writers). See [docs/cli.md](docs/cli.md#ingestion-requires-outmemagent)
+for `--register-only`, re-ingest semantics, and file-type rules.
+
+### Import an existing markdown vault (Obsidian, plain notes folder)
+
+```bash
+outmem import /path/to/obsidian-vault
+outmem lint                            # surface anything not auto-resolved
+```
+
+One-shot bulk import: walks the source for `*.md`, generates
+frontmatter, normalises slugs to outmem's flat namespace, rewrites
+wikilinks, and commits everything as `import: <vault-name>`. Hidden
+dirs (`.obsidian/`, `.git/`, …) are skipped. See
+[docs/cli.md](docs/cli.md#import-existing-markdown-vault) for
+collision handling and `--force` semantics.
+
+### Edit wiki files manually (Obsidian, vim, VS Code)
+
+Edit `wiki/*.md` however you like — outmem keeps the agent happy
+as long as you commit through git. Install the pre-commit hook once
+so the auto-maintained `wiki/index.md` and the semantic vector DB
+stay in lockstep with your edits:
+
+```bash
+outmem hook install
+```
+
+Without the hook, the explicit commands are `outmem index rebuild`
+and `outmem reindex`. See [docs/cli.md](docs/cli.md#sync-derived-artefacts-after-manual-edits).
+
+### Sync across machines
+
+```bash
+outmem pull
+outmem push
+```
+
+The agent does pull-rebase-push around every `outmem ask` by default
+(disable with `--no-pull` / `--no-push`). outmem's git operations
+are vanilla; any other tool (Obsidian Git plugin, GitHub Desktop,
+plain `git`) interoperates.
+
+---
+
+## Wiki page format
+
+Every `wiki/<slug>.md` opens with YAML frontmatter:
+
+```yaml
+---
+title: Pricing formula
+slug: pricing-formula
+provenance:
+  - path: raw/pricing-deck-2026-Q1.md
+    drive_path: /shared/Sales/2026-Q1-pricing-deck.pdf
+    sha256: 9e2c1f00aa
+created: 2026-01-15T10:30:00Z
+updated: 2026-05-04T11:32:00Z
+tags: [pricing, contracts, finance]
+---
+
+The 2026 pricing formula is **cost-plus 35%**…
+
+See also [[acme-msa]] for the Acme exception.
+```
+
+`provenance:` accepts plain path strings or dicts with richer
+ingestion metadata. There's no `authority` field — anyone (human or
+agent) may edit any page; "who wrote what" is reconstructed from
+`git log` / `git blame`. Wikilinks (`[[slug]]`) resolve to
+`wiki/<slug>.md`.
+
+Full schema: [docs/python-api.md](docs/python-api.md#writing--three-paths).
+
+---
+
+## Embed in your own PydanticAI agent
+
+If you want outmem as a *component* of a larger agent (virtual
+assistant, RAG pipeline, etc.) rather than as its own runtime,
+attach the tools + the same system prompt `outmem ask` uses to your
+own `pydantic_ai.Agent`:
+
+```python
+from pydantic_ai import Agent
+
+from outmem import WikiStore
+from outmem.adapters.pydantic_ai import wiki_tools
+from outmem.agent import render_system_prompt
+
+store = WikiStore.open("/path/to/wiki")
+
+agent = Agent(
+    "anthropic:claude-sonnet-4-6",
+    tools=wiki_tools(store),                   # 9 typed functions
+    system_prompt=render_system_prompt(store), # identical to outmem ask's
+)
+```
+
+`render_system_prompt(store)` returns the **exact same prompt string**
+the bundled `outmem ask` runtime sends — the three-phase framing
+(orient / retrieve / compact), recent human commits as steering
+signal, the bundled skill bodies (`search` / `evolution` / `write`),
+and your wiki's `AGENTS.md`. Tools + prompt → your agent has prompt-
+level parity with `outmem ask`. Pass `include_steering=False` if you
+don't want the phase-1 steering signal injected (handy for stateless
+assistant turns).
+
+**What you don't get in embed mode** (intentional — these are runtime
+concerns):
+
+- Mandatory writeback enforcement
+- Pull-before / push-after / record-run lifecycle
+- HITL approval gate around `write_page` / `extend_page`
+- Default `max_tokens=16384` and Anthropic prompt caching
+
+If you want those too, the all-in-one is `outmem.agent.ask_sync(store, query=…)`.
+
+**Fine-grained control** — if you want to compose the prompt yourself
+(e.g. prepend your own preamble, swap skill selection, omit AGENTS.md),
+the building blocks are all public:
+
+```python
+from outmem.adapters.pydantic_ai import skill_text, wiki_tools
+
+agents_md = store.read_agents_md() or ""
+agent = Agent(
+    "anthropic:claude-sonnet-4-6",
+    tools=wiki_tools(store),
+    system_prompt=(
+        "You are a helpful assistant.\n\n"
+        + skill_text("search")
+        + skill_text("write")
+        + (f"\n# Wiki conventions\n\n{agents_md}" if agents_md else "")
+    ),
+)
+```
+
+---
+
+## Read-only consult — wiki as a tool in someone else's agent
+
+When you've curated a wiki and want an *external* agent to consult it
+without ever modifying it, there's a one-call factory:
+
+```python
+from pydantic_ai import Agent
+from outmem.adapters.pydantic_ai import build_consult_wiki
+
+consult_wiki = build_consult_wiki("/srv/curated-wiki")
+
+my_assistant = Agent(
+    "anthropic:claude-sonnet-4-6",
+    tools=[consult_wiki],
+    system_prompt=(
+        "You're a helpful assistant. For questions about internal "
+        "policies, decisions, or customer history, call `consult_wiki`."
+    ),
+)
+result = my_assistant.run_sync("What's our pricing policy?")
+```
+
+`build_consult_wiki` opens the wiki via
+`WikiStore.open(path, read_only=True)` and builds an inner PydanticAI
+agent with the read-only tool palette (search / read / list / backlinks
+/ history / evolution / sources) plus a tight "cite by `[[slug]]`,
+explicitly say so if the wiki has nothing on the topic" system prompt
+and the same `max_tokens=16384` + Anthropic prompt-caching settings as
+the full `outmem ask` runtime. The outer agent gets a black-box
+`consult_wiki(question) -> str` tool — outmem-internal vocabulary
+never leaks across the boundary.
+
+What `read_only=True` guarantees:
+
+- Every commit-producing entry point on `WikiStore` (`write_page`,
+  `extend_page`, `append_log`, `add_source`, `record_ingestion`,
+  `rebuild_index`, `import_vault`) raises `OutmemError` via a single
+  guard in `_commit_paths`. `pull()` is also refused (rebase mutates
+  the working tree). `push()` stays unguarded — nothing local to push.
+- The read-tool palette doesn't even expose write tools (defense in
+  depth — the model never sees the write API).
+- `WikiStore.open(read_only=True)` skips the directory-creating layout
+  step, skips the stale `.git/index.lock` cleanup, and the backlinks
+  cache runs memo-only (no writes to `.outmem/`). The wiki's
+  filesystem state is left exactly as the caller found it, which makes
+  the mode safe to use on a literally read-only mount.
+
+For finer-grained control (custom system prompt, your own retry logic):
+
+```python
+from pydantic_ai import Agent
+from outmem import WikiStore
+from outmem.adapters.pydantic_ai import wiki_read_tools
+
+store = WikiStore.open("/srv/curated-wiki", read_only=True)
+agent = Agent(
+    "anthropic:claude-sonnet-4-6",
+    tools=wiki_read_tools(store),
+    system_prompt="You answer from the wiki only. Cite [[slugs]].",
+)
+```
+
+---
+
+## Where to go next
+
+- [`docs/cli.md`](docs/cli.md) — every subcommand with examples.
+- [`docs/python-api.md`](docs/python-api.md) — `WikiStore` + the
+  PydanticAI adapter + the standalone agent runtime.
+- [`docs/growing-the-wiki.md`](docs/growing-the-wiki.md) — reading
+  the log + lint signals to figure out what to ingest next.
+- [`docs/configuration.md`](docs/configuration.md) — `wiki/AGENTS.md`,
+  `config.yaml`, `.env`, environment variables, system requirements.
+- [`docs/features.md`](docs/features.md) — semantic index, write
+  approval, Logfire, dashboard (all opt-in).
+- [`docs/development.md`](docs/development.md) — dev install,
+  repository layout.
+- [`specs/concept.md`](specs/concept.md) — the original pattern this
+  implements.
+- [`specs/spec.md`](specs/spec.md) — v0.1 implementation spec.
+
+---
+
+## Status
+
+v0.1 + a v0.10-tagged batch of refinements (SQLite source registry,
+parallel-safe ingest, shared `_sqlite` / `_time` / `_logfire` helpers,
+`wiki/AGENTS.md` schema doc, `store.py` split into facets, README
+restructure). Tests + ruff + mypy strict clean.
+
+To give feedback, report at <https://github.com/phiweger/outmem/issues>.