matrx-rag 0.1.0__tar.gz

matrx_rag-0.1.0/.gitignore

@@ -0,0 +1,257 @@
+*.pyc
+secrets/
+ignore/
+temp/
+logs/
+todo
+text_notes/
+aidream/secrets/2.env
+automation_matrix/matrix_processing/temp/*
+cd
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+.venv/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+# The blanket lib/ rule above is from the standard Python .gitignore template
+# and was silently swallowing TS source under the SPA `src/lib/` folders.
+# Re-allow them explicitly so frontend builds don't ship without their lib layer.
+!dashboard/src/lib/
+!dashboard/src/lib/**
+!workflow-studio/src/lib/
+!workflow-studio/src/lib/**
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+ai/tests/clean_response.json
+ai/tests/cx_storage_response.json
+ai/tests/execution_test.py
+ai/tests/final_response.json
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env_remote
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.env.armanonly
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# random armani files
+/armani_dev/secrets/
+/armani/
+/_armani/
+
+
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.idea/
+.vscode/
+/node_modules/
+
+dump.rdb
+
+frontend/
+
+# AME Temp Files and directory structure
+# Ignore all files in the temp directory and its subdirectories
+/temp/**/*
+/tmp/**/*
+
+# Allow .gitkeep files to retain directory structure
+!/temp/**/.gitkeep
+!/tmp/**/.gitkeep
+
+# Armani
+.history*
+.history/
+local_data/
+local_reports_data/
+webscraper/quick_scrapes/temp/
+automation_matrix/ai_apis/fireworks/_dev/*
+automation_matrix/ai_apis/fireworks/_dev/fireworks_sample.py
+*.pdf
+*.flac
+*.mp3
+*.wav
+miniconda.sh
+/database/python_sql/temp_data/
+ .history*
+ .history/
+.history/
+
+_dev/
+/_dev/
+requirements_filtered.txt
+
+# matrx-dev-tools backups
+.env-backups/
+# Matrx Ship config (contains API key)
+.matrx-ship.json
+
+# Matrx config (contains API keys)
+.matrx.json
+.matrx-tools.conf
+
+# Claude Code local worktrees and per-user settings
+.claude/worktrees/
+.claude/settings.local.json
+
+# Append-only snapshots from matrx_utils.update_history (unbounded; do not commit)
+common/utils/data_in_code/data_history.json
+packages/matrx-utils/matrx_utils/data_in_code/data_history.json
+
+# Tool-dispatch debug logs — one file per server start, never committed
+.matrx-debug/
+
+# macOS Finder metadata
+.DS_Store
+**/.DS_Store
+
+# Environment files
+.env
+.env.*
+*.env
+*.env.*
+
+# Keep safe templates trackable
+!.env.example
+!.env.sample
+!.env.template

matrx_rag-0.1.0/CLAUDE.md

@@ -0,0 +1,136 @@
+# matrx-rag — agent guide
+
+Multi-tenant RAG extracted from `aidream/services/rag/`. Aidream is the
+reference host, but the package is **standalone-first**: it is installed
+directly for clients against a database with the EXACT schema the hosted
+product uses. **The shared frontend reads that database directly — one
+missing column or relationship and the install is useless.** Every design
+rule below serves that invariant.
+
+## Hard rules
+
+1. **No imports from `aidream.*`, ever.** Host capabilities arrive through
+   `matrx_rag.configure(...)` seams (document_writer, embedding_provider,
+   llm_provider, vector_store, ner_extractor, auth_mode, error classes).
+   Adding a `from aidream.*` import here is a regression.
+
+2. **This package owns the canonical rag.\* model classes.**
+   `matrx_rag/db/models_rag.py` (13 models) and the public.\* subset it reads
+   (`matrx_rag/db/models.py`) are GENERATED — by the host's
+   `python db/generate.py` via the `output:` entries in aidream's
+   `db/matrx_orm.yaml`. **Never hand-edit generated files**; change the live
+   schema, then regenerate. aidream imports these classes directly
+   (`from matrx_rag.db.models_rag import ...`) — it keeps NO copy and NO shim.
+
+3. **One pool, decided by binding — the package NEVER opens its own
+   connection while hosted.** Models bake `_database = "matrx_rag"`.
+   - Hosted: `configure(db_config_name=...)` → `bind_to_host()` registers a
+     matrx-orm **name alias** onto the host's already-registered pool and
+     registers ONLY the rag.\* models (public.\* names resolve to the HOST's
+     classes via `matrx_orm.model_registry` — host always wins).
+   - Standalone: `matrx_rag.bootstrap_db()` registers the pool from
+     `MATRX_RAG_POSTGRES_*` env vars and registers BOTH model files.
+   - `configure(db_models=...)` is an OVERRIDE for a genuinely divergent
+     host DB — normal hosts must not pass it.
+
+4. **DB access goes through `get_db_model(name)`** (`matrx_rag._ext`) —
+   resolution: injected override → `model_registry`. It always resolves once
+   configure/bootstrap ran; raising on an unconfigured app is correct.
+   `try_get_db_model` returning None means "not configured", not
+   "standalone" — the old raw-asyncpg fallback forks are gone; do not
+   reintroduce them.
+
+5. **Raw SQL is allowed ONLY at the documented exception sites**, each marked
+   `# RAW-SQL EXCEPTION: <reason>` in code and ratcheted by
+   `tests/test_no_new_raw_rag_sql.py` (lower the baseline in the same change:
+   `REWRITE_RAW_RAG_SQL_BASELINE=1 uv run pytest ...`). The permanent set:
+   - `ingestion.upsert_chunks` / `stages.run_chunk` /
+     `extraction_indexing._upsert_derivative_chunks`: advisory-lock
+     transactions — every statement must share the lock-holding connection;
+     bulk primitives lack RETURNING ids; the columnar unnest write is
+     load-bearing (1.6k-chunk timeout incident).
+   - `search._build_visibility_clause` (8-branch OR + correlated EXISTS into
+     public ACL tables), `_vector_recall` (dynamic embedding table JOIN +
+     pgvector `<=>`), `_lexical_recall` (FTS — no ORM primitive).
+   - `ner._bump_confidence_avg` (correlated UPDATE...FROM avg — atomicity).
+   - `awareness.py` aggregates; public.\* host-table reads in
+     sources/stages/extraction_indexing/pdf_pipeline/library/data_stores
+     label-enrich (future DocumentWriter/label-resolver seam work).
+   - `db.apply_migration_file` DDL.
+   Pushing below this needs matrx-orm primitives (bulk RETURNING,
+   connection-scoped QueryBuilder writes, FTS, raw-WHERE fragments) — they
+   are tracked as feature requests, don't hand-roll around their absence.
+
+6. **Migrations are owned by this package** (`matrx_rag/migrations/`) and
+   must reproduce the live rag schema EXACTLY (frontend-identical
+   invariant). Idempotent: every CREATE has IF NOT EXISTS, every policy
+   DROPs first, FKs into public.\* are catalog-guarded (`to_regclass`).
+   They are mirrored in aidream's `db/migrations/` — edit BOTH until the
+   package has its own runner. A standalone install targets a full
+   Matrx-schema database (the frontend needs all tables); this package
+   ships migrations for rag.\* only, models that *work against* public.\*,
+   never migrations that create public tables.
+
+7. **Aidream sees this package via a re-export shim** at
+   `aidream/services/rag/` (models are imported directly from
+   `matrx_rag.db.models_rag` — no model shim). Every public symbol must stay
+   importable through that path — verified by `aidream/api/tests/`.
+
+8. **Behavior parity is the contract.** Conversions must preserve exact
+   semantics: `IS NOT DISTINCT FROM` on nullable params → branch
+   `field=value` / `field__isnull=True`; never drop organization_id/owner_id
+   scoping; server-side `now()` → Python UTC only where the file already
+   does so.
+
+## What goes where
+
+- `matrx_rag/db/` — binding (`bootstrap_db`, `bind_to_host`, `get_pool`,
+  `PACKAGE_DB_NAME`) + generated models/managers. `get_models()` returns the
+  full name→class map.
+- `matrx_rag/ingestion.py` — `ingest_source`, `upsert_chunks`,
+  `materialize_code_graph`. The high-level entry the host calls.
+- `matrx_rag/chunking.py` — recursive text + line-window code chunker.
+- `matrx_rag/code_chunker.py` — tree-sitter symbol chunker.
+- `matrx_rag/embeddings.py` — OpenAI + Voyage adapters with batching.
+- `matrx_rag/search.py` — hybrid retrieval, RRF fusion, MMR, rerank,
+  priority-aware ranking.
+- `matrx_rag/pdf_pipeline.py` — page-aware extraction + OCR + LLM cleanup.
+- `matrx_rag/image_pipeline.py` — vision-LLM captions → chunks.
+- `matrx_rag/repo_ingest.py` — git repo walker.
+- `matrx_rag/library.py` — library document workflow.
+- `matrx_rag/data_stores.py` — curated content buckets.
+- `matrx_rag/grounding.py` — answer-faithfulness verifier.
+- `matrx_rag/stages.py` — resumable multi-step orchestrator.
+- `matrx_rag/sources.py` — `resolve(source_kind, source_id)` adapter
+  (note / code_file / cld_file / library_doc / transcript / scraped /
+  repository / task / project).
+- `matrx_rag/ner.py` — NER → kg_entities widening. `extract_entities(...)`
+  dedups typed entities by `(org, kind, lower(name))`, writes
+  `kg_chunk_entities` mentions + spans, emits `co_occurs_with` edges,
+  maintains `kg_entities.confidence_avg`. The extractor LLM call is
+  HOST-INJECTED (`configure(ner_extractor=...)`); no extractor ⇒ clean no-op.
+- `matrx_rag/awareness.py` — agent RAG-awareness fragments.
+- `matrx_rag/llm.py` — cleanup + contextualize prompts.
+- `matrx_rag/processed_doc_lookup.py` — processed-document accessors.
+- `matrx_rag/progress.py` — `ProgressEmitter` wrapper.
+- `matrx_rag/extraction_indexing.py` — agent-extraction → RAG bridge.
+- `matrx_rag/eval/` — retrieval + answer quality eval harness.
+
+## Testing
+
+```bash
+uv run pytest packages/matrx-rag/tests        # must pass with no aidream on the path
+```
+
+`test_no_new_raw_rag_sql.py` is the raw-SQL ratchet — it fails on ANY change
+to the per-file counts (up = regression, down = lower the baseline in the
+same change).
+
+## When NOT to use this package
+
+- aidream code that ONLY reads `processed_documents` / `processed_document_pages`
+  (those tables belong to `aidream.services.documents`, not RAG)
+- File metadata / storage (that's matrx-utils)
+- Anything that doesn't involve chunking + embedding + retrieval
+
+— Verified against live code + Supabase project txzxabzwovsujtloxrus, 2026-06-09.

matrx_rag-0.1.0/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: matrx-rag
+Version: 0.1.0
+Summary: Multi-tenant RAG: hybrid retrieval, chunking, embeddings, ingestion, PDF + image + repo pipelines, agent-extraction indexing, priority-aware ranking.
+Author-email: Matrx <admin@aimatrx.com>
+License: MIT
+Keywords: embeddings,matrx,pgvector,rag,retrieval,supabase
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.13
+Requires-Dist: asyncpg>=0.30
+Requires-Dist: cohere>=5.0
+Requires-Dist: langchain-text-splitters>=0.3
+Requires-Dist: matrx-orm>=3.1
+Requires-Dist: matrx-utils
+Requires-Dist: openai>=2.0
+Requires-Dist: pdfplumber>=0.11
+Requires-Dist: pydantic>=2.12
+Requires-Dist: tiktoken>=0.8
+Requires-Dist: voyageai>=0.3
+Provides-Extra: api
+Requires-Dist: fastapi>=0.115; extra == 'api'
+Requires-Dist: matrx-connect; extra == 'api'
+Provides-Extra: dev
+Requires-Dist: fastapi>=0.115; extra == 'dev'
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: host
+Requires-Dist: matrx-connect; extra == 'host'
+Description-Content-Type: text/markdown
+
+# matrx-rag
+
+Multi-tenant RAG: hybrid retrieval, chunking, embeddings, ingestion,
+PDF + image + repo pipelines, agent-extraction indexing, NER/KG writes,
+priority-aware ranking — extracted from `aidream/services/rag/` so any
+host (aidream's cloud server, matrx-local) can consume the same stack.
+
+**Documentation hub:** [`docs/rag_and_ner/README.md`](../../docs/rag_and_ner/README.md)  
+(Vision in `docs/knowledge/` · code truth in `docs/rag_and_ner/reality/` · backlog in `00_CLEANUP.md`.)
+
+## What it does
+
+- **Ingestion**: `ingest_source(source_kind, source_id, ...)` — chunk → embed → upsert; NER inline for non-code sources.
+- **PDF / image / code repo** pipelines with provenance.
+- **Hybrid search**: pgvector HNSW + lexical FTS → RRF → optional Cohere → MMR.
+- **Priority-aware ranking**: agent-extracted chunks with non-zero `priority`.
+- **Agent-extraction → RAG**: `extraction_indexing` materializes page-extraction payloads as chunks.
+- **Data stores**: curated buckets with scoped search.
+- **Eval harness**: retrieval + answer quality assessment.
+
+## Host integration
+
+- **aidream** wires the package in `aidream/package_integration.py::_configure_matrx_rag` (NER extractor, embedding cache, ORM models, search callable).
+- **aidream/services/rag/** is a re-export shim; HTTP routes live in `aidream/api/routers/rag.py`.
+- **Auto-ingest** (gates, budget, scope suggestions) lives in `aidream/services/auto_ingest/` — not in this package.
+
+## Schema
+
+Migrations: `matrx_rag/migrations/` (mirrored in `db/migrations/` until package owns the runner).  
+ORM / pgvector notes: [`docs/rag_and_ner/reality/04_ORM_AND_SCHEMA.md`](../../docs/rag_and_ner/reality/04_ORM_AND_SCHEMA.md).
+
+## Package development
+
+See [`CLAUDE.md`](CLAUDE.md) for injection seams, test baselines, and the no-`aidream`-imports rule.

matrx_rag-0.1.0/README.md

@@ -0,0 +1,34 @@
+# matrx-rag
+
+Multi-tenant RAG: hybrid retrieval, chunking, embeddings, ingestion,
+PDF + image + repo pipelines, agent-extraction indexing, NER/KG writes,
+priority-aware ranking — extracted from `aidream/services/rag/` so any
+host (aidream's cloud server, matrx-local) can consume the same stack.
+
+**Documentation hub:** [`docs/rag_and_ner/README.md`](../../docs/rag_and_ner/README.md)  
+(Vision in `docs/knowledge/` · code truth in `docs/rag_and_ner/reality/` · backlog in `00_CLEANUP.md`.)
+
+## What it does
+
+- **Ingestion**: `ingest_source(source_kind, source_id, ...)` — chunk → embed → upsert; NER inline for non-code sources.
+- **PDF / image / code repo** pipelines with provenance.
+- **Hybrid search**: pgvector HNSW + lexical FTS → RRF → optional Cohere → MMR.
+- **Priority-aware ranking**: agent-extracted chunks with non-zero `priority`.
+- **Agent-extraction → RAG**: `extraction_indexing` materializes page-extraction payloads as chunks.
+- **Data stores**: curated buckets with scoped search.
+- **Eval harness**: retrieval + answer quality assessment.
+
+## Host integration
+
+- **aidream** wires the package in `aidream/package_integration.py::_configure_matrx_rag` (NER extractor, embedding cache, ORM models, search callable).
+- **aidream/services/rag/** is a re-export shim; HTTP routes live in `aidream/api/routers/rag.py`.
+- **Auto-ingest** (gates, budget, scope suggestions) lives in `aidream/services/auto_ingest/` — not in this package.
+
+## Schema
+
+Migrations: `matrx_rag/migrations/` (mirrored in `db/migrations/` until package owns the runner).  
+ORM / pgvector notes: [`docs/rag_and_ner/reality/04_ORM_AND_SCHEMA.md`](../../docs/rag_and_ner/reality/04_ORM_AND_SCHEMA.md).
+
+## Package development
+
+See [`CLAUDE.md`](CLAUDE.md) for injection seams, test baselines, and the no-`aidream`-imports rule.

matrx_rag-0.1.0/matrx_rag/__init__.py

@@ -0,0 +1,115 @@
+"""matrx-rag — multi-tenant RAG for the matrx ecosystem.
+
+Hybrid retrieval (vector + lexical), chunking, embeddings, ingestion,
+PDF + image + repo pipelines, agent-extraction indexing, priority-
+aware ranking. Extracted from ``aidream/services/rag/`` in Phase 2a so
+matrx-local and aidream can share the same code with host-specific
+provider injection.
+
+Public API:
+
+* :func:`configure` — wire host-supplied seams (call at startup).
+
+* :class:`AuthMode` — enum for DB-auth posture (service_role / user_jwt / none).
+
+* :class:`DocumentWriter` — protocol for hosts that own the
+  processed_documents schema (aidream). matrx-local uses the no-op
+  default for local-RAG mode.
+
+* The submodules continue to expose their original public surface
+  (``matrx_rag.embeddings.embed``, ``matrx_rag.search.search``,
+  ``matrx_rag.ingestion.ingest_source`` etc.). aidream's existing
+  imports keep working via the shim at ``aidream/services/rag/``.
+
+Injection seams (all five live as of Phase 2b/iv):
+
+* ``DocumentWriter`` — processed_documents persistence (2b/i).
+* ``EmbeddingProvider`` — local vs cloud embeddings (2b/ii).
+* ``LLMProvider`` — cleanup + contextualization model routing (2b/iii).
+* ``VectorStore`` — pgvector vs sqlite-vss vs duckdb (2b/iv).
+* ``AuthMode`` — service-role vs user-JWT (2b/i).
+
+Call-site migration of the existing inline asyncpg paths to the
+``VectorStore`` seam is incremental and lands in Phase 2b/iv follow-on
+commits.
+"""
+
+from __future__ import annotations
+
+from matrx_rag._ext import (
+    AuthMode,
+    DocumentWriter,
+    EmbeddingProvider,
+    LLMProvider,
+    NoOpDocumentWriter,
+    RagNotConfiguredError,
+    VectorStore,
+    configure,
+    get_auth_mode,
+    get_db_config_name,
+    get_db_model,
+    get_db_models,
+    get_document_writer,
+    get_embedding_provider,
+    get_ext,
+    get_llm_provider,
+    get_ner_extractor,
+    get_ner_finisher,
+    get_vector_store,
+    is_configured,
+    try_get_db_model,
+)
+from matrx_rag.db import (
+    PACKAGE_DB_NAME,
+    bind_to_host,
+    bootstrap_db,
+    get_models,
+)
+from matrx_rag._observability import (
+    AICallTally,
+    announce_chat_call,
+    announce_embedding_call,
+    announce_extraction_call,
+    print_tally_summary,
+    tally_reset,
+    tally_snapshot,
+)
+
+__version__ = "0.1.0"
+
+
+__all__ = [
+    "__version__",
+    "AuthMode",
+    "DocumentWriter",
+    "EmbeddingProvider",
+    "LLMProvider",
+    "VectorStore",
+    "NoOpDocumentWriter",
+    "RagNotConfiguredError",
+    "configure",
+    "is_configured",
+    "get_ext",
+    "get_document_writer",
+    "get_embedding_provider",
+    "get_llm_provider",
+    "get_ner_extractor",
+    "get_ner_finisher",
+    "get_vector_store",
+    "get_auth_mode",
+    "get_db_config_name",
+    "get_db_model",
+    "get_db_models",
+    "try_get_db_model",
+    "PACKAGE_DB_NAME",
+    "bind_to_host",
+    "bootstrap_db",
+    "get_models",
+    "AICallTally",
+    "announce_chat_call",
+    "announce_embedding_call",
+    "announce_extraction_call",
+    "print_tally_summary",
+    "tally_reset",
+    "tally_snapshot",
+]