docsmind 0.1.0__tar.gz → 0.2.0__tar.gz

docsmind-0.2.0/.env.example

@@ -0,0 +1,29 @@
+# Copy to .env and fill in. All vars are prefixed DOCSMIND_ (see config.py).
+
+# --- storage: pick ONE path below ---
+
+# Path A: local Docker (docsmind db up)
+DOCSMIND_DATABASE_URL=postgresql+asyncpg://docsmind:docsmind@localhost:5432/docsmind
+
+# Path B: Supabase (Storage bucket doubles as blob store)
+# DOCSMIND_DATABASE_URL=postgresql+asyncpg://postgres:<password>@db.<project>.supabase.co:5432/postgres
+# DOCSMIND_BLOB_BACKEND=supabase
+# DOCSMIND_SUPABASE_URL=https://<project>.supabase.co
+# DOCSMIND_SUPABASE_KEY=<service_role_key>
+
+# Path C: Neon (pair with an S3-compatible bucket, e.g. Cloudflare R2)
+# DOCSMIND_DATABASE_URL=postgresql+asyncpg://<user>:<password>@<project>.neon.tech/docsmind
+# DOCSMIND_BLOB_BACKEND=s3
+# DOCSMIND_S3_BUCKET=docsmind-images
+# DOCSMIND_S3_ENDPOINT_URL=https://<account_id>.r2.cloudflarestorage.com
+
+# --- models ---
+DOCSMIND_LLM_PROVIDER=anthropic
+DOCSMIND_LLM_MODEL=claude-sonnet-4-6
+DOCSMIND_EMBEDDING_PROVIDER=openai
+DOCSMIND_EMBEDDING_MODEL=text-embedding-3-large
+# DOCSMIND_EMBEDDING_DIMENSIONS=3072   # native size; shorten via API's `dimensions` param if needed
+DOCSMIND_VISION_PROVIDER=anthropic
+
+DOCSMIND_ANTHROPIC_API_KEY=
+DOCSMIND_OPENAI_API_KEY=

docsmind-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,49 @@
+name: publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      test_pypi:
+        description: "Publish to TestPyPI instead of PyPI"
+        type: boolean
+        default: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-testpypi:
+    if: github.event_name == 'workflow_dispatch' && inputs.test_pypi
+    needs: build
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write # trusted publishing — no API token stored in secrets
+    steps:
+      - uses: actions/download-artifact@v4
+        with: { name: dist, path: dist }
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  publish-pypi:
+    if: github.event_name == 'release'
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with: { name: dist, path: dist }
+      - uses: pypa/gh-action-pypi-publish@release/v1

docsmind-0.2.0/.github/workflows/test.yml

@@ -0,0 +1,59 @@
+name: test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.10 is our declared floor (see pyproject.toml `requires-python`);
+        # 3.12 is the default dev version (see .python-version). Test both
+        # ends plus the one in between so a dependency silently requiring a
+        # newer interpreter shows up here instead of in someone's install.
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-groups --all-extras --python ${{ matrix.python-version }}
+
+      - name: Lint (ruff)
+        run: uv run ruff check .
+
+      - name: Type check (mypy)
+        run: uv run mypy src/docsmind
+        continue-on-error: true # tighten once the core modules are fleshed out
+
+      - name: Test
+        # testcontainers[postgres] pulls pgvector/pgvector and manages its own
+        # container via the Docker socket, which GitHub's ubuntu runners expose
+        # by default — no separate `services:` postgres block needed.
+        run: uv run pytest -v
+
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - name: Build wheel and sdist
+        run: uv build
+      - name: Check wheel installs cleanly (core deps only, no extras)
+        run: |
+          uv venv /tmp/check-venv
+          uv pip install --python /tmp/check-venv/bin/python dist/*.whl
+          /tmp/check-venv/bin/python -c "import docsmind; print(docsmind.__version__)"

docsmind-0.2.0/.gitignore

@@ -0,0 +1,63 @@
+# --- Python ---
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+.eggs/
+*.egg
+
+# --- Virtual environments ---
+.venv/
+venv/
+env/
+ENV/
+
+# --- Testing / linting / type-checking ---
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+
+# --- uv / pip ---
+uv.lock.bak
+
+# --- docsmind local runtime artifacts ---
+# Local filesystem BlobStore (extracted images from ingestion) — never
+# something to commit; it's per-machine, per-ingest state.
+.docsmind/
+# Generated by scripts/make_test_pdf.py for local testing — not a committed
+# fixture (tests/unit/test_pdf_parser.py generates its own PDF in-memory).
+/sample.pdf
+
+# --- Environment / secrets ---
+.env
+.env.*
+!.env.example
+
+# --- Postgres (local Docker volume mounts, if you ever add bind mounts) ---
+*.sql.bak
+
+# --- Editors / OS ---
+.vscode/
+.idea/
+*.swp
+.DS_Store
+Thumbs.db
+
+# --- Jupyter, if you use it for exploration ---
+.ipynb_checkpoints/
+
+# --- Alembic ---
+# (versions/ IS tracked — only local scratch files some editors drop here)
+alembic/versions/*.pyc
+
+# --- ui/ (Vite/React build output, once ui/ has a real app in it) ---
+ui/node_modules/
+ui/dist/

docsmind-0.2.0/.python-version

@@ -0,0 +1 @@
+3.12

docsmind-0.2.0/CHANGELOG.md

@@ -0,0 +1,84 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.2.0] - 2026-09-27
+
+First functional release: documents can now be ingested from PDF into
+Postgres/pgvector and inspected from the command line. Chat, embeddings and
+change analysis are scaffolded but not implemented yet (see "Not yet
+implemented" below).
+
+### Added
+
+- **PDF ingestion pipeline** (`docsmind ingest <file> --collection <name>`),
+  built on PyMuPDF:
+  - Full document text is extracted and saved per version, together with
+    stats (character, word and estimated token counts).
+  - Outline is taken from the PDF's own table of contents/bookmarks when one
+    exists. Documents without a TOC get an empty outline; structure is
+    deliberately not guessed from fonts or numbering.
+  - Text is chunked by page for vector search: one chunk per page, with
+    unusually dense pages split into ~3,500-character pieces with a
+    100-character overlap.
+  - Images are extracted with their page and bounding box, skipping tiny
+    icons, and stored with a perceptual hash for future de-duplication. They
+    are saved under a readable path
+    (`images/<collection>/<title>/<version>-<id>/`) instead of a bare UUID.
+- **Postgres storage** with pgvector: Alembic migrations for documents,
+  versions, sections, images, chunks (with a generated `tsvector` column for
+  keyword search) and editable agent tables with revision history. The vector
+  index uses `halfvec` so 3072-dimension embeddings
+  (`text-embedding-3-large`) fit within pgvector's HNSW limit.
+- **Inspection commands**: `docsmind versions`, `docsmind fulltext`,
+  `docsmind outline` and `docsmind chunks` read back what was saved, so
+  results can be checked without writing SQL.
+- **Database commands**: `docsmind db up` (local Docker Postgres with
+  pgvector), `docsmind db upgrade` (apply migrations) and `docsmind db psql`.
+- `docsmind doctor` checks the database URL, blob storage and API keys, and
+  detects Neon/Supabase connection strings.
+- `Docsmind` / `AsyncDocsmind` Python facade for ingesting documents.
+- Domain models for documents, versions, sections, chunks, images, citations,
+  change sets and editable tables, with `Protocol` ports for LLMs, embedders,
+  vision captioners, vector stores, blob stores and parsers.
+- Environment-driven configuration (`DOCSMIND_*` variables, `.env` support)
+  and a documented `.env.example`.
+- Filesystem blob storage for extracted images.
+- GitHub Actions: a test workflow across Python 3.10, 3.11 and 3.12 with a
+  wheel install check, and a publish workflow using PyPI trusted publishing.
+- Unit tests for the PDF parser and page chunker, a runnable parser demo
+  (`examples/parse_pdf_demo.py`) and a synthetic test-PDF generator.
+- `.gitignore`, `docker-compose.yml` and `.python-version`.
+
+### Changed
+
+- `--version` on `docsmind ingest` is now optional and defaults to today's
+  date.
+- The local Docker Postgres listens on port 5433 instead of 5432 to avoid
+  clashing with a locally installed Postgres.
+- Package metadata: author email, explicit Python 3.10-3.12 classifiers and
+  the issue tracker URL.
+
+### Removed
+
+- Heading detection from font size, boldness and numbering patterns, and
+  section-based chunking. Neither generalised across real-world PDFs, and
+  chunking no longer depends on document structure.
+
+### Not yet implemented
+
+- Embeddings and hybrid vector/keyword search. Chunks are stored without
+  embeddings, and `hybrid_search` raises `NotImplementedError`.
+- Image captioning with a vision model.
+- The chat agent behind `docsmind chat` and the change-analysis subagent
+  behind `docsmind diff`. Both commands exist but have no implementation
+  behind them yet.
+- DOCX and plain-text parsers, S3/Supabase blob storage, and the server/UI.
+- Multi-column reading order: PDFs with genuine multi-column layouts may have
+  their text interleaved in the saved full text.
+
+## [0.1.0] - 2026-09-12
+
+### Added
+
+- Initial release to reserve the package name on PyPI.

docsmind-0.2.0/PKG-INFO

@@ -0,0 +1,274 @@
+Metadata-Version: 2.5
+Name: docsmind
+Version: 0.2.0
+Summary: Chat with your documents. Version-aware retrieval, change analysis, and an editable table workspace, backed by Postgres/pgvector.
+Project-URL: Homepage, https://github.com/yauheniya-ai/docsmind
+Project-URL: Repository, https://github.com/yauheniya-ai/docsmind
+Project-URL: Issues, https://github.com/yauheniya-ai/docsmind/issues
+Author-email: Yauheniya Varabyova <yauheniya.ai@gmail.com>
+License: MIT
+Keywords: agent,compliance,documents,pgvector,rag
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.10
+Requires-Dist: alembic>=1.13
+Requires-Dist: asyncpg>=0.29
+Requires-Dist: httpx>=0.27
+Requires-Dist: imagehash>=4.3
+Requires-Dist: pgvector>=0.2.5
+Requires-Dist: pillow>=10.0
+Requires-Dist: pydantic-settings>=2.2
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pymupdf>=1.24
+Requires-Dist: rich>=13.7
+Requires-Dist: sqlalchemy[asyncio]>=2.0
+Requires-Dist: typer>=0.12
+Provides-Extra: all
+Requires-Dist: anthropic>=0.34; extra == 'all'
+Requires-Dist: fastapi>=0.111; extra == 'all'
+Requires-Dist: openai>=1.30; extra == 'all'
+Requires-Dist: uvicorn[standard]>=0.30; extra == 'all'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.34; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Requires-Dist: testcontainers[postgres]>=4.4; extra == 'dev'
+Provides-Extra: mlflow
+Requires-Dist: mlflow>=2.14; extra == 'mlflow'
+Provides-Extra: openai
+Requires-Dist: openai>=1.30; extra == 'openai'
+Provides-Extra: server
+Requires-Dist: fastapi>=0.111; extra == 'server'
+Requires-Dist: uvicorn[standard]>=0.30; extra == 'server'
+Description-Content-Type: text/markdown
+
+# docsmind
+
+[![PyPI version](https://img.shields.io/pypi/v/docsmind.svg)](https://pypi.org/project/docsmind/)
+[![Python versions](https://img.shields.io/pypi/pyversions/docsmind.svg)](https://pypi.org/project/docsmind/)
+[![Downloads](https://static.pepy.tech/badge/docsmind)](https://pepy.tech/project/docsmind)
+[![Downloads / month](https://static.pepy.tech/badge/docsmind/month)](https://pepy.tech/project/docsmind)
+[![Tests](https://github.com/yauheniya-ai/docsmind/actions/workflows/test.yml/badge.svg)](https://github.com/yauheniya-ai/docsmind/actions/workflows/test.yml)
+[![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/yauheniya-ai/<GIST_ID>/raw/docsmind-coverage.json)](https://github.com/yauheniya-ai/docsmind/actions/workflows/test.yml)
+[![License: MIT](https://img.shields.io/pypi/l/docsmind.svg)](https://github.com/yauheniya-ai/docsmind/blob/main/LICENSE)
+[![Code style: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Status: alpha](https://img.shields.io/badge/status-alpha-orange.svg)](#status)
+
+Chat with your documents. Track what changed between versions, and why it matters.
+
+`docsmind` ingests PDFs (and soon DOCX/text) into a version-aware Postgres/pgvector
+store, and gives you an agent, **the Mind**, to chat with them. A built-in subagent,
+**diffra**, will do change analysis between document versions: what changed, where, and
+its compliance implications, with citations back to exact pages.
+
+> **Alpha.** Ingestion and inspection work today. Embeddings, chat and change analysis
+> are designed and scaffolded but not implemented yet; see [What works today](#what-works-today).
+
+## What works today
+
+| | |
+|---|---|
+| ✅ PDF ingestion | Full text, stats, outline (from the PDF's own table of contents), page-based chunks, images |
+| ✅ Postgres + pgvector storage | Local Docker, or any Postgres with pgvector (Neon, Supabase) |
+| ✅ Inspection CLI | `versions`, `fulltext`, `outline`, `chunks` read back exactly what was saved |
+| ✅ `docsmind doctor` | Checks database, blob storage and API keys, and tells you what to fix |
+| 🚧 Embeddings + hybrid search | Chunks are stored without embeddings for now |
+| 🚧 Image captioning | Images are extracted and stored; vision captioning is next |
+| 🚧 Chat agent (the Mind) | `docsmind chat` exists as a command, without an agent behind it yet |
+| 🚧 Change analysis (diffra) | `docsmind diff` exists as a command, without an implementation yet |
+| 🚧 DOCX/text parsers, S3/Supabase blob storage, server + UI | Planned |
+
+## Install
+
+```bash
+pip install docsmind
+# or, with optional extras:
+pip install "docsmind[all]"
+```
+
+Extras: `docsmind[openai]`, `docsmind[anthropic]`, `docsmind[server]` (FastAPI, planned UI),
+`docsmind[mlflow]` (tracing/eval export). Requires Python 3.10+.
+
+## Quickstart
+
+### 1. Get a Postgres with pgvector
+
+Pick one:
+
+**A. Local Docker (recommended for trying it out)**
+```bash
+docsmind db up         # docker compose up -d, pgvector/pgvector image on port 5433
+docsmind db upgrade    # applies migrations, creates the vector extension
+```
+
+**B. Neon or Supabase** (no Docker needed; both support pgvector)
+```bash
+# in .env:
+# DOCSMIND_DATABASE_URL=postgresql+asyncpg://<user>:<password>@<host>/<db>
+docsmind db upgrade
+```
+Extracted images are saved to local disk (`DOCSMIND_BLOB_ROOT`) for now. S3 and
+Supabase Storage backends are planned.
+
+Then check that everything is wired correctly:
+
+```bash
+docsmind doctor
+```
+
+Configuration is environment-driven (`DOCSMIND_*`, or a `.env` file). Copy
+`.env.example` to `.env` to see every option. Model API keys aren't needed yet;
+they'll be required once embeddings and chat land.
+
+### 2. Ingest a PDF
+
+```bash
+docsmind ingest report.pdf --collection policies
+docsmind ingest report_v2.pdf --collection policies --version 2025   # version label is optional
+```
+
+or from Python:
+
+```python
+from docsmind import Docsmind
+
+dm = Docsmind()
+version = dm.ingest("report.pdf", collection="policies")   # version defaults to today's date
+print(version.id)
+```
+
+### 3. Look at what was saved
+
+```bash
+docsmind versions                       # every ingested version, with its id, word count and chunk count
+docsmind fulltext <version-id>          # saved full text (--full for everything)
+docsmind outline <version-id>           # outline from the PDF's table of contents
+docsmind chunks <version-id> --page 12  # the chunk(s) saved for one page
+```
+
+You can also inspect the tables directly with `docsmind db psql`.
+
+### 4. Coming next
+
+```python
+print(dm.chat("What are our data retention obligations?"))   # planned
+change_set = dm.compare(v1.id, v2.id)                        # planned (diffra)
+```
+
+## How ingestion works
+
+Ingestion is deliberately simple and predictable. There's no layout guessing, because
+heuristics that work on one PDF break on the next.
+
+- **Full text** is extracted for the whole document and saved as-is, with character,
+  word and estimated token counts.
+- **Outline** comes only from the PDF's own table of contents/bookmarks. If the PDF
+  has none, the outline is empty. Structure is never guessed from fonts or numbering.
+- **Chunks** are made per page for vector search: most pages (~3,000 characters) become
+  one chunk, and unusually dense pages are split into ~3,500-character pieces with a
+  100-character overlap. Chunks exist to find relevant passages by meaning, not to
+  reconstruct structure.
+- **Images** are extracted with their page and position, skipping tiny icons, and saved
+  under a readable path: `images/<collection>/<title>/<version>-<id>/`.
+
+## CLI reference
+
+| Command | What it does |
+|---|---|
+| `docsmind ingest <file> -c <collection> [-v <label>]` | Parse a PDF and save it |
+| `docsmind versions` | List ingested versions with ids and stats |
+| `docsmind fulltext <id> [--full]` | Print the saved full text |
+| `docsmind outline <id>` | Print the saved outline |
+| `docsmind chunks <id> [--page N] [--full]` | Print saved chunks |
+| `docsmind db up` / `upgrade` / `psql` | Start local Postgres, apply migrations, open psql |
+| `docsmind doctor` | Check your setup |
+| `docsmind chat`, `docsmind diff` | Planned |
+
+## Architecture
+
+```
+src/docsmind/
+├── __init__.py            # exports Docsmind + key models only
+├── client.py              # Docsmind facade (sync + async)
+├── config.py              # pydantic-settings, env-driven
+│
+├── domain/                # pure data + interfaces, no I/O
+│   ├── documents.py       # Document, DocumentVersion, Section, Chunk, Image, ParsedDocument
+│   ├── changes.py         # Change, ChangeSet, ImpactAssessment (diffra)
+│   ├── citations.py       # Citation
+│   ├── tables.py          # AgentTable, TableRevision
+│   └── ports.py           # LLM, Embedder, VisionCaptioner, VectorStore, BlobStore, Parser
+│
+├── ingestion/
+│   ├── pipeline.py        # parse -> save (embed step planned)
+│   ├── parsers/           # pdf.py (PyMuPDF); docx, text planned
+│   ├── chunking/          # page.py: page-based chunks with overlap
+│   └── enrichment/        # caption.py (planned)
+│
+├── adapters/              # every concrete implementation, imported lazily
+│   ├── storage/postgres/  # tables, engine, repository
+│   ├── blobs/             # filesystem.py; s3, supabase planned
+│   ├── llm/ embeddings/ vision/ tracing/   # planned
+│
+├── cli/                   # Typer app: ingest, versions, fulltext, outline, chunks, db, doctor
+├── retrieval/             # hybrid vector + keyword search (planned)
+├── agent/                 # the Mind + diffra subagent (planned)
+└── server/                # FastAPI + UI (planned)
+
+alembic/   tests/   examples/   docs/   ui/
+```
+
+### Design notes
+
+- **Ports and adapters, once.** `domain/ports.py` defines every external dependency as a
+  `Protocol`. The core never imports `openai`, `anthropic` or `mlflow` directly. Only
+  adapters do, and lazily, so a bare `pip install docsmind` doesn't pull in SDKs you're
+  not using.
+- **Text and images share one retrieval path.** An image becomes a chunk
+  (`chunk_type = 'image'`) whose content will be its caption, embedded with the same
+  embedder and stored in the same `chunks` table. One index, one citation type.
+- **Version-aware schema.** Every chunk, image and outline entry belongs to a
+  `DocumentVersion`, so results can be filtered and cited "as of" a specific revision.
+  Outline paths (e.g. `03.13.11`) give version comparison something stable to align on
+  where a document has a table of contents.
+- **Tables are shared state, not a UI feature.** `AgentTable` is a domain object that
+  both the agent and the user write to (append-only revisions).
+
+## Development
+
+```bash
+git clone https://github.com/yauheniya-ai/docsmind
+cd docsmind
+uv sync --all-groups --all-extras   # or: pip install -e ".[dev,all]"
+docsmind db up && docsmind db upgrade
+uv run pytest
+```
+
+`requires-python = ">=3.10"` is a floor, not a target. `uv sync` will use 3.12 (see
+`.python-version`), and CI (`.github/workflows/test.yml`) runs the tests across
+3.10, 3.11 and 3.12 so the floor is actually verified.
+
+To try the parser on a PDF without a database:
+
+```bash
+python examples/parse_pdf_demo.py path/to/file.pdf
+```
+
+## Status
+
+`0.x`: the public API (`Docsmind`, the domain models) may still change. Semantic
+versioning applies from `1.0`. See the [changelog](CHANGELOG.md) for what's in each release.
+
+## License
+
+MIT