doctrail 0.3.1__tar.gz

doctrail-0.3.1/.github/workflows/docs.yml

@@ -0,0 +1,51 @@
+name: docs
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+env:
+  UV_PYTHON: "3.12"
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Generate full manual
+        run: uv run python scripts/build_llms_full.py
+      - name: Check generated manual
+        run: git diff --exit-code docs/llms.txt
+      - name: Build docs
+        run: uv run --with mkdocs-material --with mkdocs-click mkdocs build --strict
+      - uses: actions/configure-pages@v5
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

doctrail-0.3.1/.github/workflows/release.yml

@@ -0,0 +1,110 @@
+name: release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+env:
+  UV_PYTHON: "3.12"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Check release tag matches package version
+        run: |
+          package_version="$(uv run --no-sync python - <<'PY'
+          import tomllib
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
+          PY
+          )"
+          tag_version="${GITHUB_REF_NAME#v}"
+          if [ "$package_version" != "$tag_version" ]; then
+            echo "Tag $GITHUB_REF_NAME does not match pyproject version $package_version" >&2
+            exit 1
+          fi
+
+      - name: Check lockfile
+        run: uv lock --check
+
+      - name: Install dependencies
+        run: uv sync --locked --extra test
+
+      - name: Generate full manual
+        run: uv run python scripts/build_llms_full.py
+
+      - name: Check generated manual
+        run: git diff --exit-code docs/llms.txt
+
+      - name: Run tests
+        run: uv run python -m pytest tests/ -x -q -k "not integration"
+
+      - name: Prepare distribution directory
+        run: |
+          uv run --no-sync python - <<'PY'
+          import shutil
+          from pathlib import Path
+          dist = Path("dist")
+          if dist.exists():
+              shutil.rmtree(dist)
+          PY
+
+      - name: Build package artifacts
+        run: uv build --no-sources
+
+      - name: Refuse paper artifacts
+        run: |
+          for archive in dist/*.tar.gz; do
+            if tar -tzf "$archive" | grep -E '^doctrail-[^/]+/(_paper|paper)(/|$)'; then
+              echo "Paper files are present in $archive" >&2
+              exit 1
+            fi
+          done
+          for wheel in dist/*.whl; do
+            if unzip -Z1 "$wheel" | grep -E '^(_paper|paper)(/|$)'; then
+              echo "Paper files are present in $wheel" >&2
+              exit 1
+            fi
+          done
+
+      - name: Check package metadata
+        run: uvx --from twine twine check dist/*
+
+      - name: Smoke test wheel
+        run: |
+          for wheel in dist/*.whl; do
+            uvx --from "$wheel" doctrail --version
+          done
+
+      - name: Smoke test source distribution
+        run: |
+          for archive in dist/*.tar.gz; do
+            uvx --from "$archive" doctrail --version
+          done
+
+      - name: Upload package artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: doctrail-dist
+          path: dist/*
+
+      - name: Publish to PyPI
+        run: uv publish --trusted-publishing always

doctrail-0.3.1/.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+env:
+  UV_PYTHON: "3.12"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Check lockfile
+        run: uv lock --check
+
+      - name: Install dependencies
+        run: uv sync --locked --extra test
+
+      - name: Run tests
+        run: uv run python -m pytest tests/ -x -q -k "not integration"

doctrail-0.3.1/.gitignore

@@ -0,0 +1,279 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env.local
+.env.*.local
+.venv
+.venv*/
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Private server configuration (contains paths, funnel slugs, etc.)
+.config.yaml
+.serve-funnel.sh
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# ==============================================================================
+# Doctrail-specific patterns
+# ==============================================================================
+
+# Archived/cut code held for possible restore (also recoverable from git history)
+/_cut/
+
+# Development artifacts
+.unison/
+.history/
+.deleted/
+.unison*
+*conflict_on_*
+/agents/
+/_paper/
+/data/
+/notes/
+/out/
+/scratch/
+/tmp/
+
+# IDE configurations
+.idea/
+.vscode/
+
+# Database files (NEVER commit production databases!)
+*.db
+*.db-shm
+*.db-wal
+*.sqlite
+
+# Test database copies - NEVER COMMIT THESE
+tests/test_copy_*.db
+tests/temp_*.db
+tests/*_test.db
+
+# Logs and temporary files
+*.log
+*.csv
+
+# Export directories
+exports/
+**/exports/*
+
+# System files
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Local notes and documentation
+.llm_notes/
+CLAUDE.md
+
+# Development and personal directories
+claude_notes/
+presentations/
+legacy/
+legacy.zip
+tests/legacy.zip
+tmp/
+.claude/
+.agents/
+.deleted/
+.history/
+.ra-aid/
+.unison/
+.scratch/
+.uv-cache/
+
+# Allow specific important database files
+!tests/test.db/tmp/
+scratch/
+agents/
+agents.md
+AGENTS.md
+agent.md
+AGENT.md
+notes/
+data/
+out/
+.beads/
+first_prompt.md
+/.doctrail/

doctrail-0.3.1/CHANGELOG.md

@@ -0,0 +1,134 @@
+# Changelog
+
+## 0.3.1 - revamp release readiness
+
+### Documentation and tutorial fixtures
+
+- Added a terminal demo GIF (ingest, codebook, enrich, query) to the README and docs landing page, rendered reproducibly from `scripts/demo.tape` via `scripts/build_demo_gif.sh`.
+- Fixed Office and ebook ingestion so tuple-returning extractors populate document text instead of silently recording empty content, and hardened the tutorial corpus against short extracted files.
+- Added committed public-domain extraction fixtures across supported ingest file families and documented the supported local file types in ingest help and quickstart docs.
+- Added generated Click-backed CLI documentation, packaged `doctrail docs` manual output, and CI drift checks for CLI, YAML snippets, and `llms-full.txt`.
+- Added packaged `doctrail skill` output and `doctrail skill --install` for installing the Doctrail operating doctrine into agent skill directories.
+- Added codebook-quality prompt guidance to presets, generated enrichment scaffolds, and YAML docs, with cache-prefix caveats for provider prompt caching.
+- Fixed `doctrail new -p` flag mode so scaffold generation stays non-interactive and reports a clear terminal-required error for wizard-only paths.
+- Added a fully offline `doctrail init test` tutorial scaffold with replay fixtures, Federalist Papers examples, UN speech excerpts, and the `doctrail run` alias.
+- Added tutorial ICR replay examples that bracket agreement quality: a crisp `mentions_climate` boolean codebook and a deliberately under-specified `optimism` score.
+- Replaced the tutorial's second enrichment with `securitization`, including replay fixtures that demonstrate gate-dependent null fields.
+- Regenerated the UN speech tutorial corpus as deterministic PDF, DOCX, and HTML containers and re-keyed the UN replay fixtures to the new file hashes.
+- Renamed the repository tutorial fixture directory from `examples/tutorial/data/` to `examples/tutorial/corpus/` so ignored `data/` directories stay local-only.
+- Added source context to model-by-model pivot views so ICR disagreements can be diagnosed directly from the generated review surface.
+
+### Storage and provenance
+
+- Renamed the package layout to `src/doctrail` and kept the legacy import surface working through compatibility exports.
+- Added normalized enrichment identity: one current row per key, enrichment name, field, model, and prompt hash, enforced by a unique index and upsert semantics.
+- Preserved superseded enrichment rows in side tables during identity migration instead of silently losing recoverable values.
+- Renamed Doctrail-managed physical tables with a leading underscore and managed views with a `v_` prefix so source tables and review surfaces are easier to distinguish.
+- Added prompt, query, run, and project provenance across `_prompts`, `_enrichment_audit`, `_enrichment_runs`, and `_enrichment_run_items`.
+- Added ordered SQLite schema migrations stamped with `PRAGMA user_version`, with the existing idempotent schema guards folded into baseline migration 1.
+- Recorded parsed null answers as completed normalized rows so append mode does not resubmit already answered rows.
+
+### Execution and review surfaces
+
+- Added query-scoped and enrichment-scoped dedupe paths so append mode can skip successful prior work without treating audit rows alone as completion.
+- Expanded run-aware view creation: run views, final views, pivot/spec/render surfaces, and editable final tables now use the persisted run ledger.
+- Added ICR and override workflows backed by SQLite tables so modeled output, human overrides, and finalized review surfaces remain separate.
+- Silenced cost/pricing warnings for replay-backed tutorial models while preserving warnings for real unknown models.
+
+### Release preparation
+
+- Added a manual-only release workflow that builds and checks artifacts by default; publishing remains inert unless explicitly enabled with a configured PyPI token.
+- Refreshed the documented configuration surface, including stable, deprecated-but-working, and internal keys.
+
+## 2026-03-30 - batch backends, rerun selectors, and env precedence
+
+### Batch execution
+
+- `--execution-mode openai-batch` now maps direct providers to their native batch APIs while keeping one doctrail-facing workflow:
+  - OpenAI: `/v1/batches` with request lines targeting `/v1/chat/completions`
+  - Anthropic: `/v1/messages/batches` with request params targeting `/v1/messages`
+  - Gemini: File API upload plus `/v1beta/models/{model}:batchGenerateContent`
+- Batch submit, poll, watch, reconcile, and cancel now work through the same CLI path for direct OpenAI, Anthropic, and Gemini models.
+- CLI help, README, docs, and the doctrail skill now make the provider-specific endpoint mapping explicit.
+
+### Anthropic batch hardening
+
+- Added direct Anthropic batch support for `claude-*` and `anthropic/*` models behind the existing batch mode.
+- Added provider-side schema compatibility handling for Anthropic structured batch output:
+  - bounded integer fields no longer emit unsupported `minimum` / `maximum` constraints into the submitted Anthropic batch schema
+  - doctrail now warns about those compatibility issues before submission
+- Fixed Anthropic batch polling so provider error objects are serialized cleanly instead of causing downstream JSON serialization failures during reconciliation.
+- Live smoke verification completed successfully with `claude-haiku-4-5`.
+
+### Gemini batch changes
+
+- Added direct Gemini batch support for `gemini-*` and `models/gemini-*`.
+- Initial Gemini support used inline requests; doctrail now defaults to Google's recommended file-backed batch input mode for Gemini jobs.
+- Gemini batch JSONL request lines are now emitted in the file-input shape Google expects, with stable per-row `key` values for reconciliation.
+- Gemini batch results are now downloaded and reconciled from the provider result file when available.
+- Live verification confirmed that the file-backed path is accepted by Google and produces real `files/...` input handles plus real `batches/...` jobs.
+- Operational caveat: Gemini Batch remains unreliable in practice. Live testing saw long-lived `BATCH_STATE_PENDING` jobs and later `503 UNAVAILABLE` responses from Google's GET batch endpoint after roughly 24 hours. This appears to be a provider-side reliability issue rather than a doctrail endpoint or model-id mismatch.
+
+### Targeted reruns
+
+- Added `doctrail enrich --where "..."` to filter an enrichment's existing base query with an outer SQL `WHERE` predicate.
+- `--query` remains available as the full-query replacement escape hatch.
+- This makes targeted reruns like date filters, `LIKE`, and explicit key lists possible without cloning the YAML prompt/schema definition.
+
+### Environment precedence
+
+- Doctrail now prefers the nearest project-local `.env` over inherited shell or global environment variables.
+- This applies to provider resolution and cost/model utilities, so a project can reliably use its own keys without depending on the caller's ambient shell state.
+
+## 2026-01-15 - UX overhaul for social scientists
+
+### New commands
+
+- **`doctrail new`** - Create custom enrichments interactively
+  - Interactive mode: `doctrail new`
+  - Quick mode: `doctrail new topic -p "Classify topic" -o topic --enum "a,b,c"`
+  - Supports: string, integer, boolean, array, enum types
+
+- **`doctrail view`** - Manage database views
+  - `doctrail view` - list views in database
+  - `doctrail view refresh` - execute all `.doctrail/views/*.sql` files
+  - `doctrail view new <name>` - create custom view SQL template
+
+- **`doctrail query`** - Query database without needing sqlite-utils
+  - `doctrail query` - list documents
+  - `doctrail query 1` - show document #1 details
+  - `doctrail query "SELECT ..."` - run arbitrary SQL
+
+### Auto-generated views
+
+After enrichment completes, a queryable view is automatically created:
+```
+📊 View updated: enrichments_doctrail_demo
+   Query with: doctrail query "SELECT * FROM enrichments_doctrail_demo LIMIT 10"
+```
+
+This pivots the long-format `_enrichments` table into wide format for easy querying.
+
+### Preset enrichments with aliases
+
+- Built-in presets: `summarize`, `language`, `sentiment`, `document_type`, `relevance`, `keywords`, `extract_entities`, `research_methods`
+- British/Australian aliases: `summarise` → `summarize`, `lang` → `language`
+- Presets auto-copy to project folder when first used (so users can edit them)
+
+### Schema fixes
+
+- Fixed bare type schemas like `{type: string}` being misinterpreted
+- Now correctly wraps with `output_column`: `{summary: {type: string}}`
+- Also handles `{enum: [...]}` and `{enum_list: [...]}` bare schemas
+
+### Project tagging
+
+- Enrichments now tagged with `project_name` from config by default
+- Enables project-based filtering and automatic view creation
+
+### Python-first extraction (from earlier session)
+
+- PDF: pymupdf as primary (260x faster than OCR-first approach)
+- EPUB: ebooklib as primary
+- DOCX: python-docx
+- System tools (pdftotext, mutool, etc.) now fallbacks

doctrail-0.3.1/CONTRIBUTING.md

@@ -0,0 +1,7 @@
+# Contributing
+
+Doctrail has both generated and hand-written documentation. `docs/cli.md` is generated at render time from the live Click command tree through `mkdocs-click`. `docs/llms-full.txt` is generated by `scripts/build_llms_full.py`; for the CLI page it renders the same Click tree as plain text so agents see real command help, not a Markdown directive.
+
+The CI drift guards check three things: YAML snippets in `docs/yaml.md` and `docs/quickstart.md` parse and validate unless marked as fragments, documented `doctrail ...` commands resolve against the Click tree, and the committed `docs/llms-full.txt` matches a fresh generator run.
+
+If you change the CLI or YAML surface, CI fails until docs match. Regenerate with `scripts/build_llms_full.py` and fix the prose in the same commit.

doctrail-0.3.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matthew P. Robertson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.