zotlib 0.4.3__tar.gz

zotlib-0.4.3/.github/workflows/publish.yml

@@ -0,0 +1,37 @@
+name: Release
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      - closed
+
+permissions:
+  contents: read
+
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    environment:
+      name: pypi
+      url: https://pypi.org/project/zotlib/
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python 3.14
+        run: uv python install 3.14
+
+      - name: Package project
+        run: uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

zotlib-0.4.3/.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches: [dev, main]
+  pull_request:
+    branches: [dev, main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: uv sync --all-groups --python ${{ matrix.python-version }}
+
+    - name: Run tests with coverage
+      run: uv run pytest -v --cov=zotlib --cov-report=term-missing

zotlib-0.4.3/.gitignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+
+# Environment
+.env
+.venv/
+
+# IDE
+.vscode/
+.idea/
+.claude
+
+# Testing
+.pytest_cache
+.coverage
+
+# Linting
+.ruff_cache
+
+# Data/output
+data
+output/
+*.csv
+*.sqlite
+zotlib.toml
+
+# Archive
+.archive/

zotlib-0.4.3/.python-version

@@ -0,0 +1 @@
+3.14

zotlib-0.4.3/PKG-INFO

@@ -0,0 +1,219 @@
+Metadata-Version: 2.4
+Name: zotlib
+Version: 0.4.3
+Summary: Extract and format bibliographic data from Zotero databases
+Requires-Python: >=3.10
+Requires-Dist: pillow<13.0.0,>=12.1.0
+Requires-Dist: polars>=1.39.2
+Requires-Dist: pymupdf<2.0.0,>=1.26.7
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.12.0
+Description-Content-Type: text/markdown
+
+# zotlib
+
+Tools for extracting and formatting bibliographic data from Zotero 8 databases.
+
+Reads directly from Zotero's local SQLite database — no API key needed. Export collections as CSV or APA-formatted references, generate PDF cover images with thumbnails, and extract annotated PDFs with baked-in highlights and markdown notes. Includes a CLI for common workflows and a Python API for custom pipelines. Built for Zotero 8; older versions are untested and likely incompatible due to schema differences.
+
+## Project Structure
+
+```
+zotlib/
+├── zotlib/                      # Python library
+│   ├── cli.py                   # CLI commands
+│   ├── config.py                # Database path discovery
+│   ├── database.py              # SQLite interface
+│   ├── extractors.py            # Data extraction functions
+│   ├── exporters.py             # Collection export (annotations + PDFs)
+│   ├── backup.py                # Zotero directory backup
+│   ├── tables.py                # Zotero database table definitions
+│   ├── covers.py                # PDF cover generation
+│   ├── paths.py                 # Path resolution and filename utilities
+│   └── formatters/apa.py        # APA citation formatter
+├── scripts/                     # Utility scripts
+│   ├── extract-annotations.js   # Annotation extractor (interactive + headless)
+│   ├── create-parent-item.js    # Create parents for standalone PDFs
+│   └── run-extract.sh           # Shell wrapper for headless extraction
+├── tests/                       # Test suite
+└── pyproject.toml               # Project configuration
+```
+
+## Installation
+
+```bash
+uv add zotlib
+```
+
+From source:
+
+```bash
+git clone https://github.com/gitronald/zotlib.git
+cd zotlib
+uv sync
+```
+
+From a specific branch:
+
+```bash
+uv add git+https://github.com/gitronald/zotlib.git@dev
+```
+
+## Configuration
+
+Run `zotlib init` to auto-discover Zotero paths and save them to `zotlib.toml`:
+
+```bash
+zotlib init
+```
+
+```
+database: /mnt/c/Users/rer/Zotero/zotero.sqlite
+pdfs_dir: /mnt/i/My Drive/zotero-pdfs
+
+Saved to zotlib.toml
+```
+
+The config file stores the database and linked PDFs directory:
+
+```toml
+[zotlib]
+database = "/path/to/zotero.sqlite"
+pdfs_dir = "/path/to/linked-pdfs"
+```
+
+Path resolution priority (for both database and PDFs dir):
+
+1. **CLI flag**: `--database`, `--pdfs-dir`
+2. **Environment variable**: `ZOTERO_DATABASE`
+3. **Config file**: `zotlib.toml`
+4. **Auto-discovery**: Checks common locations (Linux, WSL, macOS)
+
+## CLI Commands
+
+### Explore
+
+Browse collections and inspect database schema. The `show-tables` command documents Zotero's largely undocumented SQLite table structure, including column descriptions and types.
+
+```bash
+# List available collections
+zotlib show-collections
+
+# Show database tables
+zotlib show-tables
+zotlib show-tables items
+```
+
+### Export
+
+Export collection data in multiple formats. Supports linked attachments via `--pdfs-dir` for PDFs stored outside Zotero's default storage.
+
+```bash
+# Export all tables as CSV
+zotlib export-csv
+
+# Export a collection as CSV
+zotlib export-csv -c publications
+
+# Format a collection as APA references
+zotlib export-apa -c publications
+
+# Generate cover images and thumbnails
+zotlib export-covers -c publications
+
+# Export annotated PDFs and markdown notes
+zotlib export-annotations -c mycollection
+```
+
+### Backup
+
+Archive the entire Zotero data directory as a compressed `.tar.bz2` file with a progress bar. Saves to `data/backups/zotero-YYYY-MM-DD.tar.bz2` by default. Use `-o` to specify a custom output path or `-d` to point to a different database.
+
+```bash
+zotlib backup
+zotlib backup -o ~/backups/zotero-2026-03-21.tar.bz2
+```
+
+### Output structure
+
+```
+output/
+├── export-csv/                     # Bibliographic metadata
+│   └── publications.csv
+├── export-apa/                     # APA-formatted references
+│   └── publications.md
+├── export-covers/                  # PDF cover images
+│   └── publications/
+│       ├── fullsize/
+│       └── thumbnails/
+└── export-annotations/             # Annotated PDFs + notes
+    └── mycollection/
+        └── author-year-title/
+            ├── paper.pdf
+            └── annotations.md
+```
+
+### Export annotations features
+
+- Multi-attachment support: each PDF gets only its own annotations
+- Standalone attachment support: PDFs added directly to a collection
+- Linked attachment resolution via `--pdfs-dir`
+- "REVIEW: " prefix stripping from titles
+
+### Python API
+
+```python
+from zotlib import ZoteroDatabase, extract_cv_items, format_cv_as_apa
+
+db = ZoteroDatabase("/path/to/zotero.sqlite")
+items = extract_cv_items(db, collection_name="mypapers")
+apa_output = format_cv_as_apa(items, output_path="output/apa.md")
+```
+
+## Zotero JavaScript Scripts
+
+**WIP** — Utilities for Zotero's JavaScript console (Tools > Developer > Run JavaScript). The Zotero SQLite database should never be modified directly via Python — use these JS scripts (which run through Zotero's API) for any write operations.
+
+### create-parent-item.js
+
+Creates parent document items for standalone PDF attachments in a collection. Useful when PDFs were added directly without metadata — creates a parent item using the filename as the title and re-parents the attachment.
+
+### extract-annotations.js
+
+Extracts annotations from the selected item's PDFs as markdown. Auto-detects its context:
+
+- **Interactive** (Tools > Developer > Run JavaScript): shows a file save dialog
+- **Headless** (via HTTP debug API): writes to `~/Desktop/zotero-annotations/`
+
+To run headlessly:
+
+```bash
+./scripts/run-extract.sh
+```
+
+Requires: Settings > Advanced > "Allow other applications to communicate with Zotero"
+
+The shell script should work on macOS where Zotero and the terminal share the same `localhost`. On WSL, the script calls Zotero's debug HTTP endpoint on `127.0.0.1:23119`, but `localhost` does not bridge to the Windows host by default. You may need to use the Windows host IP or run the curl command from PowerShell instead.
+
+## Annotation Format
+
+### Types
+
+| Type | Extracted Data |
+|------|----------------|
+| Highlight | Text + comment + color |
+| Note | Comment text |
+| Underline | Text + comment |
+| Image | Comment only (image not exported) |
+
+### Color Labels
+
+| Hex Code | Label |
+|----------|-------|
+| `#ffd400` | yellow |
+| `#ff6666` | red |
+| `#5fb236` | green |
+| `#2ea8e5` | blue |
+| `#a28ae5` | purple |
+| `#e56eee` | magenta |
+| `#f19837` | orange |

zotlib-0.4.3/README.md

@@ -0,0 +1,207 @@
+# zotlib
+
+Tools for extracting and formatting bibliographic data from Zotero 8 databases.
+
+Reads directly from Zotero's local SQLite database — no API key needed. Export collections as CSV or APA-formatted references, generate PDF cover images with thumbnails, and extract annotated PDFs with baked-in highlights and markdown notes. Includes a CLI for common workflows and a Python API for custom pipelines. Built for Zotero 8; older versions are untested and likely incompatible due to schema differences.
+
+## Project Structure
+
+```
+zotlib/
+├── zotlib/                      # Python library
+│   ├── cli.py                   # CLI commands
+│   ├── config.py                # Database path discovery
+│   ├── database.py              # SQLite interface
+│   ├── extractors.py            # Data extraction functions
+│   ├── exporters.py             # Collection export (annotations + PDFs)
+│   ├── backup.py                # Zotero directory backup
+│   ├── tables.py                # Zotero database table definitions
+│   ├── covers.py                # PDF cover generation
+│   ├── paths.py                 # Path resolution and filename utilities
+│   └── formatters/apa.py        # APA citation formatter
+├── scripts/                     # Utility scripts
+│   ├── extract-annotations.js   # Annotation extractor (interactive + headless)
+│   ├── create-parent-item.js    # Create parents for standalone PDFs
+│   └── run-extract.sh           # Shell wrapper for headless extraction
+├── tests/                       # Test suite
+└── pyproject.toml               # Project configuration
+```
+
+## Installation
+
+```bash
+uv add zotlib
+```
+
+From source:
+
+```bash
+git clone https://github.com/gitronald/zotlib.git
+cd zotlib
+uv sync
+```
+
+From a specific branch:
+
+```bash
+uv add git+https://github.com/gitronald/zotlib.git@dev
+```
+
+## Configuration
+
+Run `zotlib init` to auto-discover Zotero paths and save them to `zotlib.toml`:
+
+```bash
+zotlib init
+```
+
+```
+database: /mnt/c/Users/rer/Zotero/zotero.sqlite
+pdfs_dir: /mnt/i/My Drive/zotero-pdfs
+
+Saved to zotlib.toml
+```
+
+The config file stores the database and linked PDFs directory:
+
+```toml
+[zotlib]
+database = "/path/to/zotero.sqlite"
+pdfs_dir = "/path/to/linked-pdfs"
+```
+
+Path resolution priority (for both database and PDFs dir):
+
+1. **CLI flag**: `--database`, `--pdfs-dir`
+2. **Environment variable**: `ZOTERO_DATABASE`
+3. **Config file**: `zotlib.toml`
+4. **Auto-discovery**: Checks common locations (Linux, WSL, macOS)
+
+## CLI Commands
+
+### Explore
+
+Browse collections and inspect database schema. The `show-tables` command documents Zotero's largely undocumented SQLite table structure, including column descriptions and types.
+
+```bash
+# List available collections
+zotlib show-collections
+
+# Show database tables
+zotlib show-tables
+zotlib show-tables items
+```
+
+### Export
+
+Export collection data in multiple formats. Supports linked attachments via `--pdfs-dir` for PDFs stored outside Zotero's default storage.
+
+```bash
+# Export all tables as CSV
+zotlib export-csv
+
+# Export a collection as CSV
+zotlib export-csv -c publications
+
+# Format a collection as APA references
+zotlib export-apa -c publications
+
+# Generate cover images and thumbnails
+zotlib export-covers -c publications
+
+# Export annotated PDFs and markdown notes
+zotlib export-annotations -c mycollection
+```
+
+### Backup
+
+Archive the entire Zotero data directory as a compressed `.tar.bz2` file with a progress bar. Saves to `data/backups/zotero-YYYY-MM-DD.tar.bz2` by default. Use `-o` to specify a custom output path or `-d` to point to a different database.
+
+```bash
+zotlib backup
+zotlib backup -o ~/backups/zotero-2026-03-21.tar.bz2
+```
+
+### Output structure
+
+```
+output/
+├── export-csv/                     # Bibliographic metadata
+│   └── publications.csv
+├── export-apa/                     # APA-formatted references
+│   └── publications.md
+├── export-covers/                  # PDF cover images
+│   └── publications/
+│       ├── fullsize/
+│       └── thumbnails/
+└── export-annotations/             # Annotated PDFs + notes
+    └── mycollection/
+        └── author-year-title/
+            ├── paper.pdf
+            └── annotations.md
+```
+
+### Export annotations features
+
+- Multi-attachment support: each PDF gets only its own annotations
+- Standalone attachment support: PDFs added directly to a collection
+- Linked attachment resolution via `--pdfs-dir`
+- "REVIEW: " prefix stripping from titles
+
+### Python API
+
+```python
+from zotlib import ZoteroDatabase, extract_cv_items, format_cv_as_apa
+
+db = ZoteroDatabase("/path/to/zotero.sqlite")
+items = extract_cv_items(db, collection_name="mypapers")
+apa_output = format_cv_as_apa(items, output_path="output/apa.md")
+```
+
+## Zotero JavaScript Scripts
+
+**WIP** — Utilities for Zotero's JavaScript console (Tools > Developer > Run JavaScript). The Zotero SQLite database should never be modified directly via Python — use these JS scripts (which run through Zotero's API) for any write operations.
+
+### create-parent-item.js
+
+Creates parent document items for standalone PDF attachments in a collection. Useful when PDFs were added directly without metadata — creates a parent item using the filename as the title and re-parents the attachment.
+
+### extract-annotations.js
+
+Extracts annotations from the selected item's PDFs as markdown. Auto-detects its context:
+
+- **Interactive** (Tools > Developer > Run JavaScript): shows a file save dialog
+- **Headless** (via HTTP debug API): writes to `~/Desktop/zotero-annotations/`
+
+To run headlessly:
+
+```bash
+./scripts/run-extract.sh
+```
+
+Requires: Settings > Advanced > "Allow other applications to communicate with Zotero"
+
+The shell script should work on macOS where Zotero and the terminal share the same `localhost`. On WSL, the script calls Zotero's debug HTTP endpoint on `127.0.0.1:23119`, but `localhost` does not bridge to the Windows host by default. You may need to use the Windows host IP or run the curl command from PowerShell instead.
+
+## Annotation Format
+
+### Types
+
+| Type | Extracted Data |
+|------|----------------|
+| Highlight | Text + comment + color |
+| Note | Comment text |
+| Underline | Text + comment |
+| Image | Comment only (image not exported) |
+
+### Color Labels
+
+| Hex Code | Label |
+|----------|-------|
+| `#ffd400` | yellow |
+| `#ff6666` | red |
+| `#5fb236` | green |
+| `#2ea8e5` | blue |
+| `#a28ae5` | purple |
+| `#e56eee` | magenta |
+| `#f19837` | orange |

zotlib-0.4.3/TODO.md

@@ -0,0 +1,19 @@
+# TODO
+
+- [ ] Test JS scripts across Windows/macOS and experiment with record modification (no plan)
+- [x] Code review cleanup: naming, SQL params, dead code, tests ([plan](.claude/plans/011-code-review-cleanup.md))
+- [x] Reorganize `zotero-js/` folder into `scripts/` ([plan](.claude/plans/010-reorganize-zotero-js.md))
+- [x] Add descriptions and columns for remaining tables in `show-tables --all` (no plan)
+- [x] Add `zotlib init` command with `zotlib.toml` config ([plan](.claude/plans/009-init-command.md))
+- [x] Restructure CLI commands with export-* naming ([plan](.claude/plans/008-restructure-cli-commands.md))
+- [x] Polish CLI schema command and standardize output dirs ([plan](.claude/plans/007-cli-polish.md))
+- [x] Convert pandas to polars ([plan](.claude/plans/006-pandas-to-polars.md))
+- [x] Export with `--base-dir` for linked attachments ([plan](.claude/plans/003-export-reviews.md))
+- [x] Verify PDF annotation baking ([plan](.claude/plans/003-export-reviews.md))
+- [x] Strip "REVIEW: " prefix from titles ([plan](.claude/plans/003-export-reviews.md))
+- [x] Add `outputs/` to `.gitignore` (no plan)
+- [x] Support standalone PDF attachments ([plan](.claude/plans/003-export-reviews.md))
+- [x] Fix multi-attachment annotation baking ([plan](.claude/plans/003-export-reviews.md))
+- [x] Rename `reviews.py` to `export.py` ([plan](.claude/plans/003-export-reviews.md))
+- [x] Create parent items for standalone attachments via Zotero JS (no plan)
+- [x] Extract titles from PDFs for untitled items ([plan](.claude/plans/003-export-reviews.md))

zotlib-0.4.3/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "zotlib"
+version = "0.4.3"
+description = "Extract and format bibliographic data from Zotero databases"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "typer>=0.12.0",
+    "rich>=13.0.0",
+    "pymupdf>=1.26.7,<2.0.0",
+    "pillow>=12.1.0,<13.0.0",
+    "polars>=1.39.2",
+]
+
+[project.scripts]
+zotlib = "zotlib.cli:app"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "ipykernel>=6.29.0",
+    "ruff>=0.15.7",
+    "pytest-cov>=7.1.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W", "UP"]

zotlib-0.4.3/scripts/create-parent-item.js

@@ -0,0 +1,74 @@
+/**
+ * Zotero Script: Create parent items for standalone PDF attachments
+ *
+ * Finds standalone PDF attachments in a collection and creates a parent
+ * document item for each, using the PDF filename (minus extension) as
+ * the title. The attachment is then re-parented under the new item.
+ *
+ * Usage: Run via Tools > Developer > Run JavaScript, or via the debug
+ * endpoint with run-create-parents.sh
+ */
+
+var COLLECTION_NAME = 'reviews';
+var output = [];
+
+// reviews collection (id=7, subcollection of writing)
+var collection = Zotero.Collections.get(7);
+
+if (!collection) {
+    output.push("");
+    output.push("ERROR: Collection '" + COLLECTION_NAME + "' not found in any library.");
+    return output;
+}
+
+output.push("");
+output.push("Found collection '" + COLLECTION_NAME + "' in library " + collection.libraryID);
+
+var itemIDs = collection.getChildItems(true);
+var items = Zotero.Items.get(itemIDs);
+output.push("Items in collection: " + items.length);
+
+var standalone = items.filter(function(item) {
+    return item.isAttachment()
+        && !item.parentItemID
+        && item.attachmentContentType === 'application/pdf';
+});
+output.push("Standalone PDFs found: " + standalone.length);
+
+function cleanTitle(rawPath) {
+    // Strip "attachments:" or "storage:" prefix, then get filename without extension
+    var clean = rawPath.replace(/^(attachments|storage):/, '');
+    var filename = clean.split('/').pop().split('\\').pop();
+    return filename.replace(/\.pdf$/i, '').replace(/_/g, ' ') || null;
+}
+
+for (var i = 0; i < standalone.length; i++) {
+    var att = standalone[i];
+    var path = att.attachmentPath || att.attachmentFilename || '';
+    var title = cleanTitle(path) || ('Untitled attachment ' + att.id);
+
+    // Create parent item
+    var parent = new Zotero.Item('document');
+    parent.libraryID = att.libraryID;
+    parent.setField('title', title);
+    await parent.saveTx();
+
+    // Add parent to the same collection
+    collection.addItem(parent.id);
+    await collection.saveTx();
+
+    // Re-parent the attachment under the new item
+    att.parentItemID = parent.id;
+    await att.saveTx();
+
+    // Remove the now-child attachment from the collection
+    collection.removeItem(att.id);
+    await collection.saveTx();
+
+    output.push("  [" + (i+1) + "/" + standalone.length + "] Created parent: " + title);
+}
+
+output.push("");
+output.push("Done. Created " + standalone.length + " parent items.");
+
+return output;