pdbminebuilder 0.2.0__tar.gz

pdbminebuilder-0.2.0/.env.example

@@ -0,0 +1,9 @@
+# PostgreSQL connection
+PGPORT=5433
+PGHOST=localhost
+PGDATA=postgres_data_5433
+PGUSER=pdbj
+PGDATABASE=pmb
+
+# Data directory (PDBj data root)
+DATA_DIR=/path/to/pdb/data

pdbminebuilder-0.2.0/.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff

pdbminebuilder-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - "*.md"
+      - "docs/**"
+      - "website/**"
+      - "LICENSE"
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - "*.md"
+      - "docs/**"
+      - "website/**"
+      - "LICENSE"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: prefix-dev/setup-pixi@v0.9.2
+        with:
+          cache: true
+
+      - name: Ruff check
+        run: pixi run lint
+
+      - name: Ruff format check
+        run: pixi run format-check
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: prefix-dev/setup-pixi@v0.9.2
+        with:
+          cache: true
+
+      - name: Run unit tests
+        run: pixi run test-unit

pdbminebuilder-0.2.0/.github/workflows/deploy-docs.yml

@@ -0,0 +1,52 @@
+name: Deploy Docs
+
+on:
+  push:
+    branches: [main]
+    paths: ["website/**"]
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: website
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: website/package-lock.json
+
+      - run: npm ci
+      - run: npm run build
+
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: website/build
+
+  deploy:
+    needs: build
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/deploy-pages@v4
+        id: deployment

pdbminebuilder-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,68 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: write
+  id-token: write  # Required for trusted publishing to PyPI
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Extract version from tag
+        id: version
+        run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
+
+      - name: Extract release notes from CHANGELOG.md
+        id: changelog
+        run: |
+          # Extract the section for the tagged version
+          version="${{ steps.version.outputs.version }}"
+          notes=$(awk -v ver="$version" '
+            /^## \[/ {
+              if (found) exit
+              if ($0 ~ "\\[" ver "\\]") found=1; next
+            }
+            found { print }
+          ' CHANGELOG.md)
+
+          if [ -z "$notes" ]; then
+            echo "::warning::No CHANGELOG entry found for version $version"
+            notes="Release $version"
+          fi
+
+          # Write to file for gh release
+          echo "$notes" > release_notes.md
+
+      - name: Create GitHub Release
+        run: |
+          gh release create "$GITHUB_REF_NAME" \
+            --title "$GITHUB_REF_NAME" \
+            --notes-file release_notes.md
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: release
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build package
+        run: |
+          pip install build
+          python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

pdbminebuilder-0.2.0/.gitignore

@@ -0,0 +1,37 @@
+# pixi environments
+.pixi/*
+!.pixi/config.toml
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+
+# Environment
+.env
+
+node_modules/
+
+# Docusaurus
+website/build/
+website/.docusaurus/
+website/node_modules/
+data
+!website/static/data/
+logs/
+postgres_data_5433
+package-lock.json
+!website/package-lock.json
+setup-db.sh
+config.yml
+dist
+.cursor
+
+# Claude Code
+plans/
+
+# Build artifacts
+*.egg-info/
+
+# Lock files (pixi manages dependencies)
+uv.lock

pdbminebuilder-0.2.0/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.2.0] - 2026-03-07
+
+Initial release as an independent Python project. Rewritten from
+[mine2updater](https://gitlab.com/pdbjapan/mine2updater) (Node.js) by PDBj.
+
+### Added
+
+- 7 data pipelines: pdbj, cc, ccmodel, prd, prd_family, vrpt, contacts
+- 2 schema-only definitions: emdb, ihm
+- Dual format support (CIF / mmJSON) for pdbj, cc, ccmodel, prd pipelines
+- Unified parsing via gemmi for both CIF and mmJSON
+- Multi-process parallel loading with ProcessPoolExecutor
+- Bulk load mode (COPY protocol) for initial data loading
+- Mtime-based skip optimization for incremental updates
+- RDKit PostgreSQL cartridge integration for chemical searches
+- SMILES generation from molecular structure via ccd2rdmol
+- SQLAlchemy Core schema definitions with Alembic migrations
+- CLI with 9 commands: sync, update, load, all, setup-rdkit, test, reset, stats, version
+- Pydantic-based configuration with YAML and environment variable support
+- Documentation website with auto-generated schema docs
+- Docker-based test environment (PostgreSQL + RDKit)
+- PyPI publishing support with trusted publishing
+- Environment version tests for Python and PostgreSQL
+- Alternative installation methods (pip, conda+pip)
+- `config.example.yml` with documented options
+- MIT license

pdbminebuilder-0.2.0/CLAUDE.md

@@ -0,0 +1,309 @@
+# pdb-mine-builder
+
+PDBj (Protein Data Bank Japan) のデータを PostgreSQL にロードする CLI ツール。
+
+## Tech Stack
+
+- **Language**: Python 3.12+
+- **Package Manager**: Pixi (Conda/PyPI hybrid)
+- **Database**: PostgreSQL 17+ (version managed by rdkit-postgresql, psycopg3)
+- **Schema**: SQLAlchemy Core (DDL only) + Alembic (migrations)
+- **CLI**: Typer + Rich
+- **Config**: Pydantic
+- **Parser**: gemmi (CIF and mmJSON unified)
+- **Chemistry**: RDKit PostgreSQL cartridge + ccd2rdmol
+
+## Quick Start
+
+```bash
+pixi install
+pixi run pmb --help
+pixi run pmb update pdbj --limit 10
+```
+
+## Project Structure
+
+```
+src/pdbminebuilder/
+├── __init__.py
+├── __main__.py             # Entry point
+├── cli.py                  # Typer CLI commands (9 commands)
+├── config.py               # Pydantic settings
+├── models/
+│   ├── __init__.py         # MetaData registry (ALL_METADATA, get_metadata())
+│   ├── cc.py               # cc schema (10 tables)
+│   ├── ccmodel.py          # ccmodel schema
+│   ├── contacts.py         # contacts schema
+│   ├── emdb.py             # emdb schema (schema only, no pipeline)
+│   ├── ihm.py              # ihm schema (schema only, no pipeline)
+│   ├── pdbj.py             # pdbj schema (~400 tables)
+│   ├── prd.py              # prd schema
+│   ├── prd_family.py       # prd_family schema
+│   └── vrpt.py             # vrpt schema
+├── db/
+│   ├── connection.py       # psycopg3 connection pool
+│   ├── delta.py            # Delta computing
+│   ├── loader.py           # Parallel loader (ProcessPoolExecutor)
+│   ├── metadata.py         # Entry metadata tracking (mtime-based skip)
+│   └── _type_utils.py      # SA type to PostgreSQL type converter
+├── parsers/
+│   ├── cif.py              # Unified parser (CIF + mmJSON via gemmi)
+│   └── mmjson.py           # Utilities: normalize_column_name(), merge_data()
+└── pipelines/
+    ├── base.py             # BasePipeline + transform_category()
+    ├── pdbj.py             # Main PDB data (CIF/mmJSON)
+    ├── cc.py               # Chemical components
+    ├── ccmodel.py          # Chemical component models
+    ├── prd.py              # BIRD data (dual data blocks)
+    ├── prd_family.py       # BIRD family data (CIF only)
+    ├── vrpt.py             # Validation reports (CIF)
+    └── contacts.py         # Contact data (custom JSON)
+alembic/                    # Alembic migration config
+├── env.py                  # Multi-schema support
+└── versions/               # Migration scripts
+docker/
+├── docker-compose.test.yml # Test DB (PostgreSQL + RDKit)
+└── init/
+    └── 01-extensions.sql   # RDKit extension setup
+scripts/                    # One-shot utility scripts
+tests/                      # Unit + integration tests (pytest)
+docs/                       # Architecture docs
+```
+
+## Pipelines
+
+### Pipeline List
+
+| Pipeline | Default Format | Notes |
+|----------|---------------|-------|
+| pdbj | CIF | File-based (~248k files), atom_site skipped |
+| cc | CIF | Single file (components.cif.gz), ~40k blocks |
+| ccmodel | CIF | Single file (chem_comp_model.cif.gz) |
+| prd | CIF | Dual file (prd-all.cif.gz + prdcc-all.cif.gz) |
+| prd_family | CIF | Single file (family-all.cif.gz), CIF only |
+| vrpt | CIF | Uses gemmi.CifWalk for nested directory structure |
+| contacts | JSON | Array format, not mmJSON |
+| emdb | - | Schema only, no pipeline implementation |
+| ihm | - | Schema only, no pipeline implementation |
+
+### Format Selection (Dual-Format Pipelines)
+
+Pipelines pdbj, cc, ccmodel, prd support both CIF and mmJSON.
+Format is selected via `format` field in `config.yml`:
+
+```yaml
+pipelines:
+  pdbj:
+    format: cif          # "cif" (default) or "mmjson"
+    data: /path/to/data/
+```
+
+### Backward Compatibility
+
+Legacy pipeline names (`pdbj-cif`, `cc-cif`, `pdbj-json`, `cc-json`, etc.)
+are still accepted but deprecated. They emit a warning and resolve to the
+base pipeline name. Note: `-json` aliases resolve to the base name but do NOT
+change the `format` config — users must set `format: mmjson` in config.yml.
+
+### Mtime-Based Skip Optimization
+
+The `entry_metadata` table tracks file modification times. During incremental
+updates, unchanged entries are automatically skipped. Use `--force` flag to
+bypass mtime checks and reprocess all entries.
+
+## CLI Commands
+
+```bash
+pixi run pmb sync <target>           # Sync data via rsync
+pixi run pmb update <pipeline>       # Incremental update (--limit, --workers, --force)
+pixi run pmb load <pipeline>         # Bulk load via COPY protocol (initial load)
+pixi run pmb all                     # Full sync + update cycle
+pixi run pmb setup-rdkit             # Setup RDKit extension
+pixi run pmb test [pipeline...]      # Run test pipelines against test DB
+pixi run pmb reset <schema|all>      # Drop and reset schemas
+pixi run pmb stats                   # Show database statistics
+pixi run pmb --version               # Show version
+```
+
+## Key Patterns
+
+### Unified Parsing (gemmi)
+Both CIF and mmJSON are parsed via gemmi, returning row-oriented dicts:
+```python
+from pdbminebuilder.parsers.cif import parse_cif_file, parse_mmjson_file
+
+# CIF files (supports .cif.gz)
+data = parse_cif_file(filepath)
+
+# mmJSON files (supports .json.gz)
+data = parse_mmjson_file(filepath)
+
+# Both return: {"category": [{"col": "val", ...}, ...], "_block_name": "..."}
+```
+
+### Column Name Normalization
+mmJSON uses `column[1][2]` → schema uses `column12`
+```python
+from pdbminebuilder.parsers.mmjson import normalize_column_name
+```
+
+### Schema Access (SQLAlchemy Core)
+Schema definitions use SQLAlchemy Core Table objects with metadata:
+```python
+from pdbminebuilder.models import get_metadata, ALL_METADATA
+from pdbminebuilder.db.loader import get_table, get_all_tables, get_entry_pk
+
+meta = get_metadata("cc")                 # MetaData with schema="cc"
+table = get_table(meta, "brief_summary")   # SA Table object
+pk = get_entry_pk(meta)                    # "comp_id"
+tables = get_all_tables(meta)              # All tables in schema
+
+# Schema/table config stored in .info dicts
+meta.info["entry_pk"]                      # Schema-level primary key
+table.info["keywords"]                     # Table-level keywords list
+```
+
+### Category Transformation
+```python
+from pdbminebuilder.pipelines.base import transform_category
+
+# mmJSON: needs normalization
+rows = transform_category(rows, table, pk_value, pk_col, normalize_column_name)
+
+# CIF: no normalization needed (pass None)
+rows = transform_category(rows, table, pk_value, pk_col, None)
+```
+
+### Chemical SMILES Generation (cc pipeline)
+Both CIF and mmJSON cc pipelines generate canonical SMILES using ccd2rdmol + RDKit.
+This ensures consistent SMILES quality regardless of input format:
+```python
+from ccd2rdmol import read_ccd_block
+
+block = gemmi.cif.read(cif_path)[0]          # CIF
+block = gemmi.cif.read_mmjson(json_path)[0]   # mmJSON
+
+result = read_ccd_block(block, sanitize_mol=True, add_conformers=False)
+smiles = Chem.MolToSmiles(result.mol, canonical=True)
+```
+
+Note: The SMILES in `pdbx_chem_comp_descriptor` is NOT used. SMILES is always
+generated from the molecular structure for consistency and quality.
+
+### RDKit PostgreSQL Cartridge
+Chemical searches use RDKit extension (auto-configured on `cc` pipeline run):
+```sql
+-- Substructure search
+SELECT * FROM cc.brief_summary WHERE mol @> 'c1ccccc1'::mol;
+
+-- Similarity search (Tanimoto)
+SELECT *, tanimoto_sml(morganbv_fp(mol), morganbv_fp('CCO'::mol))
+FROM cc.brief_summary WHERE morganbv_fp(mol) % morganbv_fp('CCO'::mol);
+```
+
+### Parallel Processing
+- Workers create own DB connections (not pool)
+- `ProcessPoolExecutor` with configurable worker count
+
+## Development
+
+```bash
+pixi run lint          # ruff check
+pixi run format        # ruff format
+pixi run test          # pytest (unit tests)
+pixi run check         # all checks (lint, format)
+```
+
+### Testing
+
+**IMPORTANT**: テスト実行前に必ず Docker の test DB を起動すること。
+
+```bash
+# 1. Test DB を起動 (PostgreSQL + RDKit)
+pixi run test-db-up
+
+# 2. Test DB の状態確認
+pixi run test-db-status
+
+# 3. テスト実行
+pixi run test              # All tests
+pixi run test-unit         # Unit tests only
+pixi run test-integration  # Integration tests (requires test DB)
+
+# 4. Test DB を停止
+pixi run test-db-down
+```
+
+Test DB details:
+- Image: `mcs07/postgres-rdkit:latest` (PostgreSQL 17 + RDKit)
+- Container: `pmb-postgres-test`
+- Port: `15433`
+- Database: `pmb_test`
+- Config: `config.test.yml`
+
+### Database Migrations (Alembic)
+
+```bash
+pixi run db-migrate "description"  # Generate migration
+pixi run db-upgrade                # Apply all pending migrations
+pixi run db-downgrade              # Rollback last migration
+pixi run db-history                # Show migration history
+```
+
+Alembic is configured for multi-schema support (all 9 schemas).
+Schema DDL is defined in `src/pdbminebuilder/models/` as SQLAlchemy Core Table objects.
+Data operations still use psycopg3 direct connections (no SQLAlchemy Engine for data).
+
+## Database
+
+```bash
+pixi run db-start      # Start PostgreSQL
+pixi run db-stop       # Stop PostgreSQL
+pixi run db-status     # Show PostgreSQL status
+```
+
+Connection: `config.yml` の `rdb.constring`
+
+### Bulk Load Mode
+
+For initial data loading, use bulk load mode to significantly improve performance:
+
+```bash
+# 1. Start PostgreSQL
+pixi run db-start
+
+# 2. Enable bulk load mode (disables fsync, autovacuum)
+pixi run db-bulkload-mode
+
+# 3. Run data loading
+pixi run pmb load cc
+pixi run pmb load pdbj
+# ... other pipelines
+
+# 4. Restore safe settings
+pixi run db-safe-mode
+
+# 5. Run VACUUM ANALYZE
+psql -c "VACUUM ANALYZE;"
+```
+
+**WARNING**: Bulk load mode disables crash safety. If PostgreSQL crashes during bulk load:
+```bash
+pixi run db-stop
+rm -rf $PGDATA
+pixi run db-init
+# Re-run data loading from scratch
+```
+
+## Configuration
+
+- `config.yml` - Production config (customize locally)
+- `config.test.yml` - Test config (uses fixture data with `${CWD}` expansion)
+- `.env` - Environment variables (gitignored)
+- `.env.example` - Template
+
+## Known Issues
+
+- Global connection pool is for main process only; workers use direct connections
+- Workers receive `schema_name: str` and import models inside worker function (avoids pickling SA objects)
+- `emdb` and `ihm` have schema definitions but no pipeline implementations yet

pdbminebuilder-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 N283T
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.