tablassert 7.0.2__tar.gz → 7.2.0__tar.gz

tablassert-7.2.0/.github/workflows/autotag.yml

@@ -0,0 +1,24 @@
+name: Auto Tag Versions
+on:
+  workflow_run:
+    workflows:
+      - "Deploy to PyPI"
+    types:
+      - completed
+    branches:
+      - main
+permissions:
+  contents: write
+jobs:
+  autotag:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Autotag using pyproject.toml
+        uses: butlerlogic/action-autotag@1.0.1
+        with:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          tag_prefix: "v"
+          strategy: python

tablassert-7.2.0/.github/workflows/docker.yml

@@ -0,0 +1,27 @@
+name: Publish Docker Image
+on:
+  push:
+    tags:
+      - "v*"
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/tablassert:latest
+            ghcr.io/${{ github.repository_owner }}/tablassert:${{ github.ref_name }}

{tablassert-7.0.2 → tablassert-7.2.0}/.github/workflows/pipy.yml

@@ -1,8 +1,8 @@
 name: Deploy to PyPI
 on:
   push:
-    branches: [main]
-    paths: [pyproject.toml]
+    branches:
+      - main
 jobs:
   publish:
     runs-on: ubuntu-latest
@@ -16,4 +16,4 @@ jobs:
       - name: Build package
         run: uv build
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1

{tablassert-7.0.2 → tablassert-7.2.0}/.gitignore

@@ -1,17 +1,17 @@
 *.egg-info
+*.python-version
 *__pycache__/
 *.logs/
 *.opencode/
 *.ruff_cache/
 *.pytest_cache/
 *plans/
-*CLAUDE.md
-*.claude/
-*.cachassert/
+*specs/
 *.storassert/
 *.logassert/
 *DATALAKE/
 *.onnxassert/
+*.claude/
 *.envrc
 *venv/
 *.log

tablassert-7.2.0/.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: local
+    hooks:
+      - id: pyright
+        name: pyright
+        entry: uv run pyright
+        language: system
+        types: [python]
+        pass_filenames: false
+      - id: pytest
+        name: pytest
+        entry: uv run pytest
+        language: system
+        types: [python]
+        pass_filenames: false

tablassert-7.2.0/AGENTS.md

@@ -0,0 +1,170 @@
+# AGENTS.md — Tablassert
+
+Guidance for AI coding agents working in this repository.
+
+## Project Overview
+
+Tablassert is a Python package (>=3.11) for tabular data assertion, normalization, and quality control. It builds declarative knowledge graphs from tabular data, exporting NCATS Translator-compliant KGX NDJSON. Uses **Polars** DataFrames, **DuckDB** for entity resolution, and **ONNX/BioBERT** for quality control. CLI built with **Typer**. Models built with **Pydantic v2**.
+
+## Quick Reference
+
+| Task | Command |
+|---|---|
+| Install | `uv sync` |
+| Run CLI | `uv run tablassert` |
+| Lint | `uv run ruff check .` |
+| Lint (fix) | `uv run ruff check --fix .` |
+| Format | `uv run ruff format .` |
+| Format check | `uv run ruff format --check .` |
+| Type check | `uv run pyright` |
+| All checks | `uv run pre-commit run --all-files` |
+| Run all tests | `uv run pytest` |
+| Run single test | `uv run pytest tests/test_foo.py::test_name` |
+| Run by keyword | `uv run pytest -k "test_pattern"` |
+| Run with print | `uv run pytest -s tests/test_foo.py` |
+| Build | `uv build` |
+| Build docs | `uv run --group dev mkdocs build` |
+| Add dependency | `uv add <package>` |
+| Add dev dependency | `uv add --group dev <package>` |
+
+## Repository Structure
+
+```
+src/tablassert/
+  cli.py          # Typer CLI (entry point: tablassert.cli:CLI)
+  lib.py          # Core logic: encodings, data loading, Tcode(Section) class
+  models.py       # Pydantic v2 models (TablaBase base class)
+  enums.py        # str, Enum subclasses (Tokens, Repositories, Comparisons, etc.)
+  fullmap.py      # NER / entity resolution (DuckDB, 16 shards)
+  qc.py           # Quality control (ONNX/BioBERT, sentence_transformers)
+  nlp.py          # Text normalization (level_one: strip+lowercase, level_two: regex)
+  ingests.py      # YAML ingestion: from_yaml(), to_sections(), fastmerge()
+  downloader.py   # Playwright-based file downloads with retries
+  utils.py        # Hashing (xxhash), STORE path, namespace UUIDs
+  log.py          # loguru logger → .logassert/logassert.log
+  __init__.py     # Empty file (lazy loading is per-module, not here)
+docs/             # MkDocs documentation source
+mkdocs.yml        # MkDocs configuration
+pyproject.toml    # Project config, dependencies, tool settings
+tests/            # Test directory (at repo root)
+```
+
+- `conftest.py` provides a `fixtures_path` fixture returning `Path(__file__).parent / "fixtures"`.
+- pytest configured via `pyproject.toml` `[tool.pytest.ini_options]` with `testpaths = ["tests"]`.
+- Test fixtures: `tests/fixtures/` contains YAML files for Section model tests.
+- Test modules: `test_enums.py`, `test_fullmap.py`, `test_ingests.py`, `test_lib.py`, `test_models.py`, `test_nlp.py`, `test_utils.py`.
+
+## Code Style
+
+### Imports
+
+- Every file starts with `from __future__ import annotations`
+- Heavy dependencies are loaded **lazily per-module** using this pattern:
+  ```python
+  from typing import TYPE_CHECKING
+  import lazy_loader as Lazy
+
+  if TYPE_CHECKING:
+      import polars as pl
+  else:
+      pl = Lazy.load("polars")
+  ```
+- Lazy-loaded deps: `polars`, `duckdb`, `orjson`, `typer`, `xxhash`, `polars_hash`, `yaml`
+- Direct (non-lazy) heavy deps: `sqlite_utils`, `rapidfuzz`, `pydantic`, `loguru`, `yaml.CLoader`
+- Previously-optional deps now in core: `sentence_transformers`, `onnxruntime`, `sklearn`, `playwright`, `pyexcel` — lazy-loaded when present
+- Some modules mix direct and lazy imports for the same package (e.g., `ingests.py` does `from yaml import CLoader` directly, then lazy-loads `yaml` for `yaml.load()`)
+- Import order: standard library → blank line → third-party → blank line → local
+- Use `from __future__ import annotations` to enable deferred evaluation
+
+### Type Annotations
+
+- **Every variable** gets a type annotation, including locals: `col: str = "name"`, `df: pl.DataFrame = ...`
+- Use `Optional[T]` and `Union[...]` (not `T | None` or `X | Y`)
+- Use `Self` for class methods returning the class type
+- Use `Path` (not `str`) for filesystem paths
+- Use `# pyright: ignore` comments to suppress false positives from lazy-loaded modules
+
+### Pydantic Models
+
+- All models inherit from `TablaBase(BaseModel)` which sets:
+  ```python
+  model_config: ConfigDict = ConfigDict(  # pyright: ignore
+      str_strip_whitespace=False,
+      validate_assignment=True,
+      use_enum_values=True,
+      extra="forbid",
+      populate_by_name=True,
+  )
+  ```
+- Required fields: `Field(...)` (ellipsis sentinel)
+- Optional fields: `Optional[T] = Field(None)`
+- All enums are `str, Enum` subclasses (defined in `enums.py`)
+
+### Enums
+
+All enums live in `enums.py` and extend `str, Enum`. Key enums: `Tokens`, `Repositories`, `Contributions`, `Comparisons`, `Functions`, `Files`, `EncodingMethods`, `FillMethods`, `Syntaxes`, `Statuses`, `Categories`, `Predicates`, `Qualifiers`.
+
+### Naming
+
+- Functions/variables: `snake_case`
+- Classes: `PascalCase`
+- Module-level constants: `UPPER_CASE`
+
+### Comments
+
+- `# ?` — descriptions / clarifications
+- `# !` — warnings / important notes
+- `# *` — stage markers (pipeline steps)
+- `# TODO:` — todos
+- No docstrings on functions; use `# ?` comment on the line above instead
+
+### Formatting (enforced by ruff)
+
+- Line length: **120**
+- Quote style: **double quotes**
+- Indent: **4 spaces**
+- `skip-magic-trailing-comma = true`
+- Target: Python >=3.11
+
+### Error Handling
+
+- Use `RuntimeError` for exceptional cases (no custom exception classes currently)
+- Use `logger.warning()` for non-fatal issues (e.g., empty subgraphs)
+- Logger: `from tablassert.log import logger`
+
+### Other Conventions
+
+- `operator.add` for Polars string concatenation on columns (not `+` directly)
+- CLI entry point: `tablassert.cli:CLI` (Typer app with `pretty_exceptions_show_locals=False`)
+- Use `rich.progress` for progress tracking in CLI
+- Data side-effects stored in hidden directories: `.logassert/`, `.storassert/`, `.onnxassert/`
+
+## Tools
+
+- **ruff** — linting (`ruff check`) and formatting (`ruff format`)
+- **pyright** — type checking (no pyrightconfig.json; uses defaults)
+- **pre-commit** — runs ruff fix, ruff-format, pyright, and pytest on all Python files
+- **pytest** — testing (>=9.0.2)
+- **uv** — package manager (use `uv run` for all commands, `uv add` for deps)
+- **hatchling** — build backend
+
+## Optional Dependency Groups
+
+Defined in `pyproject.toml` `[project.optional-dependencies]`:
+- `rtcompat` — `polars[rtcompat]` (runtime-compatible Polars build for CPUs without required instructions)
+- `rt` — alias for `rtcompat`
+
+All other dependencies (ML, web, Excel) are now in core `dependencies`.
+
+Install with: `uv sync` or `pip install tablassert`
+
+## CI Workflows
+
+- **PyPI publish** (`.github/workflows/pipy.yml`): builds and publishes on push to `main`
+- **MkDocs deploy** (`.github/workflows/docs.yml`): builds docs and deploys to GitHub Pages on push to `main`
+- **Docker publish** (`.github/workflows/docker.yml`): builds and pushes image to GHCR on tag push (`v*`)
+- **Autotag** (`.github/workflows/autotag.yml`): automatic version tagging
+
+## Key Dependencies
+
+polars, duckdb, orjson, pydantic, typer, xxhash, loguru, rapidfuzz, scikit-learn, sqlite-utils, pyyaml, lazy-loader, polars-hash, fastexcel, pyarrow, optimum-onnx

{tablassert-7.0.2 → tablassert-7.2.0}/CHANGELOG.md

@@ -2,18 +2,44 @@
 
 All notable changes to this project are documented in this file.
 
+## 7.2.0 - 2026-03-31
+
+### New Features
+- Added `tablassert version` command to display current package version.
+- Added autotag GitHub Action for automated version tagging on releases.
+- Added PyPI publishing GitHub Action.
+- Added Docker image publishing to GitHub Container Registry (ghcr.io).
+
+### Changes
+- Sharded datassert entity-resolution database into 16 DuckDB shards for parallel querying.
+- Renamed dependency from DBssert to DATASSERT throughout.
+- Separated CLI logic into dedicated `cli.py` module.
+- Extracted NLP normalization into dedicated `nlp.py` module for cleaner separation of concerns.
+- Implemented improved parallelization model for graph compilation.
+- Annotated Pydantic model fields with `Field(...)` schema metadata.
+- Renamed `fullmap.version4()` to `fullmap.resolve()` for clarity.
+- Updated `fullmap` ranking to prioritize case-insensitive exact matches between normalized terms and preferred names.
+- Updated `fullmap` term de-duplication to keep first occurrences, improving deterministic output ordering.
+- Moved MkDocs to dev-only dependencies.
+
+### Testing
+- Added basic pytest suite covering core models, enums, ingests, lib, nlp, and utils.
+
+### Maintenance
+- Improved `.gitignore` to exclude common artifacts.
+
 ## 7.0.2 - 2026-03-23
 
 ### Changes
 - Updated package metadata for the 7.0.2 release.
-- Added optional `log` and `column_context` controls to `fullmap.version4()` for more configurable entity-resolution behavior.
+- Added optional `log` and `column_context` controls to `fullmap.resolve()` for more configurable entity-resolution behavior.
 
 ### Bug Fixes
 - Reworked entity-resolution querying to register terms directly in DuckDB instead of writing temporary parquet files, removing tempfile lifecycle issues in `fullmap` query execution.
 - Isolated unmatched-entity logging into a dedicated helper and gated it behind an explicit logging flag.
 
 ### Documentation
-- Updated API reference docs to match the current `version4()` function signature and behavior.
+- Updated API reference docs to match the current `resolve()` function signature and behavior.
 - Corrected QC documentation to reflect the implemented fuzzy/BERT validation pipeline.
 - Fixed documentation path typos for cache/store artifact directories.
 
@@ -39,7 +65,7 @@ All notable changes to this project are documented in this file.
 
 ### Breaking Changes
 - Nix is no longer supported for development and installation. Use UV-based installation instead.
-- Project now requires Python 3.13+ for compatibility with UV toolchain.
+- Project now requires Python 3.11+ for compatibility with UV toolchain.
 
 ### Documentation
 - Completely rewrote installation documentation to reflect UV-based development environment.

tablassert-7.2.0/CITATION.cff

@@ -0,0 +1,34 @@
+cff-version: 1.2.0
+message: "If you use Tablassert, please cite it as below."
+type: software
+title: Tablassert
+version: 7.2.0
+license: Apache-2.0
+repository-code: https://github.com/SkyeAv/Tablassert
+abstract: Tablassert is a highly performant declarative knowledge graph backend for bioinformatics that extracts knowledge assertions from tabular data, performs entity resolution and data quality control, and exports NCATS Translator-compliant Knowledge Graph Exchange (KGX) NDJSON.
+authors:
+  - given-names: Skye Lane
+    family-names: Goetz
+    email: sgoetz@isbscience.org
+    affiliation: Institute for Systems Biology; CalPoly SLO
+  - given-names: "Gwênlyn"
+    family-names: Glusman
+    email: gglusman@isbscience.org
+    affiliation: Institute for Systems Biology
+  - given-names: Jared C.
+    family-names: Roach
+    affiliation: Institute for Systems Biology
+references:
+  - type: article
+    title: "MicrobiomeKG: bridging microbiome research and host health through knowledge graphs"
+    authors:
+      - given-names: Skye Lane
+        family-names: Goetz
+      - given-names: Alex K.
+        family-names: Glen
+      - given-names: "Gwênlyn"
+        family-names: Glusman
+    journal: Frontiers in Systems Biology
+    year: 2025
+    volume: 5
+    doi: "10.3389/fsysb.2025.1544432"

tablassert-7.2.0/CONTRIBUTING.md

@@ -0,0 +1,266 @@
+# Contributing to Tablassert
+
+Thank you for your interest in contributing to Tablassert! This guide covers everything you need to get started.
+
+For full documentation, visit [skyeav.github.io/Tablassert](https://skyeav.github.io/Tablassert/).
+
+## Getting Started
+
+### Prerequisites
+
+- **Python 3.11** or higher
+- **[UV](https://docs.astral.sh/uv/)** package manager
+- **Git**
+
+### Setup
+
+```bash
+git clone https://github.com/SkyeAv/Tablassert.git
+cd Tablassert
+uv sync
+```
+
+### Optional Dependency Groups
+
+Some features require optional dependencies:
+
+```bash
+uv sync --extra ml        # sentence-transformers, onnxruntime, scikit-learn
+uv sync --extra web        # playwright
+uv sync --extra pyexcel    # pyexcel
+uv sync --extra full       # all optional deps
+```
+
+## Development Workflow
+
+### Quick Reference
+
+| Task | Command |
+|---|---|
+| Run CLI | `uv run tablassert --help` |
+| Lint | `uv run ruff check .` |
+| Lint (fix) | `uv run ruff check --fix .` |
+| Format | `uv run ruff format .` |
+| Format check | `uv run ruff format --check .` |
+| Type check | `uv run pyright` |
+| All checks | `uv run pre-commit run --all-files` |
+| Run all tests | `uv run pytest` |
+| Run single test | `uv run pytest tests/test_foo.py::test_name` |
+| Run by keyword | `uv run pytest -k "test_pattern"` |
+| Build | `uv build` |
+
+### Branching
+
+1. Fork the repository
+2. Create a branch from `main`:
+   ```bash
+   git checkout -b my-feature
+   ```
+3. Make your changes
+4. Run all checks before committing:
+   ```bash
+   uv run ruff check --fix . && uv run ruff format . && uv run pyright && uv run pytest
+   ```
+5. Push and open a pull request
+
+### Pre-commit Hooks
+
+Pre-commit is configured to run ruff, ruff-format, pyright, and pytest on all Python files. To install the hooks:
+
+```bash
+uv run pre-commit install
+```
+
+## Pull Requests
+
+- Describe the change and its motivation
+- Link any related issues
+- Ensure all checks pass (ruff, pyright, pytest)
+- Keep PRs focused — one concern per PR is ideal
+- If adding a new feature, include tests
+
+## Code Style
+
+### Formatting
+
+Formatting is enforced by **ruff** with these settings:
+
+- Line length: **120**
+- Quote style: **double quotes**
+- Indent: **4 spaces**
+- Target: **Python >=3.11**
+
+### Naming
+
+| Element | Convention | Example |
+|---|---|---|
+| Functions / variables | `snake_case` | `process_data`, `col_name` |
+| Classes | `PascalCase` | `Tcode`, `TablaBase` |
+| Module constants | `UPPER_CASE` | `STORE`, `TOKEN_SEP` |
+
+### Comment Markers
+
+Use these prefixes for inline comments:
+
+| Marker | Meaning | Example |
+|---|---|---|
+| `# ?` | Description or clarification | `# ? strip whitespace from column names` |
+| `# !` | Warning or important note | `# ! must run before entity resolution` |
+| `# *` | Pipeline stage marker | `# * Stage 2: Entity Resolution` |
+| `# TODO:` | Todo item | `# TODO: add fuzzy matching support` |
+
+Do **not** write docstrings on functions. Use a `# ?` comment on the line above instead.
+
+### Type Annotations
+
+**Every variable** must have a type annotation, including locals:
+
+```python
+col: str = "name"
+df: pl.DataFrame = pl.DataFrame()
+result: Optional[int] = None
+```
+
+- Use `Optional[T]` and `Union[...]` (not `T | None` or `X | Y`)
+- Use `Self` for class methods returning the class type
+- Use `Path` (not `str`) for filesystem paths
+- Use `# pyright: ignore` to suppress false positives from lazy-loaded modules
+
+### Imports
+
+Every file starts with:
+
+```python
+from __future__ import annotations
+```
+
+Heavy dependencies are **lazy-loaded** per module:
+
+```python
+from typing import TYPE_CHECKING
+import lazy_loader as Lazy
+
+if TYPE_CHECKING:
+    import polars as pl
+else:
+    pl = Lazy.load("polars")
+```
+
+Lazy-loaded packages: `polars`, `duckdb`, `orjson`, `typer`, `xxhash`, `polars_hash`, `yaml`
+
+Import order: standard library → blank line → third-party → blank line → local
+
+### Pydantic Models
+
+All models inherit from `TablaBase(BaseModel)`:
+
+```python
+from tablassert.models import TablaBase
+
+class MyModel(TablaBase):
+    name: str = Field(...)
+    description: Optional[str] = Field(None)
+```
+
+- Required fields use `Field(...)` (ellipsis sentinel)
+- Optional fields use `Optional[T] = Field(None)`
+- `extra = "forbid"` — no unknown fields allowed
+- `validate_assignment = True` — re-validate on mutation
+
+### Enums
+
+All enums live in `enums.py` and extend `str, Enum`:
+
+```python
+class Tokens(str, Enum):
+    PIPE = "|"
+    COMMA = ","
+```
+
+### Error Handling
+
+- Use `RuntimeError` for exceptional cases
+- Use `logger.warning()` for non-fatal issues
+- Import logger: `from tablassert.log import logger`
+
+## Testing
+
+Tests live in the `tests/` directory at the repo root. Test fixtures are in `tests/fixtures/`.
+
+```bash
+# Run all tests
+uv run pytest
+
+# Run a specific test
+uv run pytest tests/test_lib.py::test_my_function
+
+# Run tests matching a pattern
+uv run pytest -k "encoding"
+
+# Run with print output
+uv run pytest -s tests/test_lib.py
+```
+
+`conftest.py` provides a `fixtures_path` fixture returning `Path(__file__).parent / "fixtures"`.
+
+### Adding Tests
+
+- Place test files in `tests/` following the naming convention `test_<module>.py`
+- Use the `fixtures_path` fixture for loading test data
+- Add YAML fixture files to `tests/fixtures/` as needed
+
+## AI-Assisted Contributions
+
+Tablassert supports AI-assisted development. The repository includes an `AGENTS.md` file in the root that provides detailed guidance for AI coding tools (GitHub Copilot, Cursor, Claude Code, OpenHands, etc.).
+
+If you use AI tools to contribute:
+
+- Review all generated code before submitting
+- Ensure it follows the conventions described above and in `AGENTS.md`
+- Run all checks (`ruff`, `pyright`, `pytest`) — AI-generated code often needs style adjustments
+- The conventions in this file and `AGENTS.md` help AI tools produce idiomatic Tablassert code
+
+## Reporting Issues
+
+- **Bug reports** and **feature requests**: open an issue at [github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues)
+- Please include reproduction steps for bugs and a clear description for feature requests
+
+## License
+
+By contributing to Tablassert, you agree that your contributions will be licensed under the [Apache License 2.0](LICENSE).
+
+## Code of Conduct
+
+### Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+### Our Standards
+
+Examples of behavior that contributes to a positive environment:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by mistakes
+- Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior:
+
+- The use of sexualized language or imagery, and sexual attention or advances
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information without explicit permission
+- Other conduct which could reasonably be considered inappropriate
+
+### Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project maintainer at [sgoetz@isbscience.org](mailto:sgoetz@isbscience.org). All complaints will be reviewed and investigated fairly.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions.
+
+### Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.

tablassert-7.2.0/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.14-slim
+
+RUN pip install --no-cache-dir "tablassert[full]"
+
+ENTRYPOINT ["tablassert"]
+CMD ["--help"]