norm_toolkit 1.8.0__tar.gz → 1.9.0__tar.gz

norm_toolkit-1.9.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.3
+Name: norm_toolkit
+Version: 1.9.0
+Summary: Toolkit to normalize text to UMLS / ontologies
+Author: Haydn Jones
+Author-email: Haydn Jones <haydnjonest@gmail.com>
+Requires-Dist: asyncpg>=0.29.0
+Requires-Dist: clickhouse-connect>=1.0.0
+Requires-Dist: duckdb>=1.5.0
+Requires-Dist: lvg-norm>=1.3.0
+Requires-Dist: polars[rt64]>=1.36.1
+Requires-Dist: pyarrow>=20.0.0
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: tqdm>=4.67.1
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+## ClickHouse backend
+
+The DuckDB builder remains the source of truth. Build a DuckDB file with
+`build_merged_duckdb`, then upload its canonical tables into ClickHouse:
+
+```bash
+uv run python scripts/upload_clickhouse.py data/dbs_final/SmallMolecule.duckdb --database normalization
+```
+
+The upload shows a progress bar for each copied table; pass `--no-progress` to
+silence it.
+
+Connection settings are read from `.env` with `python-dotenv` and use the
+official `clickhouse-connect` client. Set `CH_HTTP`, for example
+`http://host:8123/normalization`; `CH_USER` and `CH_PASSWORD` may be supplied
+separately and override URL credentials.
+
+Use the ClickHouse backend from Python:
+
+```python
+from norm_toolkit import ClickHouseNormalizer
+
+normalizer = ClickHouseNormalizer(database="normalization")
+result = normalizer.normalize(["aspirin"], top_k=5)
+```
+
+You can also pass a DSN in code:
+
+```python
+normalizer = ClickHouseNormalizer(
+    dsn="http://host:8123/normalization",
+    database="normalization",
+)
+```

norm_toolkit-1.9.0/README.md

@@ -0,0 +1,34 @@
+## ClickHouse backend
+
+The DuckDB builder remains the source of truth. Build a DuckDB file with
+`build_merged_duckdb`, then upload its canonical tables into ClickHouse:
+
+```bash
+uv run python scripts/upload_clickhouse.py data/dbs_final/SmallMolecule.duckdb --database normalization
+```
+
+The upload shows a progress bar for each copied table; pass `--no-progress` to
+silence it.
+
+Connection settings are read from `.env` with `python-dotenv` and use the
+official `clickhouse-connect` client. Set `CH_HTTP`, for example
+`http://host:8123/normalization`; `CH_USER` and `CH_PASSWORD` may be supplied
+separately and override URL credentials.
+
+Use the ClickHouse backend from Python:
+
+```python
+from norm_toolkit import ClickHouseNormalizer
+
+normalizer = ClickHouseNormalizer(database="normalization")
+result = normalizer.normalize(["aspirin"], top_k=5)
+```
+
+You can also pass a DSN in code:
+
+```python
+normalizer = ClickHouseNormalizer(
+    dsn="http://host:8123/normalization",
+    database="normalization",
+)
+```

{norm_toolkit-1.8.0 → norm_toolkit-1.9.0}/pyproject.toml

@@ -1,45 +1,51 @@
 [project]
 name = "norm_toolkit"
-version = "1.8.0"
+version = "1.9.0"
 description = "Toolkit to normalize text to UMLS / ontologies"
 readme = "README.md"
 authors = [{ name = "Haydn Jones", email = "haydnjonest@gmail.com" }]
 requires-python = ">=3.12"
 dependencies = [
     "asyncpg>=0.29.0",
-    "duckdb>=1.4.3",
-    "lvg-norm>=1.1.0",
+    "clickhouse-connect>=1.0.0",
+    "duckdb>=1.5.0",
+    "lvg-norm>=1.3.0",
     "polars[rt64]>=1.36.1",
     "pyarrow>=20.0.0",
     "pydantic>=2.12.5",
+    "python-dotenv>=1.2.2",
     "sqlalchemy>=2.0.0",
     "tqdm>=4.67.1",
 ]
 
 [dependency-groups]
 dev = [
-    "datasets>=4.4.1",
-    "dotenv>=0.9.9",
     "ipython>=9.8.0",
     "pytest>=8.3",
-    "rdkit>=2025.9.3",
     "ruff>=0.6.9",
     "fire>=0.7.1",
-    "joblib>=1.5.3",
+    "ipykernel>=7.2.0",
+    "ipywidgets>=8.1.8",
 ]
 
 [build-system]
-requires = ["uv_build>=0.9.11,<0.10.0"]
+requires = ["uv_build>=0.9.11,<0.13.0"]
 build-backend = "uv_build"
 
 [tool.ruff]
 line-length = 120
 indent-width = 4
-target-version = "py313"
+target-version = "py312"
+# v1 is a frozen vendored snapshot of the 1.8.0 release; don't lint/format it.
+extend-exclude = ["src/norm_toolkit/v1"]
+
+[tool.ty.src]
+# v1 is a frozen vendored snapshot of the 1.8.0 release; don't type-check it.
+exclude = ["src/norm_toolkit/v1"]
 
 [tool.ruff.lint]
 select = ["E", "F", "UP", "B", "SIM", "I", "FURB"]
-ignore = ["B905", "E501", "SIM108", "SIM103"]
+ignore = ["E501", "SIM108", "SIM103"]
 fixable = ["ALL"]
 
 [tool.ruff.format]

norm_toolkit-1.9.0/src/norm_toolkit/__init__.py

@@ -0,0 +1,18 @@
+"""
+norm_toolkit — biomedical text normalization to UMLS / custom ontologies.
+
+This package is versioned into two subpackages:
+
+- ``norm_toolkit.v1``: the 1.8.0 API (DuckDB + PostgreSQL backends)
+- ``norm_toolkit.v2``: the current API (DuckDB + ClickHouse backends)
+
+The top-level namespace re-exports the **v1** API for backwards compatibility,
+so ``from norm_toolkit import DuckDBNormalizer`` resolves to ``norm_toolkit.v1``.
+New code should import explicitly from ``norm_toolkit.v2`` (or ``norm_toolkit.v1``).
+"""
+
+from norm_toolkit import v1, v2
+from norm_toolkit.v1 import *  # noqa: F403  (default API == v1)
+from norm_toolkit.v1 import __all__ as _v1_all
+
+__all__ = ["v1", "v2", *_v1_all]

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/__init__.py

@@ -21,14 +21,14 @@ Data models:
 - SemanticType: Semantic type info (UMLS only)
 """
 
-from norm_toolkit.build_merged import build_merged_duckdb
-from norm_toolkit.build_ontology import build_ontology_duckdb
-from norm_toolkit.build_umls import build_umls_duckdb
-from norm_toolkit.constants import ONTOLOGY_DF_SCHEMA
-from norm_toolkit.models import ConceptInfo, SemanticType
-from norm_toolkit.normalizer import DuckDBNormalizer
-from norm_toolkit.normalizer_postgres import PostgresNormalizer
-from norm_toolkit.utils import prepare_ontology_df, push_to_postgres
+from .build_merged import build_merged_duckdb
+from .build_ontology import build_ontology_duckdb
+from .build_umls import build_umls_duckdb
+from .constants import ONTOLOGY_DF_SCHEMA
+from .models import ConceptInfo, SemanticType
+from .normalizer import DuckDBNormalizer
+from .normalizer_postgres import PostgresNormalizer
+from .utils import prepare_ontology_df, push_to_postgres
 
 __all__ = [
     # Build functions

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/build_ontology.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import polars as pl
 
-from norm_toolkit.build_merged import build_merged_duckdb
+from .build_merged import build_merged_duckdb
 
 
 def build_ontology_duckdb(

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/build_umls.py

@@ -6,7 +6,7 @@ This is a convenience wrapper around build_merged_duckdb for UMLS-only builds.
 
 from __future__ import annotations
 
-from norm_toolkit.build_merged import build_merged_duckdb
+from .build_merged import build_merged_duckdb
 
 
 def build_umls_duckdb(

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/normalizer.py

@@ -14,7 +14,7 @@ from textwrap import dedent
 import duckdb
 import polars as pl
 
-from norm_toolkit.constants import (
+from .constants import (
     ATOMS_TABLE,
     DEFAULT_PREFER_TTYS,
     DEFS_TABLE,
@@ -24,8 +24,8 @@ from norm_toolkit.constants import (
     NW_TABLE,
     TYPES_TABLE,
 )
-from norm_toolkit.models import ConceptInfo
-from norm_toolkit.normalizer_utils import (
+from .models import ConceptInfo
+from .normalizer_utils import (
     apply_concept_name_rows,
     apply_definition_rows,
     apply_semantic_type_rows,

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/normalizer_postgres.py

@@ -14,7 +14,7 @@ import polars as pl
 from sqlalchemy import RowMapping, text
 from sqlalchemy.ext.asyncio import AsyncEngine
 
-from norm_toolkit.constants import (
+from .constants import (
     ATOMS_TABLE,
     DEFAULT_PREFER_TTYS,
     DEFS_TABLE,
@@ -24,9 +24,9 @@ from norm_toolkit.constants import (
     NW_TABLE,
     TYPES_TABLE,
 )
-from norm_toolkit.models import ConceptInfo
-from norm_toolkit.normalizer_cache import ExpansionCache, NormalizerCache
-from norm_toolkit.normalizer_utils import (
+from .models import ConceptInfo
+from .normalizer_cache import ExpansionCache, NormalizerCache
+from .normalizer_utils import (
     apply_concept_name_rows,
     apply_definition_rows,
     apply_semantic_type_rows,

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/normalizer_utils.py

@@ -11,14 +11,14 @@ from typing import Any, Literal
 from lvg_norm import lvg_normalize
 from sqlalchemy import RowMapping
 
-from norm_toolkit.constants import (
+from .constants import (
     EXACT_BUMP,
     ISPREF_WEIGHT,
     RANK_MULTIPLIER,
     STT_WEIGHT,
     TTY_WEIGHT,
 )
-from norm_toolkit.models import ConceptInfo, SemanticType
+from .models import ConceptInfo, SemanticType
 
 
 def _coerce_synonyms_list(

{norm_toolkit-1.8.0/src/norm_toolkit → norm_toolkit-1.9.0/src/norm_toolkit/v1}/utils.py

@@ -11,7 +11,7 @@ import polars as pl
 from lvg_norm import lvg_normalize
 from tqdm import tqdm
 
-from norm_toolkit.constants import (
+from .constants import (
     ATOMS_TABLE,
     CONCEPTS_TABLE,
     DEFS_TABLE,

norm_toolkit-1.9.0/src/norm_toolkit/v2/__init__.py

@@ -0,0 +1,44 @@
+"""
+Unified normalization package.
+
+Provides normalizer implementations that work with UMLS, ontology,
+and merged databases using a standardized schema.
+
+Build functions:
+- build_merged_duckdb: Build a DuckDB database from UMLS and/or ontology data
+
+Normalizers:
+- DuckDBNormalizer: High-throughput sync normalizer for DuckDB (batch processing)
+- ClickHouseNormalizer: Normalizer backed by ClickHouse tables uploaded from a DuckDB build
+
+ClickHouse:
+- upload_duckdb_to_clickhouse: Upload a norm_toolkit DuckDB database into ClickHouse
+- clickhouse_client_from_env: Build a ClickHouse client from .env / environment settings
+
+Data models:
+- ConceptInfo: Unified concept metadata
+- SemanticType: Semantic type info (UMLS only)
+"""
+
+from .build_merged import build_merged_duckdb
+from .clickhouse_backend import ClickHouseNormalizer
+from .clickhouse_common import clickhouse_client_from_env
+from .clickhouse_upload import upload_duckdb_to_clickhouse
+from .duckdb_backend import DuckDBNormalizer
+from .schema import ONTOLOGY_DF_SCHEMA, ConceptInfo, SemanticType
+
+__all__ = [
+    # Build functions
+    "build_merged_duckdb",
+    "upload_duckdb_to_clickhouse",
+    # Normalizers
+    "ClickHouseNormalizer",
+    "DuckDBNormalizer",
+    # ClickHouse
+    "clickhouse_client_from_env",
+    # Models
+    "ConceptInfo",
+    "SemanticType",
+    # Schemas
+    "ONTOLOGY_DF_SCHEMA",
+]