norm_toolkit 1.9.1__tar.gz → 1.9.2__tar.gz

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: norm_toolkit
-Version: 1.9.1
+Version: 1.9.2
 Summary: Toolkit to normalize text to UMLS / ontologies
 Author: Haydn Jones
 Author-email: Haydn Jones <haydnjonest@gmail.com>

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "norm_toolkit"
-version = "1.9.1"
+version = "1.9.2"
 description = "Toolkit to normalize text to UMLS / ontologies"
 readme = "README.md"
 authors = [{ name = "Haydn Jones", email = "haydnjonest@gmail.com" }]

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/src/norm_toolkit/v2/clickhouse_backend.py

@@ -549,7 +549,7 @@ class ClickHouseNormalizer(BaseNormalizer):
         max_depth: int | None,
         filter_ontologies: list[str] | None,
     ) -> pl.DataFrame:
-        schema = pl.Schema({"root_id": pl.UInt64, "identifier": pl.Utf8})
+        schema = pl.Schema({"root_id": pl.UInt64, "gid": pl.UInt64, "identifier": pl.Utf8})
         unique_ids = list(dict.fromkeys(concept_ids))
         if not unique_ids:
             return pl.DataFrame(schema=schema)

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/src/norm_toolkit/v2/duckdb_backend.py

@@ -269,7 +269,7 @@ class DuckDBNormalizer(BaseNormalizer):
         max_depth: int | None,
         filter_ontologies: list[str] | None,
     ) -> pl.DataFrame:
-        schema = pl.Schema({"root_id": CONCEPT_ID_DTYPE, "identifier": pl.Utf8})
+        schema = pl.Schema({"root_id": CONCEPT_ID_DTYPE, "gid": CONCEPT_ID_DTYPE, "identifier": pl.Utf8})
         if not concept_ids:
             return pl.DataFrame(schema=schema)
         roots_df = pl.DataFrame(

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/src/norm_toolkit/v2/normalizer_base.py

@@ -16,7 +16,7 @@ from .normalizer_utils import (
     build_normalized_query_map,
     canonicalize_semantic_type_ids,
 )
-from .schema import CONCEPT_ID_DTYPE, HIT_STRUCT_TYPE, ConceptInfo, SemanticType
+from .schema import CONCEPT_ID_DTYPE, HIT_STRUCT_TYPE, NARROWER_STRUCT_TYPE, ConceptInfo, SemanticType
 
 _HIT_FIELD_NAMES = tuple(HIT_STRUCT_TYPE.to_schema())
 
@@ -72,8 +72,9 @@ class BaseNormalizer(ABC):
     ) -> pl.DataFrame:
         """Walk the edges from many roots at once, resolving identifiers in-query.
 
-        Returns a frame with `root_id` and `identifier` columns mapping each root
-        gid to its descendants' source-local identifiers (excluding the root).
+        Returns a frame with `root_id`, `gid`, and `identifier` columns mapping
+        each root gid to its descendants' gids and source-local identifiers
+        (excluding the root).
         """
 
     @abstractmethod
@@ -294,7 +295,7 @@ class BaseNormalizer(ABC):
         expanded_hit_rows = (
             hit_rows.drop("narrower")
             .join(narrower_df, on="gid", how="left")
-            .with_columns(pl.col("narrower").fill_null([]).cast(pl.List(pl.Utf8)))
+            .with_columns(pl.col("narrower").fill_null([]).cast(pl.List(NARROWER_STRUCT_TYPE)))
             .sort(["__row_idx", "__hit_idx"])
         )
         expanded_hits = expanded_hit_rows.group_by("__row_idx", maintain_order=True).agg(
@@ -314,8 +315,8 @@ class BaseNormalizer(ABC):
         max_depth: int | None,
         filter_ontologies: list[str] | None,
     ) -> pl.DataFrame:
-        """Return (gid, narrower) rows mapping each root gid to descendant identifiers."""
-        empty = pl.DataFrame(schema={"gid": CONCEPT_ID_DTYPE, "narrower": pl.List(pl.Utf8)})
+        """Return (gid, narrower) rows mapping each root gid to descendant concepts."""
+        empty = pl.DataFrame(schema={"gid": CONCEPT_ID_DTYPE, "narrower": pl.List(NARROWER_STRUCT_TYPE)})
         if not root_ids:
             return empty
 
@@ -324,8 +325,9 @@ class BaseNormalizer(ABC):
             return empty
 
         return (
-            map_df.group_by("root_id")
-            .agg(pl.col("identifier").sort().alias("narrower"))
+            map_df.sort("root_id", "identifier")
+            .group_by("root_id", maintain_order=True)
+            .agg(pl.struct("gid", "identifier").alias("narrower"))
             .rename({"root_id": "gid"})
             .with_columns(pl.col("gid").cast(CONCEPT_ID_DTYPE))
         )

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/src/norm_toolkit/v2/normalizer_utils.py

@@ -559,8 +559,8 @@ def build_narrower_identifiers_map_sql(
     descendants to their source-local identifiers in the same query.
 
     `roots_source_sql` must be a relation yielding a `concept_id` column for the
-    starting concepts. The result has (root_id, identifier) rows mapping each
-    root gid to its descendants' identifiers, excluding the root itself.
+    starting concepts. The result has (root_id, gid, identifier) rows mapping each
+    root gid to its descendants' gids and identifiers, excluding the root itself.
     """
     recurse_filters: list[str] = []
     if max_depth is not None:
@@ -580,7 +580,7 @@ def build_narrower_identifiers_map_sql(
                 ON e.parent_id = walk.concept_id
             {recurse_where}
         )
-        SELECT DISTINCT walk.root_id, c.identifier
+        SELECT DISTINCT walk.root_id, walk.concept_id AS gid, c.identifier
         FROM walk
         JOIN {concepts_table} AS c
             ON c.concept_id = walk.concept_id

{norm_toolkit-1.9.1 → norm_toolkit-1.9.2}/src/norm_toolkit/v2/schema.py

@@ -174,6 +174,15 @@ ONTOLOGY_PREF_STT = "PF"
 DEFAULT_ONTOLOGY_PREF_RANK = 3
 DEFAULT_ONTOLOGY_SYNONYM_RANK = 1
 
+# Struct for a single expanded descendant concept, carrying both the internal
+# dense gid and the source-local identifier.
+NARROWER_STRUCT_TYPE = pl.Struct(
+    {
+        "gid": CONCEPT_ID_DTYPE,
+        "identifier": pl.Utf8,
+    }
+)
+
 # Polars struct type for normalized hits. The first nine fields are produced by
 # the lookup query (see LOOKUP_HIT_FIELDS in normalizer_utils); pref_name,
 # description, and synonyms are filled afterwards by hit enrichment. `narrower`
@@ -193,7 +202,7 @@ HIT_STRUCT_TYPE = pl.Struct(
         "pref_name": pl.Utf8,
         "description": pl.Utf8,
         "synonyms": pl.List(pl.Utf8),
-        "narrower": pl.List(pl.Utf8),
+        "narrower": pl.List(NARROWER_STRUCT_TYPE),
     }
 )