resolvekit 0.0.1__py3-none-any.whl

resolvekit/api/resolver.py

@@ -0,0 +1,457 @@
+"""Main Resolver API."""
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from resolvekit.calibration.calibrator import Calibrator
+from resolvekit.calibration.models import load_calibration_model
+from resolvekit.constraints.constraint_engine import ConstraintEngine
+from resolvekit.data.alias_repository import AliasRepository
+from resolvekit.data.code_repository import CodeRepository
+from resolvekit.data.db_manager import DatabaseManager
+from resolvekit.data.entity_repository import EntityRepository
+from resolvekit.data.membership_repository import MembershipRepository
+from resolvekit.matchers.alias_exact import AliasExactMatcher
+from resolvekit.matchers.canonical_name import CanonicalNameMatcher
+from resolvekit.matchers.cascade import MatcherCascade
+from resolvekit.matchers.exact_code import ExactCodeMatcher
+from resolvekit.matchers.fts_matcher import FTSMatcher
+from resolvekit.matchers.fuzzy_matcher import FuzzyMatcher
+from resolvekit.normalization.normalizer import TextNormalizer
+from resolvekit.types import (
+    Candidate,
+    EntityType,
+    Explanation,
+    ExplanationMode,
+    MatchContext,
+    Resolution,
+)
+from resolvekit.utils.dates import parse_date
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+class Resolver:
+    """
+    Main entity resolution API.
+
+    Orchestrates matchers, constraints, and calibration to resolve
+    entity strings to canonical entities with confidence scores.
+    """
+
+    def __init__(
+        self,
+        *,
+        data_pack: str | Path | None = None,
+        overlays: list[str | Path] | None = None,
+        min_confidence: float = 0.5,
+        explanation_mode: ExplanationMode = ExplanationMode.STANDARD,
+        top_k: int = 12,
+        calibration_model: str | Path | None = None,
+    ):
+        """
+        Initialize resolver with data pack and configuration.
+
+        Args:
+            data_pack: Optional path to data pack. If None, uses default bundled pack.
+            overlays: Optional list of overlay data packs (user/org customizations)
+            min_confidence: Minimum confidence threshold (default 0.5)
+            explanation_mode: Level of detail in explanations
+            top_k: Maximum candidates to consider
+            calibration_model: Optional path to calibration model
+        """
+        self.min_confidence = min_confidence
+        self.explanation_mode = explanation_mode
+        self.top_k = top_k
+
+        # Initialize normalizer (needed by repositories)
+        self.normalizer = TextNormalizer()
+
+        # Initialize database and repositories
+        # TODO: Auto-discover default data pack if None
+        if data_pack is not None:
+            data_pack_path = (
+                Path(data_pack) if isinstance(data_pack, str) else data_pack
+            )
+        else:
+            # TODO: Auto-discover default data pack
+            data_pack_path = Path("data.db")  # Temporary placeholder
+
+        # Convert overlays to Path objects
+        overlay_paths: list[Path] | None = None
+        if overlays:
+            overlay_paths = [
+                Path(overlay) if isinstance(overlay, str) else overlay
+                for overlay in overlays
+            ]
+
+        self.db_manager = DatabaseManager(data_pack_path, overlays=overlay_paths)
+        self.db_manager.connect()
+        self.entity_repo = EntityRepository(self.db_manager)
+        self.alias_repo = AliasRepository(self.db_manager, self.normalizer)
+        self.code_repo = CodeRepository(self.db_manager, self.entity_repo)
+        self.membership_repo = MembershipRepository(self.db_manager)
+
+        # Initialize matchers
+        self.exact_code = ExactCodeMatcher(self.code_repo)
+        self.canonical_name = CanonicalNameMatcher(self.entity_repo, self.normalizer)
+        self.alias_exact = AliasExactMatcher(self.alias_repo)
+        self.fts = FTSMatcher(self.alias_repo)
+        self.fuzzy = FuzzyMatcher(self.normalizer)
+
+        # Initialize cascade
+        self.cascade = MatcherCascade(
+            exact_code=self.exact_code,
+            canonical_name=self.canonical_name,
+            alias_exact=self.alias_exact,
+            fts=self.fts,
+            fuzzy=self.fuzzy,
+            normalizer=self.normalizer,
+        )
+
+        # Initialize constraint engine
+        self.constraint_engine = ConstraintEngine(
+            entity_repo=self.entity_repo,
+            membership_repo=self.membership_repo,
+        )
+
+        # Initialize calibrator
+        if calibration_model:
+            calibration_model_path = (
+                Path(calibration_model)
+                if isinstance(calibration_model, str)
+                else calibration_model
+            )
+            model = load_calibration_model(calibration_model_path)
+            self.calibrator = Calibrator(model)
+        else:
+            # Heuristic mode (no model)
+            self.calibrator = Calibrator()
+
+    def resolve(
+        self,
+        query: str,
+        context: MatchContext | None = None,
+    ) -> Resolution:
+        """
+        Resolve single entity query.
+
+        Args:
+            query: Entity string to resolve
+            context: Optional match context for filtering/disambiguation
+
+        Returns:
+            Resolution result
+        """
+        # Stage 1: Cascade (normalization + matching)
+        candidates = self.cascade.resolve(query, context=context, top_k=self.top_k)
+
+        # No candidates found
+        if not candidates:
+            return Resolution(
+                entity=None,
+                confidence=0.0,
+                alternatives=[],
+                explanation=self._build_explanation_empty(query)
+                if self.explanation_mode != ExplanationMode.MINIMAL
+                else None,
+            )
+
+        # Stage 2: Constraint engine (KG/temporal validation)
+        candidates = self.constraint_engine.apply_constraints(candidates, context)
+
+        # All filtered by constraints
+        if not candidates:
+            return Resolution(
+                entity=None,
+                confidence=0.0,
+                alternatives=[],
+                explanation=self._build_explanation_filtered(query)
+                if self.explanation_mode != ExplanationMode.MINIMAL
+                else None,
+            )
+
+        # Stage 3: Calibration (score → confidence probability)
+        calibrated_scores = self.calibrator.calibrate_batch(candidates)
+
+        # Stage 4: Sort by calibrated confidence
+        sorted_candidates = sorted(
+            zip(candidates, calibrated_scores, strict=False),
+            key=lambda x: x[1],
+            reverse=True,
+        )
+
+        # Stage 5: Threshold filtering
+        top_candidate, top_confidence = sorted_candidates[0]
+        entity = None if top_confidence < self.min_confidence else top_candidate.entity
+
+        # Stage 6: Build alternatives (next 2-5 candidates)
+        alternatives = [c.entity for c, score in sorted_candidates[1:6]]
+
+        # Stage 7: Build explanation
+        explanation = self._build_explanation(
+            top_candidate,
+            sorted_candidates,
+            query,
+        )
+
+        return Resolution(
+            entity=entity,
+            confidence=top_confidence,
+            alternatives=alternatives,
+            explanation=explanation,
+        )
+
+    def _build_explanation_empty(self, query: str) -> Explanation | None:
+        """Build explanation for empty results."""
+        if self.explanation_mode == ExplanationMode.MINIMAL:
+            return None
+        return Explanation(
+            stages=["cascade"],
+            trace={"query": query, "reason": "no_candidates"},
+        )
+
+    def _build_explanation_filtered(self, query: str) -> Explanation | None:
+        """Build explanation for constraint-filtered results."""
+        if self.explanation_mode == ExplanationMode.MINIMAL:
+            return None
+        return Explanation(
+            stages=["cascade", "constraints"],
+            trace={"query": query, "reason": "filtered_by_constraints"},
+        )
+
+    def _build_explanation(
+        self,
+        top_candidate: Candidate,
+        sorted_candidates: list[tuple[Candidate, float]],
+        query: str,
+    ) -> Explanation | None:
+        """
+        Build explanation based on mode.
+
+        Args:
+            top_candidate: Top candidate
+            sorted_candidates: All candidates with scores
+            query: Original query
+
+        Returns:
+            Explanation or None (for MINIMAL mode)
+        """
+        if self.explanation_mode == ExplanationMode.MINIMAL:
+            return None
+
+        # STANDARD mode
+        explanation = Explanation(
+            matcher_used=top_candidate.matcher_type,
+            features=top_candidate.features.copy(),
+            stages=["cascade", "constraints", "calibration"],
+            candidates=[c for c, _ in sorted_candidates[1:4]],  # Next 3 as alternatives
+        )
+
+        # FULL mode - add trace
+        if self.explanation_mode == ExplanationMode.FULL:
+            explanation.trace = {
+                "query": query,
+                "total_candidates": len(sorted_candidates),
+                "all_candidates": [
+                    {
+                        "dcid": c.entity.dcid,
+                        "name": c.entity.canonical_name,
+                        "score": float(score),
+                        "matcher": c.matcher_type.value,
+                    }
+                    for c, score in sorted_candidates
+                ],
+            }
+
+        return explanation
+
+    def resolve_many(
+        self,
+        queries: list[str],
+        context: MatchContext | list[MatchContext | None] | None = None,
+    ) -> list[Resolution]:
+        """
+        Resolve multiple queries.
+
+        Args:
+            queries: List of query strings
+            context: Optional context - applies to all queries if MatchContext,
+                    or per-query if list (must match length of queries)
+
+        Returns:
+            List of Resolution objects (same order as queries)
+        """
+        # Validate per-query context length
+        if isinstance(context, list):
+            if len(context) != len(queries):
+                raise ValueError(
+                    f"context list length ({len(context)}) must match queries length ({len(queries)})"
+                )
+            # Per-query contexts - resolve individually
+            return [
+                self.resolve(query, ctx)
+                for query, ctx in zip(queries, context, strict=True)
+            ]
+
+        # Shared context - deduplicate queries for efficiency
+        unique_queries = list(dict.fromkeys(queries))
+
+        # Resolve only unique queries
+        unique_resolutions = {
+            query: self.resolve(query, context) for query in unique_queries
+        }
+
+        # Map results back to original query order
+        return [unique_resolutions[query] for query in queries]
+
+    def __enter__(self) -> "Resolver":
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        """Context manager exit - clean up resources."""
+        # Close database connections
+        if hasattr(self, "db_manager"):
+            self.db_manager.close()
+
+    def resolve_dataframe(  # noqa: PLR0912
+        self,
+        df: "pd.DataFrame",
+        query_column: str,
+        resolve_to: str | list[str] = "dcid",
+        *,
+        context_columns: dict[str, str] | None = None,
+        context: MatchContext | None = None,
+        include_confidence: bool = True,
+        output_prefix: str = "",
+    ) -> "pd.DataFrame":
+        """
+        Resolve entities from DataFrame column to specified code system(s).
+
+        Args:
+            df: Input DataFrame
+            query_column: Column name containing entity strings to resolve
+            resolve_to: What to resolve to (default: "dcid")
+            context_columns: Map MatchContext fields to DataFrame columns
+            context: Single MatchContext for all rows
+            include_confidence: Add confidence score column
+            output_prefix: Prefix for output columns
+
+        Returns:
+            DataFrame with resolution results added as columns
+        """
+        # Import pandas only when needed
+        try:
+            import pandas as pd
+        except ImportError as e:
+            raise ImportError("pandas is required for resolve_dataframe(). ") from e
+
+        # Validate inputs
+        if query_column not in df.columns:
+            raise ValueError(f"Column '{query_column}' not found in DataFrame")
+
+        # Normalize resolve_to to list
+        if isinstance(resolve_to, str):
+            resolve_to = [resolve_to]
+
+        # Deduplication optimization
+        if context_columns is None:
+            # Simple case: shared context, dedupe on query only
+            unique_queries = df[query_column].unique().tolist()
+            unique_resolutions = self.resolve_many(unique_queries, context=context)
+
+            # Create lookup dict
+            lookup = dict(zip(unique_queries, unique_resolutions, strict=True))
+
+            # Map back to all rows
+            resolutions = [lookup[q] for q in df[query_column]]
+        else:
+            # Build per-row contexts from context_columns
+            queries = df[query_column].tolist()
+
+            # Validate context columns exist
+            for field_name, col_name in context_columns.items():
+                if col_name not in df.columns:
+                    raise ValueError(
+                        f"Context column '{col_name}' (for field '{field_name}') not found in DataFrame"
+                    )
+
+            # Build MatchContext for each row
+            # Extract only the columns we need and convert to list of dicts
+            context_df = df[list(context_columns.values())]
+            records = context_df.to_dict("records")
+
+            # Map column names to field names for lookup
+            col_to_field = {
+                col_name: field_name for field_name, col_name in context_columns.items()
+            }
+
+            # Build contexts from records
+            contexts = []
+            for record in records:
+                context_kwargs: dict[str, Any] = {}
+                for col_name_raw, value in record.items():
+                    col_name = str(col_name_raw)  # Convert Hashable to str
+                    if pd.isna(value):
+                        continue
+
+                    field_name = col_to_field[col_name]
+
+                    # Type conversions
+                    if field_name == "entity_type":
+                        context_kwargs[field_name] = EntityType(value)
+                    elif field_name == "as_of":
+                        context_kwargs[field_name] = parse_date(value)
+                    else:
+                        context_kwargs[field_name] = value
+
+                contexts.append(
+                    MatchContext(**context_kwargs) if context_kwargs else None
+                )
+
+            resolutions = self.resolve_many(queries, context=contexts)
+
+        # Build output DataFrame
+        return self._build_output_dataframe(
+            df, resolutions, resolve_to, include_confidence, output_prefix
+        )
+
+    def _build_output_dataframe(
+        self,
+        df: "pd.DataFrame",
+        resolutions: list[Resolution],
+        resolve_to: list[str],
+        include_confidence: bool,
+        output_prefix: str,
+    ) -> "pd.DataFrame":
+        """Build output DataFrame with resolution results."""
+
+        # Copy input DataFrame
+        result_df = df.copy()
+
+        # Extract requested fields from resolutions
+        for field in resolve_to:
+            col_name = f"{output_prefix}{field}"
+            values: list[str | None] = []
+
+            for res in resolutions:
+                if res.entity is None:
+                    values.append(None)
+                elif field == "dcid":
+                    values.append(res.entity.dcid)
+                elif field == "canonical_name":
+                    values.append(res.entity.canonical_name)
+                else:
+                    # Code system (iso3, m49, etc.)
+                    values.append(res.entity.codes.get(field))
+
+            result_df[col_name] = values
+
+        # Add confidence column
+        if include_confidence:
+            conf_col = f"{output_prefix}confidence"
+            result_df[conf_col] = [r.confidence for r in resolutions]
+
+        return result_df

resolvekit/builders/README.md

@@ -0,0 +1,173 @@
+# Builders Module
+
+## Purpose
+
+The builders module contains tools for building data packs from source data, including ETL pipelines, validation, and packaging.
+
+## Components
+
+### Core Builders
+
+1. **Pack Builder** (`pack_builder.py`)
+   - Orchestrates full data pack build process
+   - Runs extract, transform, validate, enrich, package stages
+   - Generates manifests and checksums
+
+2. **Schema Builder** (`schema_builder.py`)
+   - Creates SQLite databases with proper schema
+   - Builds FTS5 indexes
+   - Applies optimizations (PRAGMA, indexes)
+
+3. **Calibration Trainer** (`calibration_trainer.py`)
+   - Trains calibration models on labeled data
+   - Evaluates model quality (ECE, Brier score)
+   - Exports models to JSON
+
+4. **Sidecar Builder** (`sidecar_builder.py`)
+   - Builds ambiguity sidecar (HNSW index)
+   - Curates ambiguous aliases
+   - Generates embeddings and quantizes (Phase E)
+
+### ETL Pipeline
+
+- `extractors.py`: Extract data from sources (APIs, dumps, files)
+- `transformers.py`: Normalize, align, resolve conflicts
+- `validators.py`: Referential integrity, format validation
+- `enrichers.py`: Derive hierarchies, compute statistics
+- `packagers.py`: Create archives, generate metadata
+
+### Data Sources
+
+- `sources/`: Source-specific extractors
+  - `data_commons.py`: Data Commons entity extraction
+  - `iso.py`: ISO 3166 codes
+  - `wikidata.py`: Wikidata aliases and temporal data
+  - `geonames.py`: GeoNames data
+  - `custom.py`: Custom data ingestion
+
+### Quality Assurance
+
+- `quality_checks.py`: Data quality validation
+- `coverage_metrics.py`: Coverage analysis
+- `conflict_resolver.py`: Handle source conflicts
+- `deduplicator.py`: Entity deduplication
+
+## Build Pipeline Stages
+
+### 1. Extract
+
+```python
+# Extract from multiple sources
+entities_dc = extract_data_commons()
+entities_iso = extract_iso_codes()
+entities_wd = extract_wikidata()
+```
+
+### 2. Transform
+
+```python
+# Normalize and align entities
+entities = align_entities(entities_dc, entities_iso, entities_wd)
+entities = normalize_text(entities)
+entities = resolve_conflicts(entities, precedence_rules)
+```
+
+### 3. Validate
+
+```python
+# Validate data quality
+check_referential_integrity(entities)
+check_code_formats(entities)
+check_temporal_consistency(entities)
+flag_suspicious_data(entities)
+```
+
+### 4. Enrich
+
+```python
+# Compute derived data
+compute_hierarchy_paths(entities)
+build_fts_indexes(entities)
+generate_statistics(entities)
+```
+
+### 5. Package
+
+```python
+# Create data pack
+export_to_sqlite(entities, "base.sqlite")
+build_manifest(pack_info)
+compute_checksums(files)
+create_archive("resolvekit-data-1.2.0.tar.gz")
+```
+
+## CLI for Building
+
+```bash
+# Build full data pack
+resolvekit-build pack \
+    --sources wikidata,iso,datacommons \
+    --output packs/1.2.0/ \
+    --validate
+
+# Build overlay from CSV
+resolvekit-build overlay \
+    --input custom_aliases.csv \
+    --name my-custom-pack \
+    --output overlays/
+
+# Train calibration model
+resolvekit-build calibration \
+    --training-data labeled_pairs.csv \
+    --output calibration.json
+
+# Build ambiguity sidecar
+resolvekit-build sidecar \
+    --registry ambiguity_registry.csv \
+    --base base.sqlite \
+    --output ambiguity.hnsw
+```
+
+## Manifest Generation
+
+```python
+manifest = {
+    "pack_name": "resolvekit-core-countries",
+    "version": "1.2.0",
+    "build_time": datetime.now(UTC).isoformat(),
+    "schema_hash": compute_schema_hash(),
+    "sources": [
+        {"name": "DataCommons", "snapshot": "2025-09-15"},
+        {"name": "ISO3166", "snapshot": "2025-09-01"}
+    ],
+    "components": {
+        "base_sqlite": "base.sqlite",
+        "fts_built": True,
+        "ambiguity_registry": True,
+        "calibration": "calibration.json"
+    },
+    "checksums": {
+        "base.sqlite": compute_sha256("base.sqlite"),
+        "calibration.json": compute_sha256("calibration.json")
+    },
+    "license": "CC-BY 4.0 (data), Apache-2.0 (code)",
+    "compat": {
+        "resolver_min": "0.9.0",
+        "resolver_max": "<2.0.0"
+    }
+}
+```
+
+## Design Principles
+
+1. **Automated**: CI/CD pipeline for regular builds
+2. **Reproducible**: Same sources → same output
+3. **Validated**: Quality gates prevent bad data
+4. **Versioned**: Semantic versioning for packs
+
+## Implementation Priority
+
+**Phase A** - Basic schema builder
+**Phase C** - Overlay builders
+**Phase D** - Full pack building and distribution
+**Phase E** - Sidecar builder