resolvekit 0.0.1__py3-none-any.whl

resolvekit/calibration/models.py

@@ -0,0 +1,78 @@
+"""Calibration model data structures."""
+
+import json
+from datetime import date
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
+
+from resolvekit.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class CalibrationModel(BaseModel):
+    """
+    Calibration model configuration.
+
+    Loaded from JSON files in data packs. Defines logistic regression
+    parameters for converting features to calibrated probabilities.
+
+    Attributes:
+        type: Model type (currently only "logistic" supported)
+        features: Ordered list of feature names
+        weights: Logistic regression weights (one per feature)
+        bias: Logistic regression bias term
+        ece: Expected Calibration Error (optional, for model quality tracking)
+        trained_on: Date model was trained (optional)
+        notes: Human-readable notes (optional)
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["logistic"] = Field(
+        default="logistic", description="Calibration model type"
+    )
+    features: list[str] = Field(..., min_length=1, description="Feature names in order")
+    weights: list[float] = Field(..., min_length=1, description="Logistic weights")
+    bias: float = Field(..., description="Logistic bias term")
+    ece: float | None = Field(
+        None, ge=0.0, le=1.0, description="Expected Calibration Error"
+    )
+    trained_on: date | None = Field(None, description="Training date")
+    notes: str | None = Field(None, description="Human-readable notes")
+
+    @model_validator(mode="after")
+    def validate_weights_match_features(self) -> "CalibrationModel":
+        """Validate that weights count matches features count."""
+        if len(self.weights) != len(self.features):
+            raise ValueError(
+                f"Weights count ({len(self.weights)}) must match "
+                f"features count ({len(self.features)})"
+            )
+        return self
+
+
+def load_calibration_model(path: Path) -> CalibrationModel | None:
+    """
+    Load calibration model from JSON file.
+
+    Args:
+        path: Path to calibration_model.json file
+
+    Returns:
+        CalibrationModel if file exists and is valid, None otherwise
+    """
+    if not path.exists():
+        return None
+
+    try:
+        data = json.loads(path.read_text())
+        return CalibrationModel.model_validate(data)
+    except json.JSONDecodeError as e:
+        logger.warning(f"Invalid JSON in calibration model at {path}: {e}")
+        return None
+    except ValidationError as e:
+        logger.warning(f"Invalid calibration model schema at {path}: {e}")
+        return None

resolvekit/cli/README.md

@@ -0,0 +1,215 @@
+# CLI Module
+
+## Purpose
+
+The CLI module provides a command-line interface for interactive entity resolution, batch processing, and data pack management.
+
+## Components
+
+### Command Modules
+
+1. **Main CLI** (`main.py`)
+   - Entry point for CLI application
+   - Uses `click` or `typer` for argument parsing
+   - Global options and configuration
+
+2. **Query Command** (`query.py`)
+   - Single entity resolution from command line
+   - Interactive disambiguation prompts
+   - Human-readable output with colors
+
+3. **Convert Command** (`convert.py`)
+   - Code system conversion
+   - Supports single codes or batch files
+
+4. **Batch Command** (`batch.py`)
+   - Batch processing of CSV/JSON files
+   - Progress bars for large files
+   - Error reporting and partial results
+
+5. **Extract Command** (`extract.py`)
+   - Entity extraction from text files
+   - Supports various input formats
+   - (Phase F)
+
+6. **Info Command** (`info.py`)
+   - Display detailed entity information
+   - Show codes, hierarchy, memberships
+
+7. **Members Command** (`members.py`)
+   - List group members
+   - Temporal support (as_of date)
+
+8. **Update Command** (`update.py`)
+   - Check for data pack updates
+   - Download and install new versions
+
+9. **Version Command** (`version.py`)
+   - Show tool and data pack versions
+   - Component information
+
+### Output Formatters
+
+- `formatters.py`: Output formatting (table, JSON, CSV)
+- `colors.py`: Terminal color support
+- `progress.py`: Progress bars for long operations
+
+## CLI Commands
+
+### Query
+
+```bash
+# Simple query
+resolvekit query "Georgia"
+
+# Output:
+# ⚠️  Ambiguous input. Top matches:
+#
+# 1. Georgia (Country) [dcid: country/GEO]
+#    - Confidence: 0.60
+#    - Codes: ISO2=GE, ISO3=GEO, M49=268
+#
+# 2. Georgia (US State) [dcid: geoId/13]
+#    - Confidence: 0.40
+#    - Codes: FIPS=13
+
+# With context
+resolvekit query "Georgia" --context "Tbilisi"
+
+# With constraints
+resolvekit query "Paris" --entity-type country
+resolvekit query "Ontario" --parent country/CAN
+
+# JSON output
+resolvekit query "France" --format json
+
+# With explanation
+resolvekit query "Türkiye" --explain
+```
+
+### Convert
+
+```bash
+# Single code conversion
+resolvekit convert FRA --from iso3 --to dcid
+# Output: country/FRA
+
+# Get all codes
+resolvekit convert country/FRA --from dcid
+# Output:
+# iso2: FR
+# iso3: FRA
+# m49: 250
+# nuts: FR
+# wikidata: Q142
+
+# Batch file
+resolvekit convert codes.csv --from iso3 --to dcid --output results.csv
+```
+
+### Batch
+
+```bash
+# Process CSV file
+resolvekit batch input.csv \
+    --column location \
+    --output enriched.csv \
+    --min-confidence 0.8
+
+# With additional context columns
+resolvekit batch data.csv \
+    --column country_name \
+    --context-column region \
+    --output results.csv
+
+# JSON input/output
+resolvekit batch input.json --format json --output results.json
+```
+
+### Info
+
+```bash
+# Entity details
+resolvekit info country/FRA
+
+# Output:
+# Entity: France
+# DCID: country/FRA
+# Type: country
+# Codes:
+#   ISO2: FR
+#   ISO3: FRA
+#   M49: 250
+# Parent: None
+# Children (ADM1): 18 regions
+# Memberships: EU, OECD, G7, G20, ...
+```
+
+### Members
+
+```bash
+# Current members
+resolvekit members EU
+
+# Historical members
+resolvekit members EU --as-of 2004-01-01
+
+# Output as list of DCIDs
+resolvekit members OECD --format dcids
+```
+
+### Update
+
+```bash
+# Check for updates
+resolvekit update --check
+
+# Download and install latest
+resolvekit update --install
+
+# List available versions
+resolvekit update --list
+```
+
+### Version
+
+```bash
+# Show versions
+resolvekit version
+
+# Output:
+# resolvekit: 0.1.0
+# Data pack: 1.2.0 (2025-10-20)
+# Python: 3.13
+```
+
+## Output Formats
+
+### Human-readable (default)
+- Colored terminal output
+- Formatted tables
+- Progress indicators
+
+### JSON
+- Machine-readable
+- Full result structure
+- Suitable for piping
+
+### CSV
+- Batch results
+- Compatible with spreadsheets
+- Configurable columns
+
+## Design Principles
+
+1. **Unix-friendly**: Composable with pipes, appropriate exit codes
+2. **Interactive**: Clear prompts for ambiguous cases
+3. **Informative**: Helpful error messages with suggestions
+4. **Fast feedback**: Show progress for long operations
+
+## Implementation Priority
+
+**Phase A** - Core resolver (query, batch, version)
+**Phase B** - Code conversion (convert, info, members)
+**Phase D** - Package updates (update)
+**Phase F** - Entity extraction (extract)

resolvekit/cli/main.py

@@ -0,0 +1,18 @@
+"""CLI entry point for resolvekit."""
+
+import sys
+
+
+def cli() -> None:
+    """Main CLI entry point."""
+    print("resolvekit CLI - Coming soon!")
+    print()
+    print("Available commands (to be implemented):")
+    print("  resolvekit query <text>         - Resolve entity from text")
+    print("  resolvekit convert <code>       - Convert between code systems")
+    print("  resolvekit batch <file>         - Batch process CSV/JSON file")
+    print("  resolvekit info <dcid>          - Show entity information")
+    print("  resolvekit members <group>      - List group members")
+    print("  resolvekit update               - Update data packs")
+    print("  resolvekit version              - Show version information")
+    sys.exit(0)

resolvekit/config.py

@@ -0,0 +1,128 @@
+"""Configuration management for resolvekit using Pydantic Settings."""
+
+from pathlib import Path
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class ResolvekitConfig(BaseSettings):
+    """
+    Configuration for resolvekit resolver.
+
+    This uses Pydantic Settings to read configuration from:
+    1. Environment variables (prefixed with RESOLVEKIT_)
+    2. .env file (if present)
+    3. Default values
+
+    Example:
+        # Set via environment
+        export RESOLVEKIT_DATA_DIR="/path/to/data"
+        export RESOLVEKIT_MIN_CONFIDENCE=0.8
+
+        # Or in .env file
+        RESOLVEKIT_DATA_DIR=/path/to/data
+        RESOLVEKIT_MIN_CONFIDENCE=0.8
+
+        # Use in code
+        config = ResolvekitConfig()
+        print(config.data_dir)  # Path('/path/to/data')
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="RESOLVEKIT_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+    )
+
+    # Paths
+    data_dir: Path = Field(
+        default=Path.home() / ".resolvekit" / "data",
+        description="Directory containing data packs",
+    )
+    cache_dir: Path = Field(
+        default=Path.home() / ".resolvekit" / "cache",
+        description="Cache directory",
+    )
+
+    # Resolution settings
+    min_confidence: float = Field(
+        default=0.7,
+        ge=0.0,
+        le=1.0,
+        description="Minimum confidence threshold for matches",
+    )
+    max_candidates: int = Field(
+        default=50,
+        gt=0,
+        le=1000,
+        description="Maximum number of candidates to consider",
+    )
+    max_alternates: int = Field(
+        default=5,
+        gt=0,
+        le=100,
+        description="Maximum number of alternative matches to return",
+    )
+
+    # FTS settings
+    fts_top_k: int = Field(
+        default=50,
+        gt=0,
+        le=1000,
+        description="Top K results from FTS",
+    )
+    fuzzy_top_k: int = Field(
+        default=12,
+        gt=0,
+        le=100,
+        description="Top K results after fuzzy matching",
+    )
+
+    # Ambiguity detection
+    margin_threshold: float = Field(
+        default=0.15,
+        ge=0.0,
+        le=1.0,
+        description="Score margin threshold for ambiguity detection",
+    )
+
+    # Logging
+    log_level: str = Field(
+        default="INFO",
+        pattern=r"^(DEBUG|INFO|WARNING|ERROR|CRITICAL)$",
+        description="Logging level",
+    )
+    log_format: str = Field(
+        default="human",
+        pattern=r"^(human|json)$",
+        description="Log format (human or json)",
+    )
+    redact_queries: bool = Field(
+        default=True,
+        description="Redact query content in logs for privacy",
+    )
+
+    # Performance
+    batch_size: int = Field(
+        default=1000,
+        gt=0,
+        le=100000,
+        description="Batch size for batch processing",
+    )
+    num_workers: int = Field(
+        default=4,
+        gt=0,
+        le=64,
+        description="Number of worker processes for parallel processing",
+    )
+
+    def ensure_dirs(self) -> None:
+        """Create data and cache directories if they don't exist."""
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+
+# Global config instance (can be overridden)
+config = ResolvekitConfig()

resolvekit/constants.py

@@ -0,0 +1,252 @@
+"""Constants for resolvekit."""
+
+from resolvekit.types import EntityType
+
+# Version
+VERSION = "0.0.1"
+
+# Default configuration
+DEFAULT_MIN_CONFIDENCE = 0.7
+DEFAULT_MAX_CANDIDATES = 50
+DEFAULT_MAX_ALTERNATES = 5
+DEFAULT_FTS_TOP_K = 50
+DEFAULT_FUZZY_TOP_K = 12
+
+# Ambiguity detection
+DEFAULT_MARGIN_THRESHOLD = 0.15  # Trigger semantic if margin < this
+
+# File paths
+DEFAULT_DATA_DIR = "~/.resolvekit/data"
+DEFAULT_CACHE_DIR = "~/.resolvekit/cache"
+
+# Database configuration
+DB_PRAGMAS = {
+    "journal_mode": "OFF",
+    "synchronous": "OFF",
+    "temp_store": "MEMORY",
+    "mmap_size": 268435456,  # ~256MB
+    "cache_size": -100000,  # ~100MB
+}
+
+# FTS5 configuration
+FTS5_TOKENIZER = "unicode61 remove_diacritics 2 tokenchars '.-'"
+FTS5_PREFIX = "2,3"
+
+# Performance targets (milliseconds)
+TARGET_LATENCY_P50 = 10
+TARGET_LATENCY_P95 = 50
+TARGET_STARTUP_TIME = 5000  # 5 seconds
+
+# Memory targets (MB)
+TARGET_MEMORY_BASE = 2048  # 2GB
+TARGET_MEMORY_EMBEDDINGS = 4096  # 4GB
+TARGET_MEMORY_FULL = 6144  # 6GB
+
+# Batch processing
+DEFAULT_BATCH_SIZE = 1000
+DEFAULT_NUM_WORKERS = 4
+
+# Entity type hierarchy
+ENTITY_TYPE_HIERARCHY = {
+    EntityType.COUNTRY: [
+        EntityType.ADMIN1,
+        EntityType.ADMIN2,
+        EntityType.ADMIN3,
+        EntityType.ADMIN4,
+        EntityType.CITY,
+    ],
+    EntityType.ADMIN1: [
+        EntityType.ADMIN2,
+        EntityType.ADMIN3,
+        EntityType.ADMIN4,
+        EntityType.CITY,
+    ],
+    EntityType.ADMIN2: [EntityType.ADMIN3, EntityType.ADMIN4, EntityType.CITY],
+    EntityType.ADMIN3: [EntityType.ADMIN4, EntityType.CITY],
+    EntityType.ADMIN4: [EntityType.CITY],
+    EntityType.ORGANIZATION: [EntityType.GROUP],
+}
+
+# Code system metadata
+CODE_SYSTEMS = {
+    "dcid": {
+        "name": "Data Commons ID",
+        "authority": "Google Data Commons",
+        "format": r"^[a-zA-Z]+/[A-Z0-9]+$",
+        "example": "country/USA",
+    },
+    "iso2": {
+        "name": "ISO 3166-1 alpha-2",
+        "authority": "ISO",
+        "format": r"^[A-Z]{2}$",
+        "example": "US",
+    },
+    "iso3": {
+        "name": "ISO 3166-1 alpha-3",
+        "authority": "ISO",
+        "format": r"^[A-Z]{3}$",
+        "example": "USA",
+    },
+    "iso_numeric": {
+        "name": "ISO 3166-1 numeric",
+        "authority": "ISO",
+        "format": r"^\d{3}$",
+        "example": "840",
+    },
+    "m49": {
+        "name": "UN M49",
+        "authority": "UN Statistics Division",
+        "format": r"^\d{3}$",
+        "example": "840",
+    },
+    "nuts": {
+        "name": "NUTS codes",
+        "authority": "Eurostat",
+        "format": r"^[A-Z]{2}[A-Z0-9]{0,3}$",
+        "example": "FR",
+    },
+    "pcode": {
+        "name": "OCHA P-codes",
+        "authority": "UN OCHA",
+        "format": r"^[A-Z]{2}[A-Z0-9]+$",
+        "example": "FR01",
+    },
+    "wikidata": {
+        "name": "Wikidata QID",
+        "authority": "Wikidata",
+        "format": r"^Q\d+$",
+        "example": "Q30",
+    },
+    "geonames": {
+        "name": "GeoNames ID",
+        "authority": "GeoNames",
+        "format": r"^\d+$",
+        "example": "6252001",
+    },
+}
+
+# Source precedence (higher = higher priority)
+SOURCE_PRECEDENCE = {
+    "user": 100,
+    "custom": 90,
+    "datacommons": 80,
+    "iso": 70,
+    "un": 60,
+    "eurostat": 60,
+    "ocha": 60,
+    "wikidata": 50,
+    "geonames": 40,
+    "other": 10,
+}
+
+# Overlay precedence ranges
+OVERLAY_PRECEDENCE = {
+    "user": (100, 999),
+    "org": (10, 99),
+    "base": (0, 0),
+}
+
+# Calibration features
+CALIBRATION_FEATURES = [
+    "f_exact_code",
+    "f_canonical_exact",
+    "f_alias_exact",
+    "f_alias_type_canonical",
+    "f_alias_type_endonym",
+    "f_alias_type_exonym",
+    "f_alias_type_abbr",
+    "f_alias_type_code",
+    "f_fts_score",
+    "f_fts_rank_inv",
+    "f_edit_distance_norm",
+    "f_trigram_jaccard",
+    "f_parent_valid",
+    "f_type_valid",
+    "f_date_valid",
+    "f_sem_used",
+    "f_sem_sim",
+    "f_ambiguity_flag",
+    "f_region_hint_match",
+]
+
+# Known ambiguous terms (partial list - full list in ambiguity registry)
+KNOWN_AMBIGUOUS = {
+    "georgia": ["country/GEO", "geoId/13"],
+    "congo": ["country/COD", "country/COG"],
+    "guinea": ["country/GIN", "country/GNQ", "country/GNB", "country/PNG"],
+    "korea": ["country/KOR", "country/PRK"],
+    "macedonia": ["country/MKD", "geoId/GR-MAC"],
+    "springfield": [],  # Too many to list
+}
+
+# Common stopwords that aren't entities
+ENTITY_STOPWORDS = {
+    # Months
+    "january",
+    "february",
+    "march",
+    "april",
+    "may",
+    "june",
+    "july",
+    "august",
+    "september",
+    "october",
+    "november",
+    "december",
+    # Days
+    "monday",
+    "tuesday",
+    "wednesday",
+    "thursday",
+    "friday",
+    "saturday",
+    "sunday",
+    # Common words that look like places
+    "reading",
+    "normal",
+    "mobile",
+    "phoenix",  # Problematic: city vs. mythical bird
+    "aurora",
+    "victoria",  # Problematic: can be place
+    # Directions
+    "north",
+    "south",
+    "east",
+    "west",
+}
+
+# Data pack manifest schema version
+MANIFEST_SCHEMA_VERSION = "1.0.0"
+
+# Supported languages (ISO 639-1 codes)
+SUPPORTED_LANGUAGES = [
+    "en",  # English
+    "es",  # Spanish
+    "fr",  # French
+    "de",  # German
+    "it",  # Italian
+    "pt",  # Portuguese
+    "nl",  # Dutch
+    "ru",  # Russian (Cyrillic script)
+    "ar",  # Arabic (Arabic script)
+    "zh",  # Chinese (CJK)
+    "ja",  # Japanese (CJK)
+    "ko",  # Korean (Hangul)
+]
+
+# Logging configuration
+LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+# API defaults
+API_RATE_LIMIT = 100  # requests per second
+API_TIMEOUT = 30  # seconds
+API_MAX_BATCH_SIZE = 10000
+
+# CLI defaults
+CLI_MAX_DISPLAY_ALTERNATES = 5
+CLI_COLOR_PRIMARY = "blue"
+CLI_COLOR_SUCCESS = "green"
+CLI_COLOR_WARNING = "yellow"
+CLI_COLOR_ERROR = "red"