geo-intel-offline 1.0.1__py3-none-any.whl

geo_intel_offline/geohash.py

@@ -0,0 +1,150 @@
+"""
+Geohash encoding and decoding for spatial indexing.
+
+Geohash is a geocoding system that encodes latitude/longitude into a string.
+We use it to create a spatial index for fast candidate country filtering.
+
+Design Decision: Using precision level 6 (32-bit geohash) as a balance:
+- Precision ~1.2km × 0.6km (sufficient for country-level resolution)
+- Small index size (~200 countries × few geohashes each)
+- Fast encoding/decoding operations
+"""
+
+from typing import Tuple
+
+# Base32 encoding used by geohash
+BASE32 = "0123456789bcdefghjkmnpqrstuvwxyz"
+
+# Geohash precision for indexing (6 chars = ~1.2km precision)
+GEOHASH_PRECISION = 6
+
+
+def encode(lat: float, lon: float, precision: int = GEOHASH_PRECISION) -> str:
+    """
+    Encode latitude/longitude to geohash string.
+    
+    Args:
+        lat: Latitude (-90 to 90)
+        lon: Longitude (-180 to 180)
+        precision: Number of geohash characters (default 6)
+    
+    Returns:
+        Geohash string
+    """
+    if not (-90 <= lat <= 90):
+        raise ValueError(f"Latitude must be between -90 and 90, got {lat}")
+    if not (-180 <= lon <= 180):
+        raise ValueError(f"Longitude must be between -180 and 180, got {lon}")
+    
+    lat_range = (-90.0, 90.0)
+    lon_range = (-180.0, 180.0)
+    bits = 0
+    bits_per_char = 5
+    geohash = []
+    
+    ch = 0
+    
+    for i in range(precision * bits_per_char):
+        if i % 2 == 0:
+            # Longitude bit
+            mid = (lon_range[0] + lon_range[1]) / 2
+            if lon >= mid:
+                ch |= (1 << (bits_per_char - 1 - (i // 2) % bits_per_char))
+                lon_range = (mid, lon_range[1])
+            else:
+                lon_range = (lon_range[0], mid)
+        else:
+            # Latitude bit
+            mid = (lat_range[0] + lat_range[1]) / 2
+            if lat >= mid:
+                ch |= (1 << (bits_per_char - 1 - (i // 2) % bits_per_char))
+                lat_range = (mid, lat_range[1])
+            else:
+                lat_range = (lat_range[0], mid)
+        
+        if (i + 1) % bits_per_char == 0:
+            geohash.append(BASE32[ch])
+            ch = 0
+    
+    return ''.join(geohash)
+
+
+def decode(geohash: str) -> Tuple[float, float, Tuple[float, float], Tuple[float, float]]:
+    """
+    Decode geohash string to latitude/longitude with bounding box.
+    
+    Args:
+        geohash: Geohash string
+    
+    Returns:
+        Tuple of (lat, lon, lat_range, lon_range)
+    """
+    if not geohash:
+        raise ValueError("Geohash cannot be empty")
+    
+    lat_range = (-90.0, 90.0)
+    lon_range = (-180.0, 180.0)
+    is_even = True
+    
+    for char in geohash:
+        if char not in BASE32:
+            raise ValueError(f"Invalid geohash character: {char}")
+        
+        idx = BASE32.index(char)
+        
+        for j in range(5):
+            bit = (idx >> (4 - j)) & 1
+            if is_even:
+                mid = (lon_range[0] + lon_range[1]) / 2
+                if bit:
+                    lon_range = (mid, lon_range[1])
+                else:
+                    lon_range = (lon_range[0], mid)
+            else:
+                mid = (lat_range[0] + lat_range[1]) / 2
+                if bit:
+                    lat_range = (mid, lat_range[1])
+                else:
+                    lat_range = (lat_range[0], mid)
+            is_even = not is_even
+    
+    lat = (lat_range[0] + lat_range[1]) / 2
+    lon = (lon_range[0] + lon_range[1]) / 2
+    
+    return lat, lon, lat_range, lon_range
+
+
+def get_neighbors(geohash: str) -> list[str]:
+    """
+    Get 8 neighboring geohashes (for border cases).
+    
+    Design Decision: Check neighbors when point-in-polygon fails.
+    This handles edge cases where a point is near geohash boundaries.
+    
+    Args:
+        geohash: Geohash string
+    
+    Returns:
+        List of 8 neighboring geohashes plus the original
+    """
+    lat, lon, lat_range, lon_range = decode(geohash)
+    
+    # Calculate step size from precision
+    lat_step = lat_range[1] - lat_range[0]
+    lon_step = lon_range[1] - lon_range[0]
+    
+    neighbors = []
+    for dlat in [-lat_step, 0, lat_step]:
+        for dlon in [-lon_step, 0, lon_step]:
+            if dlat == 0 and dlon == 0:
+                continue
+            new_lat = lat + dlat
+            new_lon = lon + dlon
+            
+            # Clamp to valid ranges
+            new_lat = max(-90, min(90, new_lat))
+            new_lon = max(-180, min(180, new_lon))
+            
+            neighbors.append(encode(new_lat, new_lon, len(geohash)))
+    
+    return neighbors

geo_intel_offline/hierarchical_resolver.py

@@ -0,0 +1,136 @@
+"""
+Hierarchical resolver for country → state → city resolution.
+
+Extends the base resolver to support multi-level geo-intelligence.
+"""
+
+from typing import Optional, Dict, List, Tuple
+from .resolver import resolve as resolve_country, ResolutionResult
+from .data_loader import get_loader
+
+
+class HierarchicalResult:
+    """Result with country, state/province, and city information."""
+    
+    def __init__(
+        self,
+        country: Optional[str] = None,
+        country_iso2: Optional[str] = None,
+        country_iso3: Optional[str] = None,
+        state: Optional[str] = None,
+        state_code: Optional[str] = None,
+        city: Optional[str] = None,
+        continent: Optional[str] = None,
+        timezone: Optional[str] = None,
+        confidence: float = 0.0
+    ):
+        self.country = country
+        self.country_iso2 = country_iso2
+        self.country_iso3 = country_iso3
+        self.state = state
+        self.state_code = state_code
+        self.city = city
+        self.continent = continent
+        self.timezone = timezone
+        self.confidence = confidence
+    
+    def to_dict(self) -> Dict:
+        """Convert to dictionary."""
+        return {
+            "country": self.country,
+            "country_iso2": self.country_iso2,
+            "country_iso3": self.country_iso3,
+            "state": self.state,
+            "state_code": self.state_code,
+            "city": self.city,
+            "continent": self.continent,
+            "timezone": self.timezone,
+            "confidence": self.confidence
+        }
+
+
+def resolve_hierarchical(
+    lat: float,
+    lon: float,
+    include_states: bool = False,
+    include_cities: bool = False,
+    data_dir: Optional[str] = None
+) -> HierarchicalResult:
+    """
+    Resolve coordinates hierarchically: country → state → city.
+    
+    Args:
+        lat: Latitude
+        lon: Longitude
+        include_states: Whether to resolve state/province (requires state data)
+        include_cities: Whether to resolve city (requires city data)
+        data_dir: Optional custom data directory
+    
+    Returns:
+        HierarchicalResult with country, state, and city information
+    """
+    # First resolve country (always available)
+    country_result = resolve_country(lat, lon, data_dir)
+    
+    if not country_result.is_valid():
+        return HierarchicalResult()
+    
+    result = HierarchicalResult(
+        country=country_result.country_name,
+        country_iso2=country_result.iso2,
+        country_iso3=country_result.iso3,
+        continent=country_result.continent,
+        timezone=country_result.timezone,
+        confidence=country_result.confidence
+    )
+    
+    # Resolve state/province if requested and data available
+    if include_states:
+        state_info = _resolve_state(lat, lon, country_result.country_id, data_dir)
+        if state_info:
+            result.state = state_info.get('name')
+            result.state_code = state_info.get('code')
+            # Adjust confidence (state resolution may be less accurate)
+            result.confidence = min(result.confidence, state_info.get('confidence', result.confidence))
+    
+    # Resolve city if requested and data available
+    if include_cities:
+        city_info = _resolve_city(lat, lon, country_result.country_id, data_dir)
+        if city_info:
+            result.city = city_info.get('name')
+            # Adjust confidence
+            result.confidence = min(result.confidence, city_info.get('confidence', result.confidence))
+    
+    return result
+
+
+def _resolve_state(
+    lat: float,
+    lon: float,
+    country_id: int,
+    data_dir: Optional[str]
+) -> Optional[Dict]:
+    """
+    Resolve state/province within a country.
+    
+    Note: Requires state-level data files (states_index.json, states_polygons.json)
+    """
+    # This would load state data similar to country resolution
+    # For now, return None (state data not yet implemented)
+    return None
+
+
+def _resolve_city(
+    lat: float,
+    lon: float,
+    country_id: int,
+    data_dir: Optional[str]
+) -> Optional[Dict]:
+    """
+    Resolve city within a country.
+    
+    Note: Requires city-level data files (cities_index.json, cities_data.json)
+    """
+    # This would load city data (points or polygons)
+    # For now, return None (city data not yet implemented)
+    return None

geo_intel_offline/migrate_to_modular.py

@@ -0,0 +1,159 @@
+"""
+Migration tool: Convert monolithic data format to modular format.
+
+This tool reads existing monolithic data files and converts them to
+the new modular country-wise format.
+"""
+
+import json
+import sys
+from pathlib import Path
+from typing import Dict
+
+
+def migrate_monolithic_to_modular(
+    monolithic_dir: Path,
+    output_dir: Path
+) -> Dict:
+    """
+    Migrate monolithic data files to modular format.
+    
+    Args:
+        monolithic_dir: Directory with monolithic files (geohash_index.json, etc.)
+        output_dir: Output directory for modular format
+    
+    Returns:
+        Dict with migration statistics
+    """
+    # Load monolithic files
+    index_file = monolithic_dir / "geohash_index.json"
+    polygons_file = monolithic_dir / "polygons.json"
+    metadata_file = monolithic_dir / "metadata.json"
+    
+    if not all(f.exists() for f in [index_file, polygons_file, metadata_file]):
+        raise FileNotFoundError(
+            f"Monolithic data files not found in {monolithic_dir}. "
+            "Expected: geohash_index.json, polygons.json, metadata.json"
+        )
+    
+    with open(index_file, 'r') as f:
+        geohash_index = json.load(f)
+    
+    with open(polygons_file, 'r') as f:
+        polygons = json.load(f)
+    
+    with open(metadata_file, 'r') as f:
+        metadata = json.load(f)
+    
+    # Create output structure
+    output_dir.mkdir(parents=True, exist_ok=True)
+    continents_dir = output_dir / "continents"
+    continents_dir.mkdir(exist_ok=True)
+    
+    # Organize by country
+    master_index = {
+        'version': '1.0.0',
+        'countries': {},
+        'continents': {}
+    }
+    
+    continent_countries = {}
+    
+    # Process each country
+    for country_id_str, country_metadata in metadata.items():
+        country_id = int(country_id_str)
+        iso2 = country_metadata.get('iso2', '').upper()
+        continent_raw = country_metadata.get('continent', 'unknown')
+        continent = continent_raw.lower().replace(' ', '_')
+        
+        if not iso2:
+            # Skip countries without ISO2 code
+            continue
+        
+        # Get polygon for this country
+        country_polygon = polygons.get(country_id_str)
+        if not country_polygon:
+            continue
+        
+        # Extract geohashes for this country
+        country_geohashes = {}
+        for geohash, country_ids in geohash_index.items():
+            if country_id in (country_ids if isinstance(country_ids, list) else [country_ids]):
+                country_geohashes[geohash] = [country_id]
+        
+        # Create country data structure
+        country_data = {
+            'country_id': country_id,
+            'metadata': country_metadata,
+            'geohashes': country_geohashes,
+            'polygon': country_polygon
+        }
+        
+        # Create continent directory
+        continent_dir = continents_dir / continent
+        continent_dir.mkdir(exist_ok=True)
+        
+        # Save country file
+        country_file = continent_dir / f"{iso2}.json"
+        with open(country_file, 'w', encoding='utf-8') as f:
+            json.dump(country_data, f, separators=(',', ':'))
+        
+        # Update master index
+        relative_path = f"continents/{continent}/{iso2}.json"
+        master_index['countries'][iso2] = {
+            'id': country_id,
+            'name': country_metadata.get('name', ''),
+            'iso2': iso2,
+            'iso3': country_metadata.get('iso3', ''),
+            'continent': continent_raw,
+            'file': relative_path,
+            'size_bytes': country_file.stat().st_size
+        }
+        
+        # Track continent membership
+        if continent not in continent_countries:
+            continent_countries[continent] = []
+        continent_countries[continent].append(iso2)
+    
+    # Update continent index
+    master_index['continents'] = continent_countries
+    
+    # Save master index
+    index_output = output_dir / "index.json"
+    with open(index_output, 'w', encoding='utf-8') as f:
+        json.dump(master_index, f, indent=2, ensure_ascii=False)
+    
+    return {
+        'countries_migrated': len(master_index['countries']),
+        'continents': list(continent_countries.keys())
+    }
+
+
+def main():
+    """CLI entry point."""
+    if len(sys.argv) < 3:
+        print("Usage: python -m geo_intel_offline.migrate_to_modular <monolithic_dir> <output_dir>")
+        print("\nExample:")
+        print("  python -m geo_intel_offline.migrate_to_modular geo_intel_offline/data geo_intel_offline/data_modular")
+        sys.exit(1)
+    
+    monolithic_dir = Path(sys.argv[1])
+    output_dir = Path(sys.argv[2])
+    
+    print(f"Migrating monolithic data from {monolithic_dir} to {output_dir}...")
+    
+    try:
+        stats = migrate_monolithic_to_modular(monolithic_dir, output_dir)
+        print(f"\n✓ Migration complete!")
+        print(f"  Countries migrated: {stats['countries_migrated']}")
+        print(f"  Continents: {', '.join(stats['continents'])}")
+        print(f"\nModular data saved to: {output_dir}")
+    except Exception as e:
+        print(f"\n✗ Migration failed: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

geo_intel_offline/modular_data_loader.py

@@ -0,0 +1,212 @@
+"""
+Modular data loader - supports selective country/continent loading.
+
+Features:
+- Load specific countries or continents
+- Lazy loading of country files
+- Efficient memory usage
+- Backward compatible with monolithic format
+- Supports gzip-compressed data files
+"""
+
+import json
+import gzip
+from pathlib import Path
+from typing import Dict, List, Optional
+from .data_loader import DataLoader as MonolithicLoader
+
+
+class ModularDataLoader:
+    """
+    Modular data loader that supports selective loading.
+    
+    Can load:
+    - All countries (backward compatible)
+    - Specific countries (by ISO2 codes)
+    - Specific continents
+    - All except excluded countries
+    """
+    
+    def __init__(
+        self,
+        data_dir: Optional[str] = None,
+        countries: Optional[List[str]] = None,
+        continents: Optional[List[str]] = None,
+        exclude_countries: Optional[List[str]] = None
+    ):
+        """
+        Initialize modular data loader.
+        
+        Args:
+            data_dir: Data directory (defaults to package data)
+            countries: List of ISO2 codes to load (None = all)
+            continents: List of continent names to load (None = all)
+            exclude_countries: List of ISO2 codes to exclude
+        """
+        if data_dir is None:
+            package_dir = Path(__file__).parent
+            data_dir = package_dir / "data"
+        
+        self.data_dir = Path(data_dir)
+        self.countries_filter = set(c.upper() for c in (countries or []))
+        self.continents_filter = set(c.lower().replace(" ", "_") for c in (continents or [])) if continents else None
+        self.exclude_filter = set(c.upper() for c in (exclude_countries or []))
+        
+        # Check if modular format exists
+        self.index_path = self.data_dir / "index.json"
+        self.is_modular = self.index_path.exists()
+        
+        if not self.is_modular:
+            # Fallback to monolithic loader
+            self.monolithic_loader = MonolithicLoader(data_dir)
+            return
+        
+        # Load master index (supports gzip compression)
+        index_gzip_path = self.index_path.with_suffix(self.index_path.suffix + '.gz')
+        if index_gzip_path.exists():
+            with gzip.open(index_gzip_path, 'rt', encoding='utf-8') as f:
+                self.index = json.load(f)
+        else:
+            with open(self.index_path, 'r', encoding='utf-8') as f:
+                self.index = json.load(f)
+        
+        # Determine which countries to load
+        self._determine_countries_to_load()
+        
+        # Cache for loaded country data
+        self._loaded_countries: Dict[str, Dict] = {}
+        self._geohash_index: Dict[str, List[int]] = {}
+        self._polygons: Dict[int, Dict] = {}
+        self._metadata: Dict[int, Dict] = {}
+    
+    def _determine_countries_to_load(self):
+        """Determine which countries should be loaded based on filters."""
+        available_countries = set(self.index['countries'].keys())
+        
+        if self.countries_filter:
+            # Specific countries requested
+            to_load = available_countries & self.countries_filter
+        elif self.continents_filter:
+            # Specific continents requested
+            to_load = set()
+            for continent in self.continents_filter:
+                if continent in self.index.get('continents', {}):
+                    to_load.update(self.index['continents'][continent])
+        else:
+            # Load all countries
+            to_load = available_countries
+        
+        # Apply exclusion filter
+        to_load -= self.exclude_filter
+        
+        self.countries_to_load = to_load
+    
+    def _load_country(self, iso2: str) -> Optional[Dict]:
+        """Load a single country file."""
+        if iso2 in self._loaded_countries:
+            return self._loaded_countries[iso2]
+        
+        if iso2 not in self.index['countries']:
+            return None
+        
+        country_info = self.index['countries'][iso2]
+        country_file = self.data_dir / country_info['file']
+        
+        # Try compressed version first, fallback to uncompressed
+        country_gzip_file = country_file.with_suffix(country_file.suffix + '.gz')
+        if country_gzip_file.exists():
+            with gzip.open(country_gzip_file, 'rt', encoding='utf-8') as f:
+                country_data = json.load(f)
+        elif country_file.exists():
+            with open(country_file, 'r', encoding='utf-8') as f:
+                country_data = json.load(f)
+        else:
+            return None
+        
+        self._loaded_countries[iso2] = country_data
+        
+        # Update caches
+        country_id = country_data['country_id']
+        
+        # Add to geohash index
+        for geohash, ids in country_data['geohashes'].items():
+            if geohash not in self._geohash_index:
+                self._geohash_index[geohash] = []
+            if country_id not in self._geohash_index[geohash]:
+                self._geohash_index[geohash].append(country_id)
+        
+        # Add to polygons and metadata
+        self._polygons[country_id] = country_data['polygon']
+        self._metadata[country_id] = country_data['metadata']
+        
+        return country_data
+    
+    def _load_all_countries(self):
+        """Load all countries that match filters."""
+        for iso2 in self.countries_to_load:
+            self._load_country(iso2)
+    
+    @property
+    def geohash_index(self) -> Dict[str, List[int]]:
+        """Get geohash index (lazy-loaded)."""
+        if not self.is_modular:
+            return self.monolithic_loader.geohash_index
+        
+        if not self._geohash_index:
+            self._load_all_countries()
+        return self._geohash_index
+    
+    @property
+    def polygons(self) -> Dict[int, Dict]:
+        """Get polygons (lazy-loaded)."""
+        if not self.is_modular:
+            return self.monolithic_loader.polygons
+        
+        if not self._polygons:
+            self._load_all_countries()
+        return self._polygons
+    
+    @property
+    def metadata(self) -> Dict[int, Dict]:
+        """Get metadata (lazy-loaded)."""
+        if not self.is_modular:
+            return self.monolithic_loader.metadata
+        
+        if not self._metadata:
+            self._load_all_countries()
+        return self._metadata
+    
+    def get_candidate_countries(self, geohash: str) -> List[int]:
+        """Get candidate country IDs for a geohash."""
+        index = self.geohash_index
+        
+        # Try full geohash first
+        candidates = index.get(geohash, [])
+        
+        # If no exact match, try prefixes
+        if not candidates:
+            for prefix_len in range(len(geohash), 0, -1):
+                prefix = geohash[:prefix_len]
+                if prefix in index:
+                    candidates.extend(index[prefix])
+                    break
+        
+        return list(set(candidates))
+    
+    def get_polygon(self, country_id: int) -> Optional[Dict]:
+        """Get polygon for a country."""
+        return self.polygons.get(country_id)
+    
+    def get_metadata(self, country_id: int) -> Optional[Dict]:
+        """Get metadata for a country."""
+        return self.metadata.get(country_id)
+    
+    def get_loaded_countries(self) -> List[str]:
+        """Get list of loaded country ISO2 codes."""
+        if not self.is_modular:
+            return list(self.monolithic_loader.metadata.keys())
+        return list(self.countries_to_load)
+    
+    def get_loaded_count(self) -> int:
+        """Get count of loaded countries."""
+        return len(self.countries_to_load) if self.is_modular else len(self.monolithic_loader.metadata)