ethnidata 3.1.5__py3-none-any.whl

ethnidata/__init__.py

@@ -0,0 +1,61 @@
+"""
+EthniData v3.0.1 - ULTRA MASSIVE EXPANSION! Global Demographics Prediction
+Predict nationality, ethnicity, gender, region, language AND religion!
+
+🔥 NEW in v3.0.1 - COMPLETE RELIGIOUS COVERAGE:
+- 📊 **5.9M+ records** (14x increase from v2.0.0 - 1,326% growth!)
+- 🌍 **238 countries** - complete global coverage
+- 🗣️  **72 languages**
+- 🕌 **ALL 6 MAJOR WORLD RELIGIONS** - Complete coverage:
+  - Christianity: 3.9M+ records (65.2%)
+  - Buddhism: 1.3M+ records (22.1%)
+  - Islam: 504K+ records (8.5%)
+  - Judaism: 121K+ records (2.0%) ✡️
+  - Hinduism: 90K+ records (1.5%)
+  - Sikhism: 24K+ records (0.4%) 🪯 NEW!
+- 🌎 **Perfectly balanced regional distribution**:
+  - Asia: 33% • Americas: 32% • Africa: 30% • Europe: 3% • Oceania: 0.1%
+
+Features:
+- ✅ Nationality prediction (238 countries)
+- ✅ Religion prediction (6 major world religions)
+- ✅ Gender prediction
+- ✅ Region prediction (5 continents)
+- ✅ Language prediction (72 languages)
+- ✅ Ethnicity prediction
+- ✅ Full name analysis
+
+Usage:
+    from ethnidata import EthniData
+
+    ed = EthniData()
+
+    # Nationality
+    result = ed.predict_nationality("Ahmet")
+
+    # Religion (NOW WITH 6 RELIGIONS!)
+    result = ed.predict_religion("Muhammad")  # Islam
+    result = ed.predict_religion("Cohen")      # Judaism
+    result = ed.predict_religion("Singh")      # Sikhism
+
+    # Gender
+    result = ed.predict_gender("Emma")
+
+    # Region
+    result = ed.predict_region("Chen")
+
+    # Language
+    result = ed.predict_language("José")
+
+    # ALL at once
+    result = ed.predict_all("Maria")
+    # Returns: nationality, religion, gender, region, language, ethnicity
+"""
+
+__version__ = "3.1.0"
+__author__ = "Teyfik Oz"
+__license__ = "MIT"
+
+from .predictor import EthniData
+
+__all__ = ["EthniData"]

ethnidata/downloader.py

@@ -0,0 +1,143 @@
+"""
+Database downloader for EthniData
+Downloads the full v3.0.0 database (5.8M records) on first use
+"""
+
+import os
+import urllib.request
+import shutil
+from pathlib import Path
+from typing import Optional
+
+# Database versions and URLs
+DATABASES = {
+    'v2.0.0': {
+        'url': 'https://github.com/teyfikoz/ethnidata/releases/download/v2.0.0/ethnidata_v2.db',
+        'size': '75 MB',
+        'records': '415K',
+        'filename': 'ethnidata.db'
+    },
+    'v3.0.0': {
+        'url': 'https://github.com/teyfikoz/ethnidata/releases/download/v3.0.0/ethnidata_v3.db',
+        'size': '1.1 GB',
+        'records': '5.8M',
+        'filename': 'ethnidata_v3.db'
+    }
+}
+
+DEFAULT_VERSION = 'v2.0.0'  # Included in package
+FULL_VERSION = 'v3.0.0'     # Downloaded on demand
+
+class DatabaseDownloader:
+    """Handles database downloads"""
+
+    def __init__(self, package_dir: Path):
+        self.package_dir = package_dir
+        self.db_path = package_dir / "ethnidata.db"
+        self.v3_path = package_dir / "ethnidata_v3.db"
+
+    def check_database(self, version: str = DEFAULT_VERSION) -> bool:
+        """Check if database exists"""
+        if version == 'v2.0.0':
+            return self.db_path.exists()
+        elif version == 'v3.0.0':
+            return self.v3_path.exists()
+        return False
+
+    def download_database(self, version: str = FULL_VERSION, force: bool = False) -> str:
+        """
+        Download database if not exists
+
+        Args:
+            version: Database version to download ('v2.0.0' or 'v3.0.0')
+            force: Force download even if exists
+
+        Returns:
+            Path to database file
+        """
+        if version not in DATABASES:
+            raise ValueError(f"Unknown version: {version}. Available: {list(DATABASES.keys())}")
+
+        db_info = DATABASES[version]
+        target_path = self.v3_path if version == 'v3.0.0' else self.db_path
+
+        # Check if already exists
+        if target_path.exists() and not force:
+            print(f"✅ Database {version} already exists ({db_info['records']} records)")
+            return str(target_path)
+
+        print(f"\n📥 Downloading EthniData {version} database...")
+        print(f"   Records: {db_info['records']}")
+        print(f"   Size: {db_info['size']}")
+        print(f"   This may take a few minutes...")
+
+        try:
+            # Download with progress
+            def report_progress(block_num, block_size, total_size):
+                downloaded = block_num * block_size
+                percent = min(downloaded * 100 / total_size, 100)
+                print(f"\r   Progress: {percent:.1f}%", end='', flush=True)
+
+            urllib.request.urlretrieve(
+                db_info['url'],
+                target_path,
+                reporthook=report_progress
+            )
+            print(f"\n✅ Download complete: {target_path}")
+            return str(target_path)
+
+        except Exception as e:
+            print(f"\n❌ Download failed: {e}")
+            print(f"\n💡 You can manually download from:")
+            print(f"   {db_info['url']}")
+            print(f"   And save it as: {target_path}")
+            raise
+
+    def get_database_path(self, prefer_v3: bool = False) -> str:
+        """
+        Get database path, downloading if necessary
+
+        Args:
+            prefer_v3: If True, use v3.0.0 (5.8M records) instead of v2.0.0 (415K records)
+
+        Returns:
+            Path to database file
+        """
+        if prefer_v3:
+            # Try to use v3, download if not exists
+            if not self.v3_path.exists():
+                print(f"\n🚀 EthniData v3.0.0 offers 14x more data (5.8M vs 415K records)!")
+                print(f"   Would you like to download it? ({DATABASES['v3.0.0']['size']})")
+                response = input("   Download v3.0.0? [y/N]: ").strip().lower()
+
+                if response in ['y', 'yes']:
+                    return self.download_database('v3.0.0')
+                else:
+                    print(f"   Using v2.0.0 ({DATABASES['v2.0.0']['records']} records)")
+                    return str(self.db_path)
+            return str(self.v3_path)
+        else:
+            # Use v2 (included in package)
+            if not self.db_path.exists():
+                raise FileNotFoundError(
+                    f"Database not found at {self.db_path}. "
+                    f"Please reinstall: pip install --upgrade --force-reinstall ethnidata"
+                )
+            return str(self.db_path)
+
+
+def download_v3_database(package_dir: Optional[Path] = None) -> str:
+    """
+    Convenience function to download v3.0.0 database
+
+    Args:
+        package_dir: Package directory (auto-detected if None)
+
+    Returns:
+        Path to downloaded database
+    """
+    if package_dir is None:
+        package_dir = Path(__file__).parent
+
+    downloader = DatabaseDownloader(package_dir)
+    return downloader.download_database('v3.0.0')

ethnidata/predictor.py

@@ -0,0 +1,560 @@
+"""
+EthniData Predictor v2.0 - Gelişmiş özelliklerle
+Yeni özellikler:
+- Gender prediction (Cinsiyet tahmini)
+- Region prediction (Bölge: Europe, Asia, Americas, Africa, Oceania)
+- Language prediction (Yaygın dil tahmini)
+"""
+
+import sqlite3
+from pathlib import Path
+from typing import Dict, List, Optional, Literal
+from unidecode import unidecode
+import pycountry
+
+class EthniData:
+    """Ethnicity, Nationality, Gender, Region and Language predictor"""
+
+    def __init__(self, db_path: Optional[str] = None, use_v3: bool = False):
+        """
+        Initialize EthniData predictor
+
+        Args:
+            db_path: Path to SQLite database. If None, uses default location.
+            use_v3: If True, attempts to use v3.0.0 database (5.8M records).
+                   If False, uses v2.0.0 database (415K records, included in package).
+        """
+        if db_path is None:
+            package_dir = Path(__file__).parent
+
+            if use_v3:
+                # Try to use v3 database
+                v3_path = package_dir / "ethnidata_v3.db"
+                if v3_path.exists():
+                    db_path = v3_path
+                else:
+                    print(f"\n💡 EthniData v3.0.0 (5.8M records) is not installed.")
+                    print(f"   To download: from ethnidata.downloader import download_v3_database")
+                    print(f"   download_v3_database()")
+                    print(f"\n   Using v2.0.0 (415K records) for now...")
+                    db_path = package_dir / "ethnidata.db"
+            else:
+                db_path = package_dir / "ethnidata.db"
+
+        self.db_path = Path(db_path)
+
+        if not self.db_path.exists():
+            raise FileNotFoundError(
+                f"Database not found: {self.db_path}\n"
+                f"Please reinstall: pip install --upgrade --force-reinstall ethnidata"
+            )
+
+        self.conn = sqlite3.connect(self.db_path)
+        self.conn.row_factory = sqlite3.Row
+
+    def __del__(self):
+        """Close database connection"""
+        if hasattr(self, 'conn'):
+            self.conn.close()
+
+    @staticmethod
+    def normalize_name(name: str) -> str:
+        """Normalize name (lowercase, remove accents)"""
+        return unidecode(name.strip().lower())
+
+    def predict_nationality(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first",
+        top_n: int = 5
+    ) -> Dict:
+        """
+        Predict nationality from name
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+            top_n: Number of top predictions
+
+        Returns:
+            {
+                'name': str,
+                'country': str (ISO 3166-1 alpha-3),
+                'country_name': str,
+                'confidence': float (0-1),
+                'region': str,  # NEW
+                'language': str,  # NEW
+                'top_countries': [...]
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        query = """
+            SELECT country_code, region, language, COUNT(*) as frequency
+            FROM names
+            WHERE name = ? AND name_type = ?
+            GROUP BY country_code, region, language
+            ORDER BY frequency DESC
+            LIMIT ?
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized, name_type, top_n))
+        results = cursor.fetchall()
+
+        if not results:
+            return {
+                'name': normalized,
+                'country': None,
+                'country_name': None,
+                'confidence': 0.0,
+                'region': None,
+                'language': None,
+                'top_countries': []
+            }
+
+        # Calculate probabilities
+        total_freq = sum(row['frequency'] for row in results)
+
+        top_countries = []
+        for row in results:
+            prob = row['frequency'] / total_freq
+
+            try:
+                country = pycountry.countries.get(alpha_3=row['country_code'])
+                country_name = country.name if country else row['country_code']
+            except:
+                country_name = row['country_code']
+
+            top_countries.append({
+                'country': row['country_code'],
+                'country_name': country_name,
+                'region': row['region'],
+                'language': row['language'],
+                'probability': round(prob, 4),
+                'frequency': row['frequency']
+            })
+
+        top = top_countries[0]
+
+        return {
+            'name': normalized,
+            'country': top['country'],
+            'country_name': top['country_name'],
+            'confidence': top['probability'],
+            'region': top['region'],  # NEW
+            'language': top['language'],  # NEW
+            'top_countries': top_countries
+        }
+
+    def predict_gender(
+        self,
+        name: str
+    ) -> Dict:
+        """
+        Predict gender from first name
+
+        Args:
+            name: First name
+
+        Returns:
+            {
+                'name': str,
+                'gender': str ('M' or 'F' or None),
+                'confidence': float,
+                'distribution': {'M': prob, 'F': prob, None: prob}
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        query = """
+            SELECT gender, COUNT(*) as count
+            FROM names
+            WHERE name = ? AND name_type = 'first'
+            GROUP BY gender
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized,))
+        results = cursor.fetchall()
+
+        if not results:
+            return {
+                'name': normalized,
+                'gender': None,
+                'confidence': 0.0,
+                'distribution': {}
+            }
+
+        # Count by gender
+        gender_counts = {}
+        total = 0
+
+        for row in results:
+            gender = row['gender']
+            count = row['count']
+            gender_counts[gender] = count
+            total += count
+
+        # Calculate probabilities
+        distribution = {g: round(c / total, 4) for g, c in gender_counts.items()}
+
+        # Top gender
+        top_gender = max(gender_counts.items(), key=lambda x: x[1])[0]
+        confidence = gender_counts[top_gender] / total
+
+        return {
+            'name': normalized,
+            'gender': top_gender,
+            'confidence': round(confidence, 4),
+            'distribution': distribution
+        }
+
+    def predict_region(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first"
+    ) -> Dict:
+        """
+        Predict geographic region from name
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+
+        Returns:
+            {
+                'name': str,
+                'region': str (Europe, Asia, Americas, Africa, Oceania, Other),
+                'confidence': float,
+                'distribution': {region: probability, ...}
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        query = """
+            SELECT region, COUNT(*) as total_freq
+            FROM names
+            WHERE name = ? AND name_type = ?
+            GROUP BY region
+            ORDER BY total_freq DESC
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized, name_type))
+        results = cursor.fetchall()
+
+        if not results:
+            return {
+                'name': normalized,
+                'region': None,
+                'confidence': 0.0,
+                'distribution': {}
+            }
+
+        total = sum(row['total_freq'] for row in results)
+
+        distribution = {}
+        for row in results:
+            region = row['region']
+            prob = row['total_freq'] / total
+            distribution[region] = round(prob, 4)
+
+        top_region = results[0]['region']
+        confidence = results[0]['total_freq'] / total
+
+        return {
+            'name': normalized,
+            'region': top_region,
+            'confidence': round(confidence, 4),
+            'distribution': distribution
+        }
+
+    def predict_language(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first",
+        top_n: int = 5
+    ) -> Dict:
+        """
+        Predict most likely language from name
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+            top_n: Number of top predictions
+
+        Returns:
+            {
+                'name': str,
+                'language': str,
+                'confidence': float,
+                'top_languages': [{language, probability}, ...]
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        query = """
+            SELECT language, COUNT(*) as total_freq
+            FROM names
+            WHERE name = ? AND name_type = ? AND language IS NOT NULL
+            GROUP BY language
+            ORDER BY total_freq DESC
+            LIMIT ?
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized, name_type, top_n))
+        results = cursor.fetchall()
+
+        if not results:
+            return {
+                'name': normalized,
+                'language': None,
+                'confidence': 0.0,
+                'top_languages': []
+            }
+
+        total = sum(row['total_freq'] for row in results)
+
+        top_languages = []
+        for row in results:
+            lang = row['language']
+            prob = row['total_freq'] / total
+            top_languages.append({
+                'language': lang,
+                'probability': round(prob, 4)
+            })
+
+        return {
+            'name': normalized,
+            'language': top_languages[0]['language'],
+            'confidence': top_languages[0]['probability'],
+            'top_languages': top_languages
+        }
+
+    def predict_religion(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first",
+        top_n: int = 5
+    ) -> Dict:
+        """
+        Predict religion from name - NEW in v1.3.0!
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+            top_n: Number of top predictions
+
+        Returns:
+            {
+                'name': str,
+                'religion': str (Christianity, Islam, Hinduism, Buddhism, Judaism),
+                'confidence': float,
+                'top_religions': [{religion, probability}, ...]
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        query = """
+            SELECT religion, COUNT(*) as total_freq
+            FROM names
+            WHERE name = ? AND name_type = ? AND religion IS NOT NULL
+            GROUP BY religion
+            ORDER BY total_freq DESC
+            LIMIT ?
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized, name_type, top_n))
+        results = cursor.fetchall()
+
+        if not results:
+            return {
+                'name': normalized,
+                'religion': None,
+                'confidence': 0.0,
+                'top_religions': []
+            }
+
+        total = sum(row['total_freq'] for row in results)
+
+        top_religions = []
+        for row in results:
+            religion = row['religion']
+            prob = row['total_freq'] / total
+            top_religions.append({
+                'religion': religion,
+                'probability': round(prob, 4)
+            })
+
+        return {
+            'name': normalized,
+            'religion': top_religions[0]['religion'],
+            'confidence': top_religions[0]['probability'],
+            'top_religions': top_religions
+        }
+
+    def predict_ethnicity(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first"
+    ) -> Dict:
+        """Predict ethnicity from name (uses nationality as proxy)"""
+
+        # Use nationality as ethnicity proxy since we don't have separate ethnicity data
+        nationality = self.predict_nationality(name, name_type, top_n=1)
+
+        return {
+            'name': nationality['name'],
+            'ethnicity': nationality['country_name'],  # Use country as ethnicity
+            'country': nationality['country'],
+            'country_name': nationality['country_name'],
+            'region': nationality.get('region'),
+            'language': nationality.get('language'),
+            'confidence': nationality['confidence']
+        }
+
+    def predict_full_name(
+        self,
+        first_name: str,
+        last_name: str,
+        top_n: int = 5
+    ) -> Dict:
+        """
+        Predict from full name (first + last) - ENHANCED
+
+        Returns nationality, region, language
+        """
+
+        first_pred = self.predict_nationality(first_name, "first", top_n=top_n)
+        last_pred = self.predict_nationality(last_name, "last", top_n=top_n)
+
+        # Combine scores
+        combined_scores = {}
+
+        for item in first_pred['top_countries']:
+            combined_scores[item['country']] = {
+                'score': item['probability'] * 0.4,
+                'region': item['region'],
+                'language': item['language']
+            }
+
+        for item in last_pred['top_countries']:
+            if item['country'] in combined_scores:
+                combined_scores[item['country']]['score'] += item['probability'] * 0.6
+            else:
+                combined_scores[item['country']] = {
+                    'score': item['probability'] * 0.6,
+                    'region': item['region'],
+                    'language': item['language']
+                }
+
+        # Sort
+        sorted_countries = sorted(
+            combined_scores.items(),
+            key=lambda x: x[1]['score'],
+            reverse=True
+        )[:top_n]
+
+        # Format
+        top_countries = []
+        for country_code, data in sorted_countries:
+            try:
+                country = pycountry.countries.get(alpha_3=country_code)
+                country_name = country.name if country else country_code
+            except:
+                country_name = country_code
+
+            top_countries.append({
+                'country': country_code,
+                'country_name': country_name,
+                'region': data['region'],
+                'language': data['language'],
+                'probability': round(data['score'], 4)
+            })
+
+        top = top_countries[0] if top_countries else {}
+
+        return {
+            'first_name': self.normalize_name(first_name),
+            'last_name': self.normalize_name(last_name),
+            'country': top.get('country'),
+            'country_name': top.get('country_name'),
+            'region': top.get('region'),  # NEW
+            'language': top.get('language'),  # NEW
+            'confidence': top.get('probability', 0.0),
+            'top_countries': top_countries
+        }
+
+    def predict_all(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first"
+    ) -> Dict:
+        """
+        Predict ALL attributes at once - UPDATED v1.3.0
+        Now includes: nationality, gender, region, language, religion, ethnicity
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+
+        Returns:
+            {
+                'name': str,
+                'nationality': {...},
+                'gender': {...},  # Only for first names
+                'region': {...},
+                'language': {...},
+                'religion': {...},  # NEW in v1.3.0!
+                'ethnicity': {...}
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        result = {
+            'name': normalized,
+            'nationality': self.predict_nationality(name, name_type),
+            'region': self.predict_region(name, name_type),
+            'language': self.predict_language(name, name_type),
+            'religion': self.predict_religion(name, name_type),  # NEW!
+            'ethnicity': self.predict_ethnicity(name, name_type)
+        }
+
+        # Gender only for first names
+        if name_type == "first":
+            result['gender'] = self.predict_gender(name)
+
+        return result
+
+    def get_stats(self) -> Dict:
+        """Get database statistics"""
+
+        cursor = self.conn.cursor()
+
+        stats = {}
+
+        cursor.execute("SELECT COUNT(*) as count FROM names WHERE name_type = 'first'")
+        stats['total_first_names'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(*) as count FROM names WHERE name_type = 'last'")
+        stats['total_last_names'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(DISTINCT country_code) as count FROM names")
+        stats['countries'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(DISTINCT region) as count FROM names WHERE region IS NOT NULL")
+        stats['regions'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(DISTINCT language) as count FROM names WHERE language IS NOT NULL")
+        stats['languages'] = cursor.fetchone()['count']
+
+        return stats

ethnidata/predictor_old.py

@@ -0,0 +1,277 @@
+"""
+EthniData Predictor - Ana tahmin modülü
+"""
+
+import sqlite3
+from pathlib import Path
+from typing import Dict, List, Optional, Literal
+from unidecode import unidecode
+import pycountry
+
+class EthniData:
+    """Ethnicity and Nationality Data predictor"""
+
+    def __init__(self, db_path: Optional[str] = None):
+        """
+        Initialize EthniData predictor
+
+        Args:
+            db_path: Path to SQLite database. If None, uses default location.
+        """
+        if db_path is None:
+            db_path = Path(__file__).parent / "ethnidata.db"
+
+        self.db_path = Path(db_path)
+
+        if not self.db_path.exists():
+            raise FileNotFoundError(
+                f"Database not found: {self.db_path}\n"
+                f"Please run scripts/6_create_database.py first"
+            )
+
+        self.conn = sqlite3.connect(self.db_path)
+        self.conn.row_factory = sqlite3.Row
+
+    def __del__(self):
+        """Close database connection"""
+        if hasattr(self, 'conn'):
+            self.conn.close()
+
+    @staticmethod
+    def normalize_name(name: str) -> str:
+        """Normalize name (lowercase, remove accents)"""
+        return unidecode(name.strip().lower())
+
+    def predict_nationality(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first",
+        top_n: int = 5
+    ) -> Dict:
+        """
+        Predict nationality from name
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+            top_n: Number of top predictions to return
+
+        Returns:
+            {
+                'name': normalized name,
+                'country': top country code (ISO 3166-1 alpha-3),
+                'confidence': confidence score (0-1),
+                'top_countries': [{country, probability, frequency}, ...]
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        table = "first_names" if name_type == "first" else "last_names"
+
+        # Query database
+        query = f"""
+            SELECT country_code, frequency
+            FROM {table}
+            WHERE name = ?
+            ORDER BY frequency DESC
+            LIMIT ?
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized, top_n))
+
+        results = cursor.fetchall()
+
+        if not results:
+            return {
+                'name': normalized,
+                'country': None,
+                'confidence': 0.0,
+                'top_countries': []
+            }
+
+        # Calculate probabilities
+        total_frequency = sum(row['frequency'] for row in results)
+
+        top_countries = []
+        for row in results:
+            prob = row['frequency'] / total_frequency
+
+            # Country name lookup
+            try:
+                country = pycountry.countries.get(alpha_3=row['country_code'])
+                country_name = country.name if country else row['country_code']
+            except:
+                country_name = row['country_code']
+
+            top_countries.append({
+                'country': row['country_code'],
+                'country_name': country_name,
+                'probability': round(prob, 4),
+                'frequency': row['frequency']
+            })
+
+        # Top prediction
+        top = top_countries[0]
+
+        return {
+            'name': normalized,
+            'country': top['country'],
+            'country_name': top['country_name'],
+            'confidence': top['probability'],
+            'top_countries': top_countries
+        }
+
+    def predict_ethnicity(
+        self,
+        name: str,
+        name_type: Literal["first", "last"] = "first"
+    ) -> Dict:
+        """
+        Predict ethnicity from name
+
+        Args:
+            name: First or last name
+            name_type: "first" or "last"
+
+        Returns:
+            {
+                'name': normalized name,
+                'ethnicity': predicted ethnicity,
+                'country': most likely country,
+                'confidence': confidence score
+            }
+        """
+
+        normalized = self.normalize_name(name)
+
+        table = "first_names" if name_type == "first" else "last_names"
+
+        # Query with ethnicity
+        query = f"""
+            SELECT country_code, ethnicity, frequency
+            FROM {table}
+            WHERE name = ? AND ethnicity IS NOT NULL
+            ORDER BY frequency DESC
+            LIMIT 1
+        """
+
+        cursor = self.conn.cursor()
+        cursor.execute(query, (normalized,))
+
+        result = cursor.fetchone()
+
+        if result:
+            # Country name
+            try:
+                country = pycountry.countries.get(alpha_3=result['country_code'])
+                country_name = country.name if country else result['country_code']
+            except:
+                country_name = result['country_code']
+
+            return {
+                'name': normalized,
+                'ethnicity': result['ethnicity'],
+                'country': result['country_code'],
+                'country_name': country_name,
+                'frequency': result['frequency']
+            }
+
+        # Fallback to nationality prediction
+        nationality = self.predict_nationality(name, name_type, top_n=1)
+
+        return {
+            'name': normalized,
+            'ethnicity': None,
+            'country': nationality['country'],
+            'country_name': nationality.get('country_name'),
+            'confidence': nationality['confidence']
+        }
+
+    def predict_full_name(
+        self,
+        first_name: str,
+        last_name: str,
+        top_n: int = 5
+    ) -> Dict:
+        """
+        Predict nationality from full name (first + last)
+
+        Combines predictions from both first and last names
+
+        Args:
+            first_name: First name
+            last_name: Last name
+            top_n: Number of top predictions
+
+        Returns:
+            Combined prediction with country probabilities
+        """
+
+        first_pred = self.predict_nationality(first_name, "first", top_n=top_n)
+        last_pred = self.predict_nationality(last_name, "last", top_n=top_n)
+
+        # Combine probabilities
+        combined_scores = {}
+
+        for item in first_pred['top_countries']:
+            combined_scores[item['country']] = item['probability'] * 0.4
+
+        for item in last_pred['top_countries']:
+            if item['country'] in combined_scores:
+                combined_scores[item['country']] += item['probability'] * 0.6
+            else:
+                combined_scores[item['country']] = item['probability'] * 0.6
+
+        # Sort by combined score
+        sorted_countries = sorted(
+            combined_scores.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )[:top_n]
+
+        # Format results
+        top_countries = []
+        for country_code, score in sorted_countries:
+            try:
+                country = pycountry.countries.get(alpha_3=country_code)
+                country_name = country.name if country else country_code
+            except:
+                country_name = country_code
+
+            top_countries.append({
+                'country': country_code,
+                'country_name': country_name,
+                'probability': round(score, 4)
+            })
+
+        return {
+            'first_name': self.normalize_name(first_name),
+            'last_name': self.normalize_name(last_name),
+            'country': top_countries[0]['country'] if top_countries else None,
+            'country_name': top_countries[0]['country_name'] if top_countries else None,
+            'confidence': top_countries[0]['probability'] if top_countries else 0.0,
+            'top_countries': top_countries
+        }
+
+    def get_stats(self) -> Dict:
+        """Get database statistics"""
+
+        cursor = self.conn.cursor()
+
+        stats = {}
+
+        cursor.execute("SELECT COUNT(*) as count FROM first_names")
+        stats['total_first_names'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(*) as count FROM last_names")
+        stats['total_last_names'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(DISTINCT country_code) as count FROM first_names")
+        stats['countries_first'] = cursor.fetchone()['count']
+
+        cursor.execute("SELECT COUNT(DISTINCT country_code) as count FROM last_names")
+        stats['countries_last'] = cursor.fetchone()['count']
+
+        return stats

ethnidata-3.1.5.dist-info/METADATA

@@ -0,0 +1,257 @@
+Metadata-Version: 2.4
+Name: ethnidata
+Version: 3.1.5
+Summary: Predict nationality, ethnicity, gender, region, language and religion from names - 238 countries, 6 major religions, 5.9M+ names, complete religious coverage
+Author-email: Teyfik OZ <teyfikoz@example.com>
+License: MIT
+Project-URL: Homepage, https://github.com/teyfikoz/ethnidata
+Project-URL: Documentation, https://github.com/teyfikoz/ethnidata#readme
+Project-URL: Repository, https://github.com/teyfikoz/ethnidata.git
+Project-URL: Issues, https://github.com/teyfikoz/ethnidata/issues
+Keywords: names,nationality,ethnicity,demographics,prediction,NLP
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pycountry>=22.3.5
+Requires-Dist: unidecode>=1.3.6
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Provides-Extra: build
+Requires-Dist: requests>=2.31.0; extra == "build"
+Requires-Dist: pandas>=2.0.0; extra == "build"
+Requires-Dist: numpy>=1.24.0; extra == "build"
+Requires-Dist: beautifulsoup4>=4.12.0; extra == "build"
+Requires-Dist: lxml>=4.9.0; extra == "build"
+Requires-Dist: tqdm>=4.65.0; extra == "build"
+Requires-Dist: wikipedia-api>=0.6.0; extra == "build"
+Requires-Dist: sqlalchemy>=2.0.0; extra == "build"
+Dynamic: license-file
+
+# EthniData - Ethnicity and Nationality Prediction
+
+[![Python](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![PyPI version](https://badge.fury.io/py/ethnidata.svg)](https://badge.fury.io/py/ethnidata)
+
+Predict **nationality**, **ethnicity**, and **demographics** from names using a comprehensive global database built from multiple authoritative sources.
+
+## 🌟 Features
+
+- **190+ Countries** - Comprehensive coverage from Wikipedia/Wikidata
+- **106 Countries** - Enhanced with names-dataset
+- **120 Years** of Olympic athlete names
+- **Multiple Sources** - Phone directories, census data, public records
+- **Fast Predictions** - SQLite-based for instant lookups
+- **Normalized Data** - Unicode-aware, case-insensitive matching
+- **Ethnicity Support** - Where available in source data
+- **Simple API** - Easy to use Python interface
+
+## 📊 Data Sources
+
+1. **Wikipedia/Wikidata** - 190+ countries, biographical data with ethnicity
+2. **names-dataset** - 106 countries, curated name lists
+3. **Olympics Dataset** - 120 years of athlete names (271,116 records)
+4. **Phone Directories** - Public domain name lists from multiple countries
+5. **Census Data** - US Census and other government open data
+
+## 🚀 Installation
+
+```bash
+pip install ethnidata
+```
+
+## 📖 Usage
+
+### Basic Usage
+
+```python
+from ethnidata import EthniData
+
+# Initialize
+ed = EthniData()
+
+# Predict nationality from first name
+result = ed.predict_nationality("Ahmet", name_type="first")
+print(result)
+# {
+#   'name': 'ahmet',
+#   'country': 'TUR',
+#   'country_name': 'Turkey',
+#   'confidence': 0.89,
+#   'top_countries': [
+#     {'country': 'TUR', 'country_name': 'Turkey', 'probability': 0.89},
+#     {'country': 'DEU', 'country_name': 'Germany', 'probability': 0.07},
+#     ...
+#   ]
+# }
+
+# Predict from last name
+result = ed.predict_nationality("Tanaka", name_type="last")
+print(result['country'])  # 'JPN'
+
+# Predict from full name (combines both)
+result = ed.predict_full_name("Wei", "Chen")
+print(result['country'])  # 'CHN'
+
+# Predict ethnicity (when available)
+result = ed.predict_ethnicity("Muhammad", name_type="first")
+print(result)
+# {
+#   'name': 'muhammad',
+#   'ethnicity': 'Arab',
+#   'country': 'SAU',
+#   'country_name': 'Saudi Arabia'
+# }
+```
+
+### Advanced Usage
+
+```python
+# Get top 10 predictions
+result = ed.predict_nationality("Maria", name_type="first", top_n=10)
+
+for country in result['top_countries']:
+    print(f"{country['country_name']}: {country['probability']:.2%}")
+# Spain: 35.4%
+# Italy: 28.2%
+# Portugal: 15.1%
+# ...
+
+# Database statistics
+stats = ed.get_stats()
+print(stats)
+# {
+#   'total_first_names': 123456,
+#   'total_last_names': 234567,
+#   'countries_first': 195,
+#   'countries_last': 198
+# }
+```
+
+## 🏗️ Project Structure
+
+```
+ethnidata/
+├── ethnidata/                # Main package
+│   ├── __init__.py
+│   ├── predictor.py          # Core prediction logic
+│   └── ethnidata.db          # SQLite database
+├── scripts/                  # Data collection scripts
+│   ├── 1_fetch_names_dataset.py
+│   ├── 2_fetch_wikipedia.py
+│   ├── 3_fetch_olympics.py
+│   ├── 4_fetch_phone_directories.py
+│   ├── 5_merge_all_data.py
+│   └── 6_create_database.py
+├── tests/                    # Unit tests
+├── examples/                 # Example scripts
+├── docs/                     # Documentation
+├── setup.py
+├── pyproject.toml
+└── README.md
+```
+
+## 🔬 Accuracy & Methodology
+
+### How it works
+
+1. **Name Normalization**: Names are lowercased and Unicode-normalized (e.g., "José" → "jose")
+2. **Database Lookup**: Queries SQLite database for matching names
+3. **Frequency-Based Scoring**: Countries are ranked by how often the name appears
+4. **Probability Calculation**: Frequencies are converted to probabilities
+5. **Full Name Combination**: First name (40%) + last name (60%) weights
+
+### Limitations
+
+- **Bias**: Database reflects historical Olympic participation, Wikipedia coverage
+- **Missing Names**: Rare or new names may not be in database
+- **Ethnicity**: Only available where source data included it
+- **Migration**: Doesn't account for diaspora or modern migration patterns
+- **Multiple Origins**: Common names (e.g., "Ali", "Maria") exist in many cultures
+
+## 🛠️ Development
+
+### Build Database from Scratch
+
+```bash
+git clone https://github.com/teyfikoz/ethnidata.git
+cd ethnidata
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Fetch all data (takes 10-30 minutes)
+cd scripts
+python 1_fetch_names_dataset.py
+python 2_fetch_wikipedia.py
+python 3_fetch_olympics.py
+python 4_fetch_phone_directories.py
+python 5_merge_all_data.py
+python 6_create_database.py
+```
+
+### Run Tests
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+## 📜 License
+
+MIT License - see [LICENSE](LICENSE) file for details
+
+## 🤝 Contributing
+
+Contributions welcome! Please:
+
+1. Fork the repository
+2. Create a feature branch
+3. Commit your changes
+4. Push to the branch
+5. Open a Pull Request
+
+## 📚 Citations
+
+If you use this database in research, please cite:
+
+```bibtex
+@software{ethnidata_2024,
+  title = {EthniData: Ethnicity and Nationality Prediction from Names},
+  author = {Oz, Teyfik},
+  year = {2024},
+  url = {https://github.com/teyfikoz/ethnidata}
+}
+```
+
+### Data Source Citations
+
+- **Olympics Data**: Randi Griffin (2018). 120 years of Olympic history. [Kaggle](https://www.kaggle.com/datasets/heesoo37/120-years-of-olympic-history-athletes-and-results)
+- **names-dataset**: Philippe Remy (2021). [name-dataset](https://github.com/philipperemy/name-dataset)
+- **Wikidata**: Wikimedia Foundation. [Wikidata](https://www.wikidata.org)
+
+## 🔗 Related Projects
+
+- [ethnicolr](https://github.com/appeler/ethnicolr) - Ethnicity prediction using LSTM
+- [name-dataset](https://github.com/philipperemy/name-dataset) - Name database (106 countries)
+- [gender-guesser](https://github.com/lead-ratings/gender-guesser) - Gender prediction
+
+## 📧 Contact
+
+- GitHub Issues: [Report bugs or request features](https://github.com/teyfikoz/ethnidata/issues)
+- GitHub: [@teyfikoz](https://github.com/teyfikoz)
+
+---
+
+**Built with ❤️ using open data**

ethnidata-3.1.5.dist-info/RECORD

@@ -0,0 +1,10 @@
+ethnidata/__init__.py,sha256=xeg_6_eiFa05LNQKzUtQ1F1EHhQbGRrBSkq6lLuZ1CY,1858
+ethnidata/downloader.py,sha256=GNohBtHyn_14TuWPhRUMxGNHy0UieXzwFCC5z-oiVQs,5057
+ethnidata/ethnidata.db,sha256=edDXYMOoNNtVprbYGqQ7qPgdn2U7e6Y1PPyocBADVOs,78405632
+ethnidata/predictor.py,sha256=fmmLSVpluMpBeKxABAwT7OVIaY4c_H5TksiZu7dETEQ,17687
+ethnidata/predictor_old.py,sha256=dGmfYWTO2BRYxQUzzE7foZMEEDaOd6VWPZk4ib5Gp9E,8696
+ethnidata-3.1.5.dist-info/licenses/LICENSE,sha256=p5pRNvuSoG_JxH4Xy11FK2iXc3hyAnzOKUx9gBltulk,1095
+ethnidata-3.1.5.dist-info/METADATA,sha256=WEsx8FKprwzqhA2GmYGN7i52FzTCpqs-lTZqMV2040M,8182
+ethnidata-3.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ethnidata-3.1.5.dist-info/top_level.txt,sha256=V5Cuyv_Ib3mDSp2KL8MocXzLyp3o3r2FG01rFA7Iatk,10
+ethnidata-3.1.5.dist-info/RECORD,,

ethnidata-3.1.5.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ethnidata-3.1.5.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 NBD Database Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ethnidata-3.1.5.dist-info/top_level.txt

@@ -0,0 +1 @@
+ethnidata