glfm 1.0.4__py3-none-any.whl

glfm/__init__.py

@@ -0,0 +1,19 @@
+# __init__.py
+"""
+GLFM – Global Language Family Mapper
+
+Kielimetatietokanta ja -työkalut 7900+ kielelle (ISO 639-3).
+
+Yksi totuuden lähde versiolle: _version.py
+"""
+
+try:
+    from _version import __version__, __version_tuple__
+except ImportError:
+    try:
+        from glfm._version import __version__, __version_tuple__
+    except ImportError:
+        __version__ = "1.0.4"
+        __version_tuple__ = (1, 0, 4)
+
+__all__ = ["__version__", "__version_tuple__"]

glfm/_version.py

@@ -0,0 +1,18 @@
+# glfm/_version.py
+"""
+Yksi totuuden lähde versiolle.
+
+Käyttö:
+    from glfm._version import __version__
+    from glfm._version import __version_tuple__
+
+Versio noudattaa PEP 440 -standardia:
+    MAJOR.MINOR.PATCH[.devN | .aN | .bN | .rcN]
+"""
+
+__version__ = "1.0.4"
+__version_tuple__ = tuple(int(x) for x in __version__.split(".")[:3])
+
+# Valinnainen: git-commit, build-päivä (voi päivittää CI:ssä)
+__build_date__ = "2026-10-26"
+__git_commit__ = "unknown"

glfm/build_unified.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# GLFM Project
+# Copyright (c) 2026 Tuomas Lähteenmäki
+# Version 1.0.4
+#
+# https://codeberg.org/lahtis/GLFM
+# Licensed under the MIT License.
+#
+
+import gzip
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict
+
+from glfm.loaders.load_iso_639 import load_iso_639
+from glfm.loaders.load_cldr_likely_subtags import load_cldr_likely_subtags
+from glfm.loaders.load_wiktionary_languages import load_wiktionary_languages
+from glfm.loaders.load_uralic_languages import load_uralic_languages
+from glfm.loaders.load_written_languages import load_written_languages
+from glfm.loaders.load_glottolog import load_glottolog
+from glfm.loaders.load_pos_stats import load_pos_stats
+from glfm.loaders.load_override_scripts import load_override_scripts
+
+from glfm.logic.decide_default_script import decide_default_script
+from glfm.logic.decide_default_region import decide_default_region
+from glfm.logic.build_bcp47 import build_bcp47
+from glfm.logic.decide_fallback import decide_fallback
+from glfm.logic.is_uralic import is_uralic
+from glfm.logic.set_uralic_langs import set_uralic_langs
+
+from glfm.models.language import Language
+from glfm.paths import get_output_dir
+
+log = logging.getLogger(__name__)
+
+# ---------------------------------------------------------
+# ETÄISYYS- JA LÄHDEMETATIEDOT (URIEL / lang2vec)
+# ---------------------------------------------------------
+DISTANCE_SOURCE = "lang2vec / URIEL"
+DISTANCE_TYPE = "fam"
+DISTANCE_BASES = ["WALS", "SSWL", "PHOIBLE", "Ethnologue", "Glottolog"]
+
+# ---------------------------------------------------------
+# OUTPUT-kansiot (glfm.paths:stä)
+# ---------------------------------------------------------
+
+OUTPUT_ROOT = get_output_dir()
+
+SUBFOLDERS = ["unified", "validation", "dialects", "final", "logs"]
+
+OUTPUT = OUTPUT_ROOT / "unified" / "unified_languages.json"
+OUTPUT_GZ = OUTPUT_ROOT / "unified" / "unified_languages.json.gz"
+
+
+def _ensure_dirs() -> None:
+    """Luo output-kansiot. Kutsutaan main():sta, ei importin aikana."""
+    for folder in SUBFOLDERS:
+        (OUTPUT_ROOT / folder).mkdir(parents=True, exist_ok=True)
+
+
+# ---------------------------------------------------------
+# Unified-rakenteen rakentaminen
+# ---------------------------------------------------------
+
+def build_unified() -> Dict[str, Any]:
+
+    # --- Lataa datalähteet ---
+    iso = load_iso_639()
+    cldr = load_cldr_likely_subtags()
+    wikt = load_wiktionary_languages()
+    written = load_written_languages()
+    uralic_set = load_uralic_languages()
+    glottolog = load_glottolog()
+    pos_stats = load_pos_stats()
+    overrides = load_override_scripts()
+
+    if not uralic_set:
+        log.warning("uralic_languages.json puuttuu tai on tyhjä – "
+                    "kaikki uralicNLP-flagit ovat False")
+
+    set_uralic_langs(uralic_set)
+
+    unified: Dict[str, Any] = {}
+
+    # --- Käy läpi kaikki ISO-kielet ---
+    for lang_id, iso_info in iso.items():
+
+        w = wikt.get(lang_id, {})
+        w_written = written.get(lang_id, {})
+        glotto = glottolog.get(lang_id, {})
+        pos = pos_stats.get(lang_id, {})
+
+        # Nimet
+        name = w.get("name") or iso_info.get("name") or lang_id
+        official_name = w.get("official_name") or name
+
+        # ISO-koodit
+        iso1 = iso_info.get("iso639_1", "")
+        iso2B = iso_info.get("iso639_2B", "")
+        iso2T = iso_info.get("iso639_2T", "")
+        iso3 = iso_info.get("iso639_3", "")
+        iso5 = iso_info.get("iso639_5", "")
+        language_type = iso_info.get("language_type", "")
+
+        # --- Skripti ja alue dynaamisesti lähteistä ---
+        script = decide_default_script(lang_id, w, cldr, w_written, iso_info, overrides)
+        region = decide_default_region(lang_id, w, cldr, iso_info)
+
+        # --- Kirjoitetun kielen tila määräytyy YKSINOMAAN todennetusta skriptistä ---
+        is_written_flag = bool(script and script.upper() != "UNKNOWN")
+        scripts = [script] if is_written_flag else []
+
+        # --- BCP-47 ---
+        bcp47 = build_bcp47(lang_id, script if is_written_flag else "", region, iso_info)
+
+        # --- Fallback ja Uralic-tuki ---
+        fallback = decide_fallback(lang_id, w)
+        uralic = is_uralic(lang_id)
+
+        # --- Glottolog-kentät ---
+        glottocode = (glotto.get("glottocode")
+                      or w_written.get("glottocode", ""))
+        family = (glotto.get("family")
+                  or w_written.get("family", ""))
+
+        # Luo Language-olio
+        lang_obj = Language(
+            id=lang_id,
+            name=name,
+            official_name=official_name,
+            iso639_1=iso1,
+            iso639_2B=iso2B,
+            iso639_2T=iso2T,
+            iso639_3=iso3,
+            iso639_5=iso5,
+            language_type=language_type,
+            default_script=script if is_written_flag else "",
+            default_region=region,
+            bcp47=bcp47,
+            fallback=fallback,
+            uralicNLP=uralic,
+            written=is_written_flag,
+            written_scripts=scripts,
+            glottocode=glottocode,
+            family=family,
+            glottolog=glotto,
+            pos_stats=pos,
+        )
+
+        lang_dict = lang_obj.__dict__
+
+        # --- Etäisyysmetatiedot: AINA läsnä ---
+        lang_dict["distance_source"] = DISTANCE_SOURCE
+        lang_dict["distance_type"] = DISTANCE_TYPE
+        lang_dict["distance_bases"] = DISTANCE_BASES
+        lang_dict["nearest_languages"] = []
+
+        unified[lang_id] = lang_dict
+
+    return unified
+
+
+# ---------------------------------------------------------
+# Tallennus
+# ---------------------------------------------------------
+
+def _save(unified: Dict[str, Any]) -> None:
+    """Tallentaa sekä pretty-JSON (debug) että gzip-JSON (tuotanto)."""
+    # Pretty (debug, gitignoressa)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(unified, f, ensure_ascii=False, indent=2)
+    log.info("Kirjoitettu: %s (%.1f MB)",
+             OUTPUT, OUTPUT.stat().st_size / 1e6)
+
+    # Gzip (tuotanto, pieni)
+    with gzip.open(OUTPUT_GZ, "wt", encoding="utf-8") as f:
+        json.dump(unified, f, ensure_ascii=False, separators=(",", ":"))
+    log.info("Kirjoitettu: %s (%.1f MB)",
+             OUTPUT_GZ, OUTPUT_GZ.stat().st_size / 1e6)
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+    _ensure_dirs()
+    unified = build_unified()
+    _save(unified)
+    log.info("Valmis: %d kieltä", len(unified))
+
+
+if __name__ == "__main__":
+    main()

glfm/config.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# GLFM Project
+# Copyright (c) 2026 Tuomas Lähteenmäki
+# Version 1.0.4
+#
+# https://codeberg.org/lahtis/GLFM
+# Licensed under the MIT License.
+#
+# Konfiguraation lataus: config.toml + ympäristömuuttujat.
+#
+
+import copy
+import logging
+import os
+from pathlib import Path
+
+from glfm.paths import get_data_dir, get_output_dir
+
+log = logging.getLogger(__name__)
+
+# Oletuskonfiguraatio – ylikirjoitetaan config.toml:llä ja env-muuttujilla
+DEFAULT_CONFIG = {
+    "sources": {
+        "iso_639_3": {
+            "url": (
+                "https://iso639-3.sil.org/sites/iso639-3/files/"
+                "downloads/iso-639-3.tab"
+            ),
+        },
+        "iso_639_3_names": {
+            "url": (
+                "https://iso639-3.sil.org/sites/iso639-3/files/"
+                "downloads/iso-639-3_Name_Index.tab"
+            ),
+        },
+        "iso_639_3_macrolanguages": {
+            "url": (
+                "https://iso639-3.sil.org/sites/iso639-3/files/"
+                "downloads/iso-639-3-macrolanguages.tab"
+            ),
+        },
+        "iso_639_5": {
+            "url": "https://id.loc.gov/vocabulary/iso639-5.skos.rdf",
+        },
+        "cldr": {
+            "version": "48.0.0",
+            "url": (
+                "https://raw.githubusercontent.com/unicode-org/"
+                "cldr-json/48.0.0/cldr-json/cldr-core/supplemental/"
+                "likelySubtags.json"
+            ),
+        },
+        "wikidata": {
+            "endpoint": "https://query.wikidata.org/sparql",
+        },
+    },
+    "download": {
+        "url": (
+            "https://codeberg.org/lahtis/GLFM/releases/download/"
+            "v1.0.4/data.zip"
+        ),
+        "sha256": "",
+        "timeout": 300,
+    },
+    "http": {
+        "user_agent": "glfm/1.0.4 (https://codeberg.org/lahtis/GLFM)",
+        "timeout": 60,
+    },
+    "paths": {
+        "cache_dir": "",
+        "data_dir": "",
+        "cache_dir_name": "glfm",
+        "data_subdir": "data",
+    },
+    "logging": {
+        "level": "INFO",
+    },
+}
+
+# Ympäristömuuttujat → config-avaimet
+ENV_MAP = {
+    "GLFM_ISO_639_3_URL": ("sources", "iso_639_3", "url"),
+    "GLFM_ISO_639_3_NAMES_URL": ("sources", "iso_639_3_names", "url"),
+    "GLFM_ISO_639_3_MACRO_URL": ("sources", "iso_639_3_macrolanguages", "url"),
+    "GLFM_ISO_639_5_URL": ("sources", "iso_639_5", "url"),
+    "GLFM_CLDR_VERSION": ("sources", "cldr", "version"),
+    "GLFM_CLDR_URL": ("sources", "cldr", "url"),
+    "GLFM_WIKIDATA_ENDPOINT": ("sources", "wikidata", "endpoint"),
+    "GLFM_DATA_URL": ("download", "url"),
+    "GLFM_DATA_SHA256": ("download", "sha256"),
+    "GLFM_DATA_TIMEOUT": ("download", "timeout"),
+    "GLFM_USER_AGENT": ("http", "user_agent"),
+    "GLFM_HTTP_TIMEOUT": ("http", "timeout"),
+    "GLFM_CACHE_DIR": ("paths", "cache_dir"),
+    "GLFM_DATA_DIR": ("paths", "data_dir"),
+    "GLFM_LOG_LEVEL": ("logging", "level"),
+}
+
+
+def load_config() -> dict:
+    """
+    Lataa konfiguraation seuraavassa järjestyksessä:
+
+      1. DEFAULT_CONFIG (kovakoodatut oletukset)
+      2. config.toml (repon juuressa tai ~/.config/glfm/config.toml)
+      3. Ympäristömuuttujat (GLFM_*)
+
+    Palauttaa syvän kopion – ei jaa tilaa DEFAULT_CONFIG:n kanssa.
+    """
+    config = copy.deepcopy(DEFAULT_CONFIG)
+
+    # 1. config.toml
+    config_path = get_config_path()
+    if config_path.exists():
+        try:
+            import tomllib
+            with open(config_path, "rb") as f:
+                user_config = tomllib.load(f)
+            _merge(config, user_config)
+            log.info("Luettu config: %s", config_path)
+        except Exception as e:
+            log.warning("Config-virhe (%s): %s", config_path, e)
+    else:
+        log.debug("config.toml puuttuu (%s), käytetään oletuksia",
+                  config_path)
+
+    # 2. Ympäristömuuttujat
+    for env_key, path in ENV_MAP.items():
+        if value := os.environ.get(env_key):
+            _set_nested(config, path, value)
+            log.debug("Env: %s → %s", env_key, ".".join(path))
+
+    return config
+
+
+def get_config_path() -> Path:
+    """Palauttaa config.toml-polkua."""
+    if env := os.environ.get("GLFM_CONFIG"):
+        return Path(env).expanduser().resolve()
+
+    xdg = os.environ.get("XDG_CONFIG_HOME")
+    if xdg:
+        return Path(xdg) / "glfm" / "config.toml"
+
+    return Path.home() / ".config" / "glfm" / "config.toml"
+
+
+def _set_nested(config: dict, path: tuple, value) -> None:
+    """Asettaa arvon syvään dictiin."""
+    current = config
+    for key in path[:-1]:
+        if key not in current or not isinstance(current[key], dict):
+            current[key] = {}
+        current = current[key]
+    current[path[-1]] = value
+
+
+def _merge(base: dict, override: dict) -> None:
+    """Syvä merge: override korvaa base:n arvot."""
+    for key, value in override.items():
+        if (
+            key in base
+            and isinstance(base[key], dict)
+            and isinstance(value, dict)
+        ):
+            _merge(base[key], value)
+        else:
+            base[key] = value

glfm/data/override_scripts.json

@@ -0,0 +1,23 @@
+{
+  "ang": ["Latn"],
+  "arc": ["Armi", "Syrc"],
+  "akk": ["Xsux"],
+  "cop": ["Copt", "Grek"],
+  "egy": ["Egyp", "Egyh", "Egyd"],
+  "fro": ["Latn"],
+  "got": ["Goth"],
+  "grc": ["Grek", "Linb"],
+  "hca": ["Latn"],
+  "hdn": ["Latn"],
+  "hit": ["Xsux"],
+  "mga": ["Ogam", "Latn"],
+  "mvi": ["Jpan"],
+  "mvb": ["Latn"],
+  "non": ["Runr", "Latn"],
+  "orv": ["Cyrs", "Cyrl"],
+  "peo": ["Xpeo"],
+  "phn": ["Phnx"],
+  "san": ["Deva", "Shar", "Brah"],
+  "sux": ["Xsux"],
+  "lat": ["Latn"]
+}

glfm/data/uralic_languages.json

@@ -0,0 +1,8 @@
+[
+  "hun","kca","mns","ekk","fin","fkv","izh","krl","zkv","liv",
+  "olo","lud","fit","vep","vot","vro","mrj","mhr","myv","mdf",
+  "udm","koi","kpv","sia","smn","sjk","sjd","sms","sjt","sju",
+  "smj","sme","sje","sma","mtm","yrk","nio","enf","enh","rts",
+  "xas","sel"
+]
+

glfm/download.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# GLFM Project
+# Copyright (c) 2026 Tuomas Lähteenmäki
+# Version 1.0.4
+#
+# https://codeberg.org/lahtis/GLFM
+# Licensed under the MIT License.
+#
+# Lataa ladattavan datan (~/.cache/glfm/data/).
+# Käyttö:
+#     python3 -m glfm.download
+#     python3 -m glfm.download --force
+#     python3 -m glfm.download --target /tmp/glfm-data
+#
+
+import argparse
+import hashlib
+import io
+import logging
+import shutil
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def download_data(force: bool = False) -> Path:
+    """
+    Lataa datan cacheen (~/.cache/glfm/data/).
+
+    Args:
+        force: pakota uudelleenlataus, vaikka data olisi jo olemassa
+
+    Returns:
+        Polku data-hakemistoon
+    """
+    from glfm.config import load_config
+    from glfm.paths import get_cache_dir
+
+    config = load_config()
+    cache = get_cache_dir() / "data"
+
+    if cache.exists() and not force:
+        log.info("Data jo olemassa: %s", cache)
+        return cache
+
+    url = config["download"]["url"]
+    sha256 = config["download"]["sha256"]
+    timeout = config["download"]["timeout"]
+
+    log.info("Ladataan: %s", url)
+
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as resp:
+            data = resp.read()
+    except Exception as e:
+        log.error("Lataus epäonnistui: %s", e)
+        raise
+
+    # Tarkista hash
+    if sha256:
+        h = hashlib.sha256(data).hexdigest()
+        if h != sha256:
+            raise RuntimeError(f"SHA256 ei täsmää: {h} != {sha256}")
+
+    # Pura
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    tmp = cache.parent / "data.tmp"
+    if tmp.exists():
+        shutil.rmtree(tmp)
+    tmp.mkdir()
+
+    try:
+        with zipfile.ZipFile(io.BytesIO(data)) as z:
+            z.extractall(tmp)
+    except zipfile.BadZipFile as e:
+        shutil.rmtree(tmp, ignore_errors=True)
+        raise RuntimeError(f"Ladattu tiedosto ei ole kelvollinen zip: {e}")
+
+    # Atominen uudelleennimeäminen
+    if cache.exists():
+        shutil.rmtree(cache)
+    tmp.rename(cache)
+
+    log.info("Valmis: %s", cache)
+    return cache
+
+
+def _data_size(path: Path) -> int:
+    """Palauttaa hakemiston koon tavuina."""
+    return sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Lataa GLFM-data (~/.cache/glfm/data/).",
+    )
+    parser.add_argument(
+        "--force", action="store_true",
+        help="Pakota uudelleenlataus (ohita olemassa oleva data)",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s %(message)s",
+    )
+
+    try:
+        path = download_data(force=args.force)
+    except RuntimeError as e:
+        log.error("%s", e)
+        return 1
+
+    size_kt = _data_size(path) / 1024
+    print()
+    print(f"Data: {path}")
+    print(f"Koko: {size_kt:.1f} kt")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

glfm/loaders/__init__.py

@@ -0,0 +1,18 @@
+# loaders/__init__.py
+"""
+GLFM – datalähteiden latausmoduulit.
+
+Jokainen loaderi lukee yhden datalähteen ja palauttaa jäsennellyn dictin.
+"""
+
+__all__ = [
+    "load_cldr_likely_subtags",
+    "load_glottolog",
+    "load_iso_639",
+    "load_override_scripts",
+    "load_pos_stats",
+    "load_uralic_languages",
+    "load_wiktionary_languages",
+    "load_written_languages",
+]
+