sdmxlib 0.8.0__py3-none-any.whl

sdmxlib/storage/schema.py

@@ -0,0 +1,289 @@
+"""DuckDB schema for SDMX structural metadata.
+
+Tables are generated programmatically so that language columns
+(``name_en``, ``label_fr``, etc.) are driven by a configurable
+``languages`` list rather than being hard-coded.
+
+Example::
+
+    import duckdb
+    from sdmxlib.storage.schema import create_tables, get_languages
+
+    conn = duckdb.connect("store.duckdb")
+    create_tables(conn, languages=["en", "fr"])
+
+    # Later — discover configured languages
+    langs = get_languages(conn)  # ["en", "fr"]
+"""
+
+from collections.abc import Sequence
+
+import duckdb
+
+# ── Table definitions ────────────────────────────────────────────────────
+#
+# Each entry: (table_name, fixed_columns, lang_prefixes)
+#   fixed_columns: list of (col_name, col_type) tuples
+#   lang_prefixes: list of column-name prefixes that get a _{lang} suffix
+#     for each configured language
+
+_TABLE_DEFS: list[tuple[str, list[tuple[str, str]], list[str]]] = [
+    (
+        "artefacts",
+        [
+            ("urn", "VARCHAR PRIMARY KEY"),
+            ("artefact_type", "VARCHAR NOT NULL"),
+            ("agency", "VARCHAR NOT NULL"),
+            ("id", "VARCHAR NOT NULL"),
+            ("version", "VARCHAR NOT NULL DEFAULT '1.0'"),
+            ("is_final", "BOOLEAN DEFAULT FALSE"),
+            # language columns inserted here
+            ("_document", "VARCHAR"),
+        ],
+        ["name", "description"],
+    ),
+    (
+        "codelists",
+        [
+            ("urn", "VARCHAR PRIMARY KEY"),
+            ("agency", "VARCHAR NOT NULL"),
+            ("id", "VARCHAR NOT NULL"),
+            ("version", "VARCHAR NOT NULL DEFAULT '1.0'"),
+            # language columns inserted here
+            ("is_partial", "BOOLEAN DEFAULT FALSE"),
+        ],
+        ["name"],
+    ),
+    (
+        "codes",
+        [
+            ("codelist_urn", "VARCHAR NOT NULL"),
+            ("code_id", "VARCHAR NOT NULL"),
+            ("parent_code_id", "VARCHAR"),
+            ("position", "INTEGER"),
+            # language columns inserted here
+            ("_annotations", "VARCHAR"),
+        ],
+        ["label", "description"],
+    ),
+    (
+        "data_structures",
+        [
+            ("urn", "VARCHAR PRIMARY KEY"),
+            ("agency", "VARCHAR NOT NULL"),
+            ("id", "VARCHAR NOT NULL"),
+            ("version", "VARCHAR NOT NULL DEFAULT '1.0'"),
+        ],
+        ["name"],
+    ),
+    (
+        "dsd_components",
+        [
+            ("dsd_urn", "VARCHAR NOT NULL"),
+            ("component_id", "VARCHAR NOT NULL"),
+            ("component_type", "VARCHAR NOT NULL"),
+            ("position", "INTEGER"),
+            ("concept_id", "VARCHAR"),
+            ("concept_urn", "VARCHAR"),
+            # language columns inserted here
+            ("codelist_urn", "VARCHAR"),
+            ("attachment_level", "VARCHAR"),
+        ],
+        ["concept_name"],
+    ),
+    (
+        "dataflows",
+        [
+            ("urn", "VARCHAR PRIMARY KEY"),
+            ("agency", "VARCHAR NOT NULL"),
+            ("id", "VARCHAR NOT NULL"),
+            ("version", "VARCHAR NOT NULL DEFAULT '1.0'"),
+            # language columns inserted here
+            ("dsd_urn", "VARCHAR"),
+        ],
+        ["name", "description"],
+    ),
+    (
+        "concept_schemes",
+        [
+            ("urn", "VARCHAR PRIMARY KEY"),
+            ("agency", "VARCHAR NOT NULL"),
+            ("id", "VARCHAR NOT NULL"),
+            ("version", "VARCHAR NOT NULL DEFAULT '1.0'"),
+        ],
+        ["name"],
+    ),
+    (
+        "concepts",
+        [
+            ("scheme_urn", "VARCHAR NOT NULL"),
+            ("concept_id", "VARCHAR NOT NULL"),
+        ],
+        ["name", "description"],
+    ),
+    (
+        "hierarchy_nodes",
+        [
+            ("hierarchy_urn", "VARCHAR NOT NULL"),
+            ("node_id", "VARCHAR NOT NULL"),
+            ("parent_node_id", "VARCHAR"),
+            ("code_urn", "VARCHAR"),
+            # language columns inserted here
+            ("_annotations", "VARCHAR"),
+            ("valid_from", "TIMESTAMP"),
+            ("valid_to", "TIMESTAMP"),
+        ],
+        ["name", "description"],
+    ),
+]
+
+# Primary keys that span multiple columns (table_name → comma-separated cols)
+_COMPOSITE_PKS: dict[str, str] = {
+    "codes": "codelist_urn, code_id",
+    "dsd_components": "dsd_urn, component_id",
+    "concepts": "scheme_urn, concept_id",
+    "hierarchy_nodes": "hierarchy_urn, node_id",
+}
+
+# Indexes to create after tables
+_INDEXES: list[tuple[str, str, str]] = [
+    ("idx_artefacts_type", "artefacts", "artefact_type"),
+    ("idx_artefacts_agency_id", "artefacts", "agency, id"),
+    ("idx_codes_codelist", "codes", "codelist_urn"),
+    ("idx_hnodes_parent", "hierarchy_nodes", "hierarchy_urn, parent_node_id"),
+]
+
+
+# ── DDL generation ───────────────────────────────────────────────────────
+
+
+def _build_create_table(
+    table_name: str,
+    fixed_columns: list[tuple[str, str]],
+    lang_prefixes: list[str],
+    languages: Sequence[str],
+) -> str:
+    """Build a CREATE TABLE statement with language columns injected."""
+    # Find insertion point: language columns go before the first column
+    # whose name starts with '_' or is a trailing fixed column.
+    # Strategy: insert lang columns right after the last non-underscore,
+    # non-trailing column. Simpler: build all columns in order, injecting
+    # lang columns after fixed columns that precede the underscore/trailing ones.
+
+    cols: list[str] = []
+    lang_cols_added = False
+    lang_col_strs = [f"    {prefix}_{lang} VARCHAR" for prefix in lang_prefixes for lang in languages]
+
+    for col_name, col_type in fixed_columns:
+        # Insert language columns before underscore-prefixed columns
+        if not lang_cols_added and col_name.startswith("_"):
+            cols.extend(lang_col_strs)
+            lang_cols_added = True
+        cols.append(f"    {col_name} {col_type}")
+
+    # If no underscore column was found, append lang columns at the end
+    if not lang_cols_added:
+        cols.extend(lang_col_strs)
+
+    # Add composite primary key if needed
+    if table_name in _COMPOSITE_PKS:
+        cols.append(f"    PRIMARY KEY ({_COMPOSITE_PKS[table_name]})")
+
+    if table_name == "artefacts":
+        cols.append("    UNIQUE (artefact_type, agency, id, version)")
+
+    body = ",\n".join(cols)
+    return f"""\
+CREATE TABLE IF NOT EXISTS {table_name} (
+{body}
+);"""
+
+
+def _build_create_index(index_name: str, table_name: str, columns: str) -> str:
+    return f"""\
+CREATE INDEX IF NOT EXISTS {index_name}
+ON {table_name}({columns});"""
+
+
+# ── Public API ───────────────────────────────────────────────────────────
+
+
+def create_tables(
+    conn: duckdb.DuckDBPyConnection,
+    languages: Sequence[str] = ("en",),
+) -> None:
+    """Create all SDMX structural metadata tables.
+
+    Args:
+        conn: DuckDB connection (file or ``:memory:``).
+        languages: Language codes for translatable columns.
+            Each configured language generates ``name_{lang}``,
+            ``label_{lang}``, ``description_{lang}`` etc. columns
+            on the relevant tables. Defaults to ``["en"]``.
+
+    The configured language list is persisted in a ``_meta`` table
+    so that writers and readers can discover it later via
+    :func:`get_languages`.
+    """
+    if not languages:
+        msg = "languages must contain at least one language code"
+        raise ValueError(msg)
+
+    # Meta table for configuration
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS _meta (
+            key VARCHAR PRIMARY KEY,
+            value VARCHAR
+        );
+    """)
+    conn.execute(
+        """
+        INSERT OR REPLACE INTO _meta (key, value)
+        VALUES ($key, $value)
+        """,
+        {"key": "languages", "value": ",".join(languages)},
+    )
+
+    # Create all structural tables
+    for table_name, fixed_columns, lang_prefixes in _TABLE_DEFS:
+        ddl = _build_create_table(table_name, fixed_columns, lang_prefixes, languages)
+        conn.execute(ddl)
+
+    # Create indexes
+    for index_name, table_name, columns in _INDEXES:
+        conn.execute(_build_create_index(index_name, table_name, columns))
+
+
+def get_languages(conn: duckdb.DuckDBPyConnection) -> list[str]:
+    """Read the configured language list from the ``_meta`` table.
+
+    Returns:
+        List of language codes (e.g. ``["en", "fr"]``).
+
+    Raises:
+        RuntimeError: If the ``_meta`` table or languages key is missing
+            (i.e. :func:`create_tables` was never called).
+    """
+    try:
+        row = conn.execute("""
+            SELECT value FROM _meta WHERE key = 'languages'
+        """).fetchone()
+    except duckdb.CatalogException:
+        msg = "Storage not initialised — call create_tables() first"
+        raise RuntimeError(msg) from None
+    if row is None:
+        msg = "Storage not initialised — call create_tables() first"
+        raise RuntimeError(msg)
+    return row[0].split(",")
+
+
+def tables_exist(conn: duckdb.DuckDBPyConnection) -> bool:
+    """Check whether the storage schema has been created."""
+    try:
+        conn.execute("""
+            SELECT 1 FROM _meta LIMIT 1
+        """)
+    except duckdb.CatalogException:
+        return False
+    else:
+        return True