openlakeforge-domain-model 0.2.0a1__tar.gz

openlakeforge_domain_model-0.2.0a1/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: openlakeforge-domain-model
+Version: 0.2.0a1
+Summary: Canonical provider-neutral domain model for OpenLakeForge
+Requires-Python: >=3.12
+Requires-Dist: PyYAML<7,>=6.0

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain/__init__.py

@@ -0,0 +1,67 @@
+"""Canonical provider-neutral lakehouse descriptors and inventory."""
+
+from __future__ import annotations
+
+from .descriptors import (
+    LAKEHOUSE_API_VERSION,
+    LAKEHOUSE_KIND,
+    SOURCE_API_VERSION,
+    SOURCE_KIND,
+    LakehouseDescriptorError,
+    load_lakehouse_descriptor,
+    load_source_descriptor,
+    validate_lakehouse_descriptor,
+    validate_source_descriptor,
+)
+from .inventory import (
+    ArtifactPrefixes,
+    CatalogNamespace,
+    Dashboard,
+    Domain,
+    LakehouseInventory,
+    PhysicalInventory,
+    PhysicalDomainNames,
+    PhysicalProductNames,
+    PhysicalSourceNames,
+    Product,
+    GoldTable,
+    SilverTable,
+    SourceResource,
+    Source,
+    Table,
+    inventory_for,
+    load_lakehouse_inventory,
+    load_lakehouse_inventory_from_descriptors,
+    load_transitional_lakehouse_inventory,
+)
+
+__all__ = [
+    "LAKEHOUSE_API_VERSION",
+    "LAKEHOUSE_KIND",
+    "SOURCE_API_VERSION",
+    "SOURCE_KIND",
+    "ArtifactPrefixes",
+    "CatalogNamespace",
+    "Dashboard",
+    "Domain",
+    "LakehouseDescriptorError",
+    "LakehouseInventory",
+    "PhysicalInventory",
+    "PhysicalDomainNames",
+    "PhysicalProductNames",
+    "PhysicalSourceNames",
+    "Product",
+    "GoldTable",
+    "SilverTable",
+    "SourceResource",
+    "Source",
+    "Table",
+    "inventory_for",
+    "load_lakehouse_descriptor",
+    "load_lakehouse_inventory",
+    "load_lakehouse_inventory_from_descriptors",
+    "load_transitional_lakehouse_inventory",
+    "load_source_descriptor",
+    "validate_lakehouse_descriptor",
+    "validate_source_descriptor",
+]

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain/descriptors.py

@@ -0,0 +1,336 @@
+"""Versioned descriptor validation for the Lakehouse and Source kinds.
+
+The canonical v1alpha3 model is the ``Lakehouse`` descriptor at
+``lakehouse_code/lakehouse.yaml`` together with one ``Source`` descriptor per
+bronze source at ``lakehouse_code/bronze/<source>/source.yaml``.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+LAKEHOUSE_API_VERSION = "openlakeforge.io/v1alpha3"
+SOURCE_API_VERSION = "openlakeforge.io/v1alpha3"
+LAKEHOUSE_KIND = "Lakehouse"
+SOURCE_KIND = "Source"
+_PRODUCT_IDENTIFIER_PATTERN = r"[a-z][a-z0-9_]*"
+_IDENTIFIER_PATTERN = re.compile(rf"^{_PRODUCT_IDENTIFIER_PATTERN}$")
+
+
+class LakehouseDescriptorError(ValueError):
+    """Raised when a v1alpha3 lakehouse or source descriptor is invalid."""
+
+
+def _identifier(value: object, *, field: str, source: str) -> str:
+    if not isinstance(value, str) or not _IDENTIFIER_PATTERN.fullmatch(value):
+        raise LakehouseDescriptorError(f"{source}: {field} must match '^[a-z][a-z0-9_]*$'")
+    return value
+
+
+def _non_empty_string(value: object, *, field: str, source: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise LakehouseDescriptorError(f"{source}: {field} must be a non-empty string")
+    return value
+
+
+def _optional_string(value: object, *, field: str, source: str) -> None:
+    if value is not None and not isinstance(value, str):
+        raise LakehouseDescriptorError(f"{source}: {field} must be a string")
+
+
+def validate_source_descriptor(document: Mapping[str, Any], *, source: str = "source.yaml") -> None:
+    """Validate a v1alpha3 Source descriptor envelope and resource list."""
+    if document.get("apiVersion") != SOURCE_API_VERSION:
+        raise LakehouseDescriptorError(
+            f"{source}: unsupported apiVersion {document.get('apiVersion')!r}; expected {SOURCE_API_VERSION!r}"
+        )
+    if document.get("kind") != SOURCE_KIND:
+        raise LakehouseDescriptorError(f"{source}: kind must be {SOURCE_KIND!r}")
+    for field in ("name", "displayName", "description", "status", "resources"):
+        if field not in document:
+            raise LakehouseDescriptorError(f"{source}: missing required field {field!r}")
+    _identifier(document["name"], field="name", source=source)
+    _non_empty_string(document["displayName"], field="displayName", source=source)
+    _non_empty_string(document["status"], field="status", source=source)
+    _optional_string(document["description"], field="description", source=source)
+    resources = document["resources"]
+    if not isinstance(resources, list) or not resources:
+        raise LakehouseDescriptorError(f"{source}: resources must be a non-empty array")
+    seen: set[str] = set()
+    for index, resource in enumerate(resources):
+        if not isinstance(resource, Mapping):
+            raise LakehouseDescriptorError(f"{source}: resources[{index}] must be an object")
+        if "name" not in resource:
+            raise LakehouseDescriptorError(f"{source}: resources[{index}] is missing required field 'name'")
+        name = _identifier(resource["name"], field=f"resources[{index}].name", source=source)
+        if name in seen:
+            raise LakehouseDescriptorError(f"{source}: duplicate resource name {name!r}")
+        seen.add(name)
+        if "description" in resource and not isinstance(resource["description"], str):
+            raise LakehouseDescriptorError(f"{source}: resources[{index}].description must be a string")
+        unexpected = set(resource) - {"name", "description"}
+        if unexpected:
+            raise LakehouseDescriptorError(
+                f"{source}: resources[{index}] must not contain {sorted(unexpected)!r} (provider-neutral fields only)"
+            )
+    unexpected = set(document) - {"apiVersion", "kind", "name", "displayName", "description", "status", "resources"}
+    if unexpected:
+        raise LakehouseDescriptorError(
+            f"{source}: must not contain {sorted(unexpected)!r} (provider-neutral fields only)"
+        )
+
+
+def _validate_table_group(product: Mapping[str, Any], *, group: str, source: str) -> None:
+    spec = product.get(group)
+    if not isinstance(spec, Mapping):
+        raise LakehouseDescriptorError(f"{source}: product {product.get('id')!r}: {group} must be an object")
+    tables = spec.get("tables")
+    if not isinstance(tables, list) or not tables:
+        raise LakehouseDescriptorError(f"{source}: product {product.get('id')!r}: {group}.tables must be a non-empty array")
+    seen: set[str] = set()
+    for index, table in enumerate(tables):
+        if not isinstance(table, Mapping) or not _IDENTIFIER_PATTERN.fullmatch(table.get("name") or ""):
+            raise LakehouseDescriptorError(
+                f"{source}: product {product.get('id')!r}: {group}.tables[{index}].name must match '^[a-z][a-z0-9_]*$'"
+            )
+        if table["name"] in seen:
+            raise LakehouseDescriptorError(
+                f"{source}: product {product.get('id')!r}: {group}.tables must not contain duplicate names"
+            )
+        seen.add(table["name"])
+        if "description" in table and not isinstance(table["description"], str):
+            raise LakehouseDescriptorError(f"{source}: product {product.get('id')!r}: {group}.tables[{index}].description must be a string")
+        unexpected = set(table) - {"name", "description"}
+        if unexpected:
+            raise LakehouseDescriptorError(
+                f"{source}: product {product.get('id')!r}: {group}.tables[{index}] must not contain "
+                f"{sorted(unexpected)!r} (provider-neutral fields only)"
+            )
+    if "schema" in spec:
+        raise LakehouseDescriptorError(
+            f"{source}: product {product.get('id')!r}: {group}.schema must be derived from provider contracts"
+        )
+    unexpected = set(spec) - {"tables"}
+    if unexpected:
+        raise LakehouseDescriptorError(
+            f"{source}: product {product.get('id')!r}: {group} must not contain {sorted(unexpected)!r} "
+            "(provider-neutral fields only)"
+        )
+
+
+def _validate_product(product: Mapping[str, Any], *, domain_name: str, source: str) -> None:
+    label = product.get("id") or f"product {product!r}"
+    for field in ("id", "displayName", "description", "status"):
+        if field not in product:
+            raise LakehouseDescriptorError(f"{source}: product {label!r}: missing required field {field!r}")
+    _identifier(product["id"], field="id", source=f"{source}: product {label!r}")
+    _non_empty_string(product["displayName"], field="displayName", source=f"{source}: product {label!r}")
+    _non_empty_string(product["status"], field="status", source=f"{source}: product {label!r}")
+    _optional_string(product["description"], field="description", source=f"{source}: product {label!r}")
+    if "domain" in product and product["domain"] != domain_name:
+        raise LakehouseDescriptorError(
+            f"{source}: product {label!r}: domain {product['domain']!r} must match enclosing domain {domain_name!r}"
+        )
+    silver_inputs = product.get("silver_inputs")
+    if not isinstance(silver_inputs, list) or not silver_inputs:
+        raise LakehouseDescriptorError(f"{source}: product {label!r}: silver_inputs must be a non-empty array")
+    if any(not isinstance(value, str) or not _IDENTIFIER_PATTERN.fullmatch(value) for value in silver_inputs):
+        raise LakehouseDescriptorError(
+            f"{source}: product {label!r}: silver_inputs must contain identifiers matching '^[a-z][a-z0-9_]*$'"
+        )
+    if len(silver_inputs) != len(set(silver_inputs)):
+        raise LakehouseDescriptorError(f"{source}: product {label!r}: silver_inputs must not contain duplicates")
+    _validate_table_group(product, group="gold_tables", source=source)
+    unexpected = set(product) - {
+        "id", "displayName", "description", "status", "domain", "silver_inputs", "gold_tables",
+    }
+    if unexpected:
+        raise LakehouseDescriptorError(
+            f"{source}: product {label!r} must not contain {sorted(unexpected)!r} (provider-neutral fields only)"
+        )
+
+
+def _validate_domain(domain: Mapping[str, Any], *, source: str) -> None:
+    required = ("name", "displayName", "description", "status", "silver_tables", "products")
+    for field in required:
+        if field not in domain:
+            raise LakehouseDescriptorError(f"{source}: domain: missing required field {field!r}")
+    _identifier(domain["name"], field="domain.name", source=source)
+    _non_empty_string(domain["displayName"], field="domain.displayName", source=source)
+    _non_empty_string(domain["status"], field="domain.status", source=source)
+    _optional_string(domain["description"], field="domain.description", source=source)
+    _validate_silver_table_group(domain, source=source)
+    products = domain["products"]
+    if not isinstance(products, list):
+        raise LakehouseDescriptorError(f"{source}: domain {domain['name']!r}: products must be an array")
+    seen_products: set[str] = set()
+    silver_names = {table["name"] for table in domain["silver_tables"]["tables"]}
+    for product in products:
+        if not isinstance(product, Mapping):
+            raise LakehouseDescriptorError(f"{source}: domain {domain['name']!r}: products must contain objects")
+        _validate_product(product, domain_name=domain["name"], source=source)
+        if product["id"] in seen_products:
+            raise LakehouseDescriptorError(
+                f"{source}: domain {domain['name']!r}: duplicate product id {product['id']!r}"
+            )
+        seen_products.add(product["id"])
+        unknown_inputs = sorted(set(product["silver_inputs"]) - silver_names)
+        if unknown_inputs:
+            raise LakehouseDescriptorError(
+                f"{source}: product {product['id']!r}: silver_inputs reference unknown domain Silver tables "
+                f"{unknown_inputs!r}"
+            )
+    unexpected = set(domain) - {"name", "displayName", "description", "status", "silver_tables", "products"}
+    if unexpected:
+        raise LakehouseDescriptorError(
+            f"{source}: domain {domain['name']!r} must not contain {sorted(unexpected)!r} (provider-neutral fields only)"
+        )
+
+
+def _validate_silver_table_group(domain: Mapping[str, Any], *, source: str) -> None:
+    spec = domain.get("silver_tables")
+    if not isinstance(spec, Mapping) or not isinstance(spec.get("tables"), list) or not spec["tables"]:
+        raise LakehouseDescriptorError(f"{source}: domain {domain.get('name')!r}: silver_tables.tables must be a non-empty array")
+    if set(spec) != {"tables"}:
+        unexpected = sorted(set(spec) - {"tables"})
+        raise LakehouseDescriptorError(
+            f"{source}: domain {domain.get('name')!r}: silver_tables must not contain {unexpected!r} "
+            "(provider-neutral fields only)"
+        )
+    seen: set[str] = set()
+    for index, table in enumerate(spec["tables"]):
+        if not isinstance(table, Mapping) or not _IDENTIFIER_PATTERN.fullmatch(table.get("name") or ""):
+            raise LakehouseDescriptorError(f"{source}: domain {domain.get('name')!r}: silver_tables.tables[{index}].name must be an identifier")
+        if table["name"] in seen:
+            raise LakehouseDescriptorError(f"{source}: domain {domain.get('name')!r}: silver_tables.tables must not contain duplicate names")
+        seen.add(table["name"])
+        if not _IDENTIFIER_PATTERN.fullmatch(table.get("source") or "") or not _IDENTIFIER_PATTERN.fullmatch(table.get("resource") or ""):
+            raise LakehouseDescriptorError(f"{source}: domain {domain.get('name')!r}: silver_tables.tables[{index}] must declare source and resource identifiers")
+        if "description" in table and not isinstance(table["description"], str):
+            raise LakehouseDescriptorError(f"{source}: domain {domain.get('name')!r}: silver_tables.tables[{index}].description must be a string")
+        unexpected = set(table) - {"name", "source", "resource", "description"}
+        if unexpected:
+            raise LakehouseDescriptorError(f"{source}: domain {domain.get('name')!r}: silver_tables.tables[{index}] must not contain {sorted(unexpected)!r}")
+
+
+def _validate_dashboard(dashboard: Mapping[str, Any], *, source: str) -> None:
+    for field in ("name", "products"):
+        if field not in dashboard:
+            raise LakehouseDescriptorError(f"{source}: dashboard: missing required field {field!r}")
+    _identifier(dashboard["name"], field="dashboard.name", source=source)
+    products = dashboard["products"]
+    if not isinstance(products, list) or not products:
+        raise LakehouseDescriptorError(f"{source}: dashboard {dashboard['name']!r}: products must be a non-empty array")
+    if any(not isinstance(product, str) or not _IDENTIFIER_PATTERN.fullmatch(product) for product in products):
+        raise LakehouseDescriptorError(f"{source}: dashboard {dashboard['name']!r}: products must contain identifiers")
+    if len(products) != len(set(products)):
+        raise LakehouseDescriptorError(f"{source}: dashboard {dashboard['name']!r}: products must not contain duplicates")
+    unexpected = set(dashboard) - {"name", "products"}
+    if unexpected:
+        raise LakehouseDescriptorError(
+            f"{source}: dashboard {dashboard['name']!r} must not contain {sorted(unexpected)!r} "
+            "(provider-neutral fields only)"
+        )
+
+
+def validate_lakehouse_descriptor(
+    document: Mapping[str, Any], *, source: str = "lakehouse.yaml", allow_incomplete: bool = False
+) -> None:
+    """Validate the v1alpha3 Lakehouse envelope, domains, and dashboards."""
+    if document.get("apiVersion") != LAKEHOUSE_API_VERSION:
+        raise LakehouseDescriptorError(
+            f"{source}: unsupported apiVersion {document.get('apiVersion')!r}; expected {LAKEHOUSE_API_VERSION!r}"
+        )
+    if document.get("kind") != LAKEHOUSE_KIND:
+        raise LakehouseDescriptorError(f"{source}: kind must be {LAKEHOUSE_KIND!r}")
+    for field in ("name", "displayName", "description", "status", "sources", "domains", "dashboards"):
+        if field not in document:
+            raise LakehouseDescriptorError(f"{source}: missing required field {field!r}")
+    _identifier(document["name"], field="name", source=source)
+    _non_empty_string(document["displayName"], field="displayName", source=source)
+    _non_empty_string(document["status"], field="status", source=source)
+    _optional_string(document["description"], field="description", source=source)
+    sources = document["sources"]
+    if not isinstance(sources, list) or (not sources and not allow_incomplete):
+        raise LakehouseDescriptorError(f"{source}: sources must be a non-empty array")
+    for index, source_name in enumerate(sources):
+        _identifier(source_name, field=f"sources[{index}]", source=source)
+    if len(sources) != len(set(sources)):
+        raise LakehouseDescriptorError(f"{source}: sources must not contain duplicates")
+    domains = document["domains"]
+    if not isinstance(domains, list) or (not domains and not allow_incomplete):
+        raise LakehouseDescriptorError(f"{source}: domains must be a non-empty array")
+    seen_domains: set[str] = set()
+    seen_products: set[str] = set()
+    for domain in domains:
+        if not isinstance(domain, Mapping):
+            raise LakehouseDescriptorError(f"{source}: domains must contain objects")
+        _validate_domain(domain, source=source)
+        if domain["name"] in seen_domains:
+            raise LakehouseDescriptorError(f"{source}: duplicate domain name {domain['name']!r}")
+        seen_domains.add(domain["name"])
+        for product in domain["products"]:
+            if product["id"] in seen_products:
+                raise LakehouseDescriptorError(
+                    f"{source}: product id {product['id']!r} must be globally unique across domains"
+                )
+            seen_products.add(product["id"])
+    if not seen_products and not allow_incomplete:
+        raise LakehouseDescriptorError(
+            f"{source}: lakehouse must declare at least one product across its domains "
+            "(an individual domain may have zero products while it is being seeded, "
+            "but the lakehouse as a whole cannot -- deployment tooling requires a "
+            "default product)"
+        )
+    dashboards = document["dashboards"]
+    if not isinstance(dashboards, list):
+        raise LakehouseDescriptorError(f"{source}: dashboards must be an array")
+    seen_dashboards: set[str] = set()
+    for dashboard in dashboards:
+        if not isinstance(dashboard, Mapping):
+            raise LakehouseDescriptorError(f"{source}: dashboards must contain objects")
+        _validate_dashboard(dashboard, source=source)
+        if dashboard["name"] in seen_dashboards:
+            raise LakehouseDescriptorError(f"{source}: duplicate dashboard name {dashboard['name']!r}")
+        seen_dashboards.add(dashboard["name"])
+        unknown_products = sorted(set(dashboard["products"]) - seen_products)
+        if unknown_products:
+            raise LakehouseDescriptorError(
+                f"{source}: dashboard {dashboard['name']!r}: products reference unknown product ids "
+                f"{unknown_products!r}"
+            )
+    unexpected = set(document) - {
+        "apiVersion", "kind", "name", "displayName", "description", "status", "sources", "domains", "dashboards",
+    }
+    if unexpected:
+        raise LakehouseDescriptorError(
+            f"{source}: must not contain {sorted(unexpected)!r} (provider-neutral fields only)"
+        )
+
+
+def load_lakehouse_descriptor(path: str | Path, *, allow_incomplete: bool = False) -> dict[str, Any]:
+    """Load and validate the v1alpha3 Lakehouse descriptor at ``path``."""
+    source = str(path)
+    with Path(path).open(encoding="utf-8") as handle:
+        document = yaml.safe_load(handle)
+    if not isinstance(document, dict):
+        raise LakehouseDescriptorError(f"{source}: descriptor must contain a YAML object")
+    validate_lakehouse_descriptor(document, source=source, allow_incomplete=allow_incomplete)
+    return document
+
+
+def load_source_descriptor(path: str | Path) -> dict[str, Any]:
+    """Load and validate the v1alpha3 Source descriptor at ``path``."""
+    source = str(path)
+    with Path(path).open(encoding="utf-8") as handle:
+        document = yaml.safe_load(handle)
+    if not isinstance(document, dict):
+        raise LakehouseDescriptorError(f"{source}: descriptor must contain a YAML object")
+    validate_source_descriptor(document, source=source)
+    return document
+

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain/inventory.py

@@ -0,0 +1,627 @@
+"""Typed, provider-neutral inventory derived from lakehouse descriptors.
+
+The canonical v1alpha3 inventory is composed from the ``Lakehouse`` descriptor
+at ``lakehouse_code/lakehouse.yaml`` and one ``Source`` descriptor per bronze
+source at ``lakehouse_code/bronze/<source>/source.yaml``.
+
+Canonical identities:
+
+* Bronze  ``<source>.<resource>``
+* Silver  ``<domain>.<table>``
+* Gold    ``<product>.<table>``
+
+with the dependency chain Source/Bronze -> Domain/Silver -> Product/Gold ->
+Dashboard. The legacy v1alpha1/v1alpha2 ``Domain`` inventory loaders below are
+preserved for migration diagnostics; the canonical loaders are the
+``load_lakehouse_inventory*`` functions.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from typing import Any
+
+from .descriptors import (
+    LakehouseDescriptorError,
+    load_lakehouse_descriptor,
+    load_source_descriptor,
+)
+
+_IDENTIFIER_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+@dataclass(frozen=True)
+class Table:
+    """A provider-neutral logical table or source resource."""
+
+    name: str
+
+
+@dataclass(frozen=True)
+class SourceResource:
+    """A source-owned Bronze resource."""
+
+    source: str
+    name: str
+
+
+@dataclass(frozen=True)
+class SilverTable:
+    """A domain-owned Silver table and its source resource."""
+
+    domain: str
+    name: str
+    source: str
+    resource: str
+
+
+@dataclass(frozen=True)
+class GoldTable:
+    """A product-owned Gold table."""
+
+    product: str
+    name: str
+
+
+@dataclass(frozen=True)
+class ArtifactPrefixes:
+    """Object-store locations derived from an owner's logical identity."""
+
+    manifest_key: str
+    floe_report_prefix: str
+    dbt_artifact_prefix: str
+
+
+@dataclass(frozen=True)
+class Source:
+    """A bronze source and the resources it owns."""
+
+    descriptor_path: Path
+    name: str
+    display_name: str
+    resources: tuple[Table, ...]
+
+    @property
+    def bronze_namespace(self) -> str:
+        return f"{self.name}_bronze"
+
+
+@dataclass(frozen=True)
+class Product:
+    """A v1alpha3 product and its provider-neutral derived identities.
+
+    Product identity is the product ``id`` (also the product ``name``); it is
+    globally unique across the lakehouse inventory. The Silver layer is
+    domain-aligned (``{domain}_silver``), the Gold layer is product-aligned
+    (``{product}_gold``).
+    """
+
+    domain_name: str
+    id: str
+    display_name: str
+    silver_inputs: tuple[str, ...]
+    gold_tables: tuple[Table, ...]
+
+    @property
+    def name(self) -> str:
+        return self.id
+
+    @property
+    def job_name(self) -> str:
+        return f"{self.id}_pipeline"
+
+    @property
+    def silver_namespace(self) -> str:
+        return f"{self.domain_name}_silver"
+
+    @property
+    def gold_namespace(self) -> str:
+        return f"{self.id}_gold"
+
+    @property
+    def definitions_module(self) -> str:
+        return f"lakehouse_code.pipelines.dagster.{self.id}"
+
+    @property
+    def dbt_artifact_prefix(self) -> str:
+        return f"run-artifacts/dbt/{self.domain_name}/{self.id}/"
+
+    @property
+    def dbt_project_dir(self) -> str:
+        return f"lakehouse_code/gold/{self.id}/dbt"
+
+    @property
+    def openmetadata_data_product_fqns(self) -> tuple[str, str]:
+        return self.id, f"{self.domain_name}.{self.id}"
+
+    @property
+    def gold_mart_names(self) -> tuple[str, ...]:
+        return tuple(f"{self.gold_namespace}.{table.name}" for table in self.gold_tables)
+
+
+@dataclass(frozen=True)
+class Domain:
+    """A v1alpha3 domain and its products."""
+
+    descriptor_path: Path
+    name: str
+    display_name: str
+    silver_tables: tuple[SilverTable, ...]
+    products: tuple[Product, ...]
+
+    @property
+    def artifact_prefixes(self) -> ArtifactPrefixes:
+        return ArtifactPrefixes(
+            manifest_key=f"floe/manifests/{self.name}/{self.name}.manifest.json",
+            floe_report_prefix=f"floe/reports/{self.name}/",
+            dbt_artifact_prefix="",
+        )
+
+    @property
+    def silver_namespace(self) -> str:
+        return f"{self.name}_silver"
+
+
+@dataclass(frozen=True)
+class Dashboard:
+    """A consumption-aligned dashboard bound to one or more products."""
+
+    name: str
+    products: tuple[str, ...]
+
+    @property
+    def report_source_dir(self) -> str:
+        return f"lakehouse_code/dashboards/superset/{self.name}"
+
+    @property
+    def superset_export_bundle_name(self) -> str:
+        return f"{self.name}_superset_assets_export.zip"
+
+
+@dataclass(frozen=True)
+class CatalogNamespace:
+    """A physical catalog namespace and its object-store location."""
+
+    name: str
+    location: str
+
+
+@dataclass(frozen=True)
+class PhysicalProductNames:
+    """Provider-specific names resolved for one logical product."""
+
+    product: str
+    gold_namespace: str
+    gold_schema_fqn: str
+
+
+@dataclass(frozen=True)
+class PhysicalDomainNames:
+    """Provider-specific names resolved for one logical Silver domain."""
+
+    domain: str
+    silver_namespace: str
+    silver_schema_fqn: str
+    manifest_uri: str
+
+
+@dataclass(frozen=True)
+class PhysicalSourceNames:
+    """Provider-specific names resolved for one logical bronze source."""
+
+    source: str
+    bronze_namespace: str
+    bronze_schema_fqn: str
+
+
+@dataclass(frozen=True)
+class PhysicalInventory:
+    """Physical catalog and artifact identities for a provider contract."""
+
+    catalog_namespaces: tuple[CatalogNamespace, ...]
+    products: tuple[PhysicalProductNames, ...]
+    domains: tuple[PhysicalDomainNames, ...] = ()
+    sources: tuple[PhysicalSourceNames, ...] = ()
+
+    @property
+    def silver_namespaces(self) -> dict[str, str]:
+        return {domain.domain: domain.silver_namespace for domain in self.domains}
+
+    @property
+    def gold_namespaces(self) -> dict[str, str]:
+        return {product.product: product.gold_namespace for product in self.products}
+
+    @property
+    def silver_schema_fqns(self) -> dict[str, str]:
+        return {domain.domain: domain.silver_schema_fqn for domain in self.domains}
+
+    @property
+    def gold_schema_fqns(self) -> dict[str, str]:
+        return {product.product: product.gold_schema_fqn for product in self.products}
+
+    @property
+    def bronze_namespaces(self) -> dict[str, str]:
+        return {source.source: source.bronze_namespace for source in self.sources}
+
+    @property
+    def bronze_schema_fqns(self) -> dict[str, str]:
+        return {source.source: source.bronze_schema_fqn for source in self.sources}
+
+    @property
+    def domain_floe_manifest_uris(self) -> dict[str, str]:
+        return {domain.domain: domain.manifest_uri for domain in self.domains}
+
+
+@dataclass(frozen=True)
+class LakehouseInventory:
+    """All sources, domains, products, and dashboards declared in one lakehouse."""
+
+    lakehouse_root: Path
+    name: str
+    sources: tuple[Source, ...]
+    domains: tuple[Domain, ...]
+    dashboards: tuple[Dashboard, ...]
+
+    @property
+    def products(self) -> tuple[Product, ...]:
+        return tuple(product for domain in self.domains for product in domain.products)
+
+    @property
+    def job_names(self) -> tuple[str, ...]:
+        return tuple(product.job_name for product in self.products)
+
+    @property
+    def catalog_namespaces(self) -> tuple[str, ...]:
+        return tuple(
+            namespace
+            for namespace in (*self.bronze_namespace_names, *self.silver_namespace_names, *self.gold_namespace_names)
+        )
+
+    @property
+    def default_product(self) -> Product:
+        if not self.products:
+            raise LakehouseDescriptorError(f"{self.lakehouse_root}: no data products were discovered")
+        return self.products[0]
+
+    @property
+    def artifact_prefixes(self) -> tuple[ArtifactPrefixes, ...]:
+        return tuple(domain.artifact_prefixes for domain in self.domains)
+
+    @property
+    def domain_names(self) -> tuple[str, ...]:
+        return tuple(domain.name for domain in self.domains)
+
+    @property
+    def source_names(self) -> tuple[str, ...]:
+        return tuple(source.name for source in self.sources)
+
+    @property
+    def silver_table_count(self) -> int:
+        """Count of distinct product-reachable physical Silver relations.
+
+        A domain may declare Silver tables ahead of any product consuming
+        them (see the product-less-domain scaffolding path), so this counts
+        only tables referenced by at least one product's ``silver_inputs``,
+        not every declared table. Two products in the same domain can
+        declare the same table (they share the domain's Silver namespace),
+        so this counts unique ``(domain, table)`` pairs rather than summing
+        per-product declarations, which would double-count a shared table.
+        """
+        return sum(
+            len({name for product in domain.products for name in product.silver_inputs})
+            for domain in self.domains
+        )
+
+    @property
+    def gold_table_count(self) -> int:
+        return sum(len(product.gold_tables) for product in self.products)
+
+    @property
+    def gold_mart_names(self) -> tuple[str, ...]:
+        return tuple(mart for product in self.products for mart in product.gold_mart_names)
+
+    @property
+    def manifest_keys(self) -> tuple[str, ...]:
+        return tuple(domain.artifact_prefixes.manifest_key for domain in self.domains)
+
+    @property
+    def openmetadata_data_products(self) -> dict[str, tuple[str, str]]:
+        return {product.name: product.openmetadata_data_product_fqns for product in self.products}
+
+    @property
+    def bronze_namespace_names(self) -> frozenset[str]:
+        return frozenset(source.bronze_namespace for source in self.sources)
+
+    @property
+    def silver_namespace_names(self) -> frozenset[str]:
+        return frozenset(domain.silver_namespace for domain in self.domains)
+
+    def domain_for_product(self, product: Product) -> Domain:
+        return next(domain for domain in self.domains if domain.name == product.domain_name)
+
+    def resolved_silver_tables(self, product: Product) -> tuple[SilverTable, ...]:
+        domain = self.domain_for_product(product)
+        by_name = {table.name: table for table in domain.silver_tables}
+        return tuple(by_name[name] for name in product.silver_inputs)
+
+    def bronze_resources_for_product(self, product: Product) -> tuple[SourceResource, ...]:
+        return tuple(SourceResource(table.source, table.resource) for table in self.resolved_silver_tables(product))
+
+    @property
+    def gold_namespace_names(self) -> frozenset[str]:
+        return frozenset(product.gold_namespace for product in self.products)
+
+    def resolve_physical_names(
+        self,
+        *,
+        catalog_database_fqn: str,
+        bronze_bucket: str,
+        silver_bucket: str,
+        gold_bucket: str,
+        manifest_base_uri: str,
+    ) -> PhysicalInventory:
+        """Resolve physical names from provider-contract values."""
+        namespaces: list[CatalogNamespace] = []
+        products: list[PhysicalProductNames] = []
+        domains: list[PhysicalDomainNames] = []
+        sources: list[PhysicalSourceNames] = []
+        for domain in self.domains:
+            namespaces.append(
+                CatalogNamespace(domain.silver_namespace, f"s3://{silver_bucket}/{domain.silver_namespace}/")
+            )
+            domains.append(
+                PhysicalDomainNames(
+                    domain=domain.name,
+                    silver_namespace=domain.silver_namespace,
+                    silver_schema_fqn=f"{catalog_database_fqn}.{domain.silver_namespace}",
+                    manifest_uri=f"{manifest_base_uri.rstrip('/')}/{domain.name}/{domain.name}.manifest.json",
+                )
+            )
+        for product in self.products:
+            namespaces.append(
+                CatalogNamespace(product.gold_namespace, f"s3://{gold_bucket}/{product.gold_namespace}/")
+            )
+            products.append(
+                PhysicalProductNames(
+                    product=product.name,
+                    gold_namespace=product.gold_namespace,
+                    gold_schema_fqn=f"{catalog_database_fqn}.{product.gold_namespace}",
+                )
+            )
+        for source in self.sources:
+            namespaces.append(
+                CatalogNamespace(source.bronze_namespace, f"s3://{bronze_bucket}/{source.bronze_namespace}/")
+            )
+            sources.append(
+                PhysicalSourceNames(
+                    source=source.name,
+                    bronze_namespace=source.bronze_namespace,
+                    bronze_schema_fqn=f"{catalog_database_fqn}.{source.bronze_namespace}",
+                )
+            )
+        return PhysicalInventory(
+            catalog_namespaces=tuple(namespaces), products=tuple(products), domains=tuple(domains), sources=tuple(sources)
+        )
+
+
+def _lakehouse_root(path: Path) -> Path:
+    candidate = path.resolve()
+    return candidate / "lakehouse_code" if (candidate / "lakehouse_code").is_dir() else candidate
+
+
+def _source_table_names(source: Mapping[str, Any]) -> set[str]:
+    resources = source.get("resources")
+    if not isinstance(resources, list):
+        return set()
+    return {resource["name"] for resource in resources if isinstance(resource, Mapping) and "name" in resource}
+
+
+def _product(
+    path: Path, domain_name: str, product: Mapping[str, Any], index: int, silver_tables: tuple[SilverTable, ...]
+) -> Product:
+    label = product.get("id") or f"products[{index}]"
+    product_id = product.get("id")
+    if not isinstance(product_id, str) or not _IDENTIFIER_PATTERN.fullmatch(product_id):
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: id must match '^[a-z][a-z0-9_]*$'")
+    display_name = product.get("displayName")
+    if not isinstance(display_name, str) or not display_name:
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: displayName is required and must be a non-empty string")
+    inputs = product.get("silver_inputs")
+    if not isinstance(inputs, list) or not inputs:
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: silver_inputs must be a non-empty array")
+    return Product(
+        domain_name=domain_name,
+        id=product_id,
+        display_name=display_name,
+        silver_inputs=tuple(inputs),
+        gold_tables=_table_group(path, product, "gold_tables", label),
+    )
+
+
+def _silver_tables(path: Path, domain_name: str, domain: Mapping[str, Any]) -> tuple[SilverTable, ...]:
+    tables = domain["silver_tables"]["tables"]
+    return tuple(
+        SilverTable(domain=domain_name, name=table["name"], source=table["source"], resource=table["resource"])
+        for table in tables
+    )
+
+
+def _table_group(path: Path, product: Mapping[str, Any], field: str, label: str) -> tuple[Table, ...]:
+    spec = product.get(field)
+    if not isinstance(spec, Mapping):
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: {field} is required and must be an object")
+    tables = spec.get("tables")
+    if not isinstance(tables, list) or not tables:
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: {field}.tables must be a non-empty array")
+    names = [table.get("name") for table in tables if isinstance(table, Mapping)]
+    if len(names) != len(tables) or any(not isinstance(name, str) or not name for name in names):
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: {field}.tables must contain non-empty string names")
+    if len(names) != len(set(names)):
+        raise LakehouseDescriptorError(f"{path}: product {label!r}: {field}.tables must not contain duplicate names")
+    return tuple(Table(name=name) for name in names)
+
+
+def _validate_inventory_identities(
+    lakehouse_root: Path,
+    sources: tuple[Source, ...],
+    domains: tuple[Domain, ...],
+    dashboards: tuple[Dashboard, ...],
+) -> None:
+    seen_sources: set[str] = set()
+    for source in sources:
+        if source.name in seen_sources:
+            raise LakehouseDescriptorError(f"{source.descriptor_path}: duplicate source name {source.name!r}")
+        seen_sources.add(source.name)
+    seen_domains: set[str] = set()
+    seen_product_ids: set[str] = set()
+    for domain in domains:
+        if domain.name in seen_domains:
+            raise LakehouseDescriptorError(f"{domain.descriptor_path}: duplicate domain name {domain.name!r}")
+        seen_domains.add(domain.name)
+        silver_by_name = {table.name: table for table in domain.silver_tables}
+        for table in domain.silver_tables:
+            source = next((candidate for candidate in sources if candidate.name == table.source), None)
+            if source is None:
+                raise LakehouseDescriptorError(
+                    f"{domain.descriptor_path}: domain {domain.name!r}: Silver table {table.name!r} references unknown source {table.source!r}"
+                )
+            if table.resource not in {resource.name for resource in source.resources}:
+                raise LakehouseDescriptorError(
+                    f"{domain.descriptor_path}: domain {domain.name!r}: Silver table {table.name!r} references unknown "
+                    f"resource {table.resource!r} of source {table.source!r}"
+                )
+        for product in domain.products:
+            if product.id in seen_product_ids:
+                raise LakehouseDescriptorError(
+                    f"{domain.descriptor_path}: product {product.id!r}: id must be globally unique across the lakehouse"
+                )
+            seen_product_ids.add(product.id)
+            unknown = [name for name in product.silver_inputs if name not in silver_by_name]
+            if unknown:
+                raise LakehouseDescriptorError(
+                    f"{domain.descriptor_path}: product {product.id!r}: silver_inputs reference unknown domain Silver tables {unknown!r}"
+                )
+    seen_dashboards: set[str] = set()
+    for dashboard in dashboards:
+        if dashboard.name in seen_dashboards:
+            raise LakehouseDescriptorError(f"{lakehouse_root}: duplicate dashboard name {dashboard.name!r}")
+        seen_dashboards.add(dashboard.name)
+        unknown_products = sorted(set(dashboard.products) - seen_product_ids)
+        if unknown_products:
+            raise LakehouseDescriptorError(
+                f"{lakehouse_root}: dashboard {dashboard.name!r}: products {unknown_products!r} must reference declared products"
+            )
+
+
+def load_lakehouse_inventory_from_descriptors(
+    lakehouse_path: str | Path,
+    source_paths: Sequence[str | Path],
+    *,
+    source_label: str | Path | None = None,
+    allow_incomplete: bool = False,
+) -> LakehouseInventory:
+    """Load and validate one Lakehouse descriptor plus its Source descriptors."""
+    lakehouse_document = load_lakehouse_descriptor(lakehouse_path, allow_incomplete=allow_incomplete)
+    sources: list[Source] = []
+    for source_path in source_paths:
+        document = load_source_descriptor(source_path)
+        name = document["name"]
+        if name != Path(source_path).parent.name:
+            raise LakehouseDescriptorError(
+                f"{source_path}: name {name!r} must match source directory {Path(source_path).parent.name!r}"
+            )
+        resources = tuple(
+            Table(name=resource["name"])
+            for resource in document["resources"]
+            if isinstance(resource, Mapping) and "name" in resource
+        )
+        sources.append(
+            Source(
+                descriptor_path=Path(source_path),
+                name=name,
+                display_name=document["displayName"],
+                resources=resources,
+            )
+        )
+    declared_sources = set(lakehouse_document["sources"])
+    discovered_sources = {source.name for source in sources}
+    if declared_sources != discovered_sources:
+        raise LakehouseDescriptorError(
+            f"{lakehouse_path}: lakehouse sources {sorted(declared_sources)!r} must match discovered "
+            f"source descriptors {sorted(discovered_sources)!r} under lakehouse_code/bronze/*/source.yaml"
+        )
+    domains: list[Domain] = []
+    for index, domain in enumerate(lakehouse_document["domains"]):
+        if not isinstance(domain, Mapping):
+            raise LakehouseDescriptorError(f"{lakehouse_path}: domains[{index}] must be an object")
+        domain_name = domain["name"]
+        silver_tables = _silver_tables(lakehouse_path, domain_name, domain)
+        products = tuple(
+            _product(lakehouse_path, domain_name, product, product_index, silver_tables)
+            for product_index, product in enumerate(domain["products"])
+            if isinstance(product, Mapping)
+        )
+        domains.append(
+            Domain(
+                descriptor_path=Path(lakehouse_path),
+                name=domain_name,
+                display_name=domain["displayName"],
+                silver_tables=silver_tables,
+                products=products,
+            )
+        )
+    dashboards = tuple(
+        Dashboard(
+            name=dashboard["name"],
+            products=tuple(dashboard["products"]),
+        )
+        for dashboard in lakehouse_document["dashboards"]
+        if isinstance(dashboard, Mapping)
+    )
+    inventory = LakehouseInventory(
+        lakehouse_root=Path(source_label) if source_label is not None else Path(lakehouse_path).parent,
+        name=lakehouse_document["name"],
+        sources=tuple(sources),
+        domains=tuple(domains),
+        dashboards=dashboards,
+    )
+    _validate_inventory_identities(inventory.lakehouse_root, inventory.sources, inventory.domains, inventory.dashboards)
+    return inventory
+
+
+def load_lakehouse_inventory(path: str | Path, *, allow_incomplete: bool = False) -> LakehouseInventory:
+    """Load and validate ``lakehouse_code/lakehouse.yaml`` plus every source."""
+    lakehouse_root = _lakehouse_root(Path(path))
+    lakehouse_descriptor = lakehouse_root / "lakehouse.yaml"
+    if not lakehouse_descriptor.is_file():
+        raise LakehouseDescriptorError(f"{lakehouse_root}: no lakehouse descriptor found at lakehouse.yaml")
+    source_descriptors = sorted(lakehouse_root.glob("bronze/*/source.yaml"))
+    if not source_descriptors and not allow_incomplete:
+        raise LakehouseDescriptorError(f"{lakehouse_root}: no source descriptors found at bronze/*/source.yaml")
+    return load_lakehouse_inventory_from_descriptors(
+        lakehouse_descriptor, source_descriptors, source_label=lakehouse_root, allow_incomplete=allow_incomplete
+    )
+
+
+def load_transitional_lakehouse_inventory(path: str | Path) -> LakehouseInventory:
+    """Load an `olf init --empty` project while it has no runnable product.
+
+    This is intentionally limited to scaffold planning. Deployment and
+    ordinary descriptor validation retain the strict v1alpha3 contract.
+    """
+    return load_lakehouse_inventory(path, allow_incomplete=True)
+
+
+@cache
+def _cached_lakehouse_inventory(resolved_path: str) -> LakehouseInventory:
+    return load_lakehouse_inventory(resolved_path)
+
+
+def inventory_for(repo_root: str | Path) -> LakehouseInventory:
+    """Load and cache the canonical lakehouse inventory for a repository root."""
+    return _cached_lakehouse_inventory(str(Path(repo_root).resolve()))
+

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain_model.egg-info/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: openlakeforge-domain-model
+Version: 0.2.0a1
+Summary: Canonical provider-neutral domain model for OpenLakeForge
+Requires-Python: >=3.12
+Requires-Dist: PyYAML<7,>=6.0

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain_model.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+pyproject.toml
+openlakeforge_domain/__init__.py
+openlakeforge_domain/descriptors.py
+openlakeforge_domain/inventory.py
+openlakeforge_domain_model.egg-info/PKG-INFO
+openlakeforge_domain_model.egg-info/SOURCES.txt
+openlakeforge_domain_model.egg-info/dependency_links.txt
+openlakeforge_domain_model.egg-info/requires.txt
+openlakeforge_domain_model.egg-info/top_level.txt

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain_model.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain_model.egg-info/requires.txt

@@ -0,0 +1 @@
+PyYAML<7,>=6.0

openlakeforge_domain_model-0.2.0a1/openlakeforge_domain_model.egg-info/top_level.txt

@@ -0,0 +1 @@
+openlakeforge_domain

openlakeforge_domain_model-0.2.0a1/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openlakeforge-domain-model"
+version = "0.2.0a1"
+description = "Canonical provider-neutral domain model for OpenLakeForge"
+requires-python = ">=3.12"
+dependencies = ["PyYAML>=6.0,<7"]
+
+[tool.setuptools.packages.find]
+include = ["openlakeforge_domain*"]

openlakeforge_domain_model-0.2.0a1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+