dmesh-sdk 0.1.0__py3-none-any.whl

dmesh/sdk/__init__.py

@@ -0,0 +1,39 @@
+from .sdk import AsyncSDK
+from .models import (
+    DataProduct,
+    DataContract,
+    OpenDataMeshError,
+    DataProductValidationError,
+    DataContractValidationError,
+)
+from .operations.data_product import create_dp, update_dp, get_dp, list_dps, delete_dp
+from .operations.data_contract import create_dc, update_dc, patch_dc, get_dc, list_dcs, delete_dc
+from .operations.discover import discover
+from .operations.utils import flush
+from .persistency.factory import RepositoryFactory
+from .config import get_settings
+
+
+__all__ = [
+    "AsyncSDK",
+    "DataProduct",
+    "DataContract",
+    "OpenDataMeshError",
+    "DataProductValidationError",
+    "DataContractValidationError",
+    "create_dp",
+    "update_dp",
+    "get_dp",
+    "list_dps",
+    "delete_dp",
+    "create_dc",
+    "update_dc",
+    "patch_dc",
+    "get_dc",
+    "list_dcs",
+    "delete_dc",
+    "discover",
+    "flush",
+    "RepositoryFactory",
+    "get_settings",
+]

dmesh/sdk/adapters/psycopg.py

@@ -0,0 +1,3 @@
+from dmesh.sdk.persistency.postgres import PostgresDataProductRepository, PostgresDataContractRepository
+
+__all__ = ["PostgresDataProductRepository", "PostgresDataContractRepository"]

dmesh/sdk/config.py

@@ -0,0 +1,152 @@
+import os
+import sys
+import tomllib
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple, Type
+
+from pydantic import BaseModel, Field, ValidationError
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+)
+
+# Deep merge for TOML config
+def _deep_update(base_dict: Dict[str, Any], update_with: Dict[str, Any]):
+    for key, value in update_with.items():
+        if isinstance(value, dict) and key in base_dict and isinstance(base_dict[key], dict):
+            _deep_update(base_dict[key], value)
+        else:
+            base_dict[key] = value
+
+class TomlConfigSettingsSource(PydanticBaseSettingsSource):
+    """
+    A custom settings source that loads configuration from one or more TOML files.
+    Loads 'config/base.toml' first, then 'config/{profile}.toml' as an override.
+    """
+    def __init__(self, settings_cls: Type[BaseSettings], profile: str = "development"):
+        super().__init__(settings_cls)
+        self.profile = profile
+
+    def get_field_value(self, field: Any, field_name: str) -> Tuple[Any, str, bool]:
+        # This is for granular mapping, but we use __call__ to return a dict
+        return None, field_name, False
+
+    def __call__(self) -> Dict[str, Any]:
+        config = {}
+        config_dir = Path("config")
+        
+        # 1. Load base.toml (lowest priority in this source)
+        base_path = config_dir / "base.toml"
+        if base_path.exists():
+            try:
+                with base_path.open("rb") as f:
+                    _deep_update(config, tomllib.load(f))
+            except Exception as e:
+                print(f"Error loading base config: {e}")
+        
+        # 2. Load profile-specific toml (e.g. development.toml, lakebase.toml)
+        profile_path = config_dir / f"{self.profile}.toml"
+        if profile_path.exists():
+            try:
+                with profile_path.open("rb") as f:
+                    _deep_update(config, tomllib.load(f))
+            except Exception as e:
+                print(f"Error loading profile config '{self.profile}': {e}")
+        
+        return config
+
+class DatabaseSettings(BaseModel):
+    host: str = "localhost"
+    port: int = 5432
+    user: str = "postgres"
+    password: str = Field(..., min_length=1) # Required secret
+    name: str = "postgres"
+
+class ApiSettings(BaseModel):
+    host: str = "0.0.0.0"
+    port: int = 8000
+    debug: bool = False
+
+class Settings(BaseSettings):
+    # Field names match the nesting in TOML and env vars
+    db: DatabaseSettings = Field(default_factory=DatabaseSettings)
+    api: ApiSettings = Field(default_factory=ApiSettings)
+    
+    @property
+    def profile(self) -> str:
+        return os.getenv("APP_ENV", "development")
+
+    model_config = SettingsConfigDict(
+        env_prefix="DMESH_",
+        env_nested_delimiter="__",
+        # .env files order defined in get_settings to handle dynamic APP_ENV
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        profile = os.getenv("APP_ENV", "development")
+        return (
+            init_settings,      # 1. CLI flags / runtime args (highest priority)
+            env_settings,       # 2. Environment variables
+            dotenv_settings,    # 3 & 4. .env.{profile} and .env files
+            TomlConfigSettingsSource(settings_cls, profile=profile), # 5 & 6. config/{profile}.toml and config/base.toml
+            # Code defaults (last / lowest priority, implicitly handled by Pydantic)
+        )
+
+_settings: Optional[Settings] = None
+
+def get_settings(**kwargs) -> Settings:
+    """
+    Initializes and returns the global settings instance.
+    The profile is chosen via APP_ENV environment variable (default: development).
+    Uses the priority stack:
+    1. CLI flags / runtime args (passed to get_settings)
+    2. Environment variables (DMESH_ pref)
+    3. .env.{APP_ENV}
+    4. .env
+    5. config/{APP_ENV}.toml
+    6. config/base.toml
+    7. Code defaults
+    """
+    global _settings
+    if _settings is None or kwargs:
+        profile = os.getenv("APP_ENV", "development")
+        
+        # We dynamicallly set env_file list to respect profile priority
+        # Pydantic-settings: later files take precedence
+        env_files = [".env", f".env.{profile}"]
+        
+        class RuntimeSettings(Settings):
+            model_config = SettingsConfigDict(
+                env_prefix="DMESH_",
+                env_nested_delimiter="__",
+                env_file=env_files,
+                env_file_encoding="utf-8",
+                extra="ignore",
+            )
+            
+        try:
+            settings_obj = RuntimeSettings(**kwargs)
+            if not kwargs: # Only cache if it's the default global settings
+                _settings = settings_obj
+            return settings_obj
+        except ValidationError as e:
+            print(f"Invalid configuration:\n{e}")
+            sys.exit(1)
+            
+    return _settings
+
+if __name__ == "__main__":
+    # Example usage / validation test
+    print("Testing configuration loading...")
+    settings = get_settings()
+    print(f"Profile: {settings.profile}")
+    print(f"DB Host: {settings.db.host}")

dmesh/sdk/core/enricher.py

@@ -0,0 +1,58 @@
+"""Enriches an ODPS spec dict before persistence."""
+from typing import Any
+
+from dmesh.sdk.core.id_generator import make_dp_id
+
+
+def enrich_dp_spec(spec: dict[str, Any]) -> dict[str, Any]:
+    """Return a new dict with deterministic id injected and defaults applied.
+
+    The id is derived from domain + name + version (deterministic).
+    Does not mutate the input dict.
+    """
+    enriched = dict(spec)
+    
+    # Apply defaults if missing
+    enriched.setdefault("apiVersion", "v1.0.0")
+    enriched.setdefault("kind", "DataProduct")
+    enriched.setdefault("version", "v1.0.0")
+    enriched.setdefault("status", "draft")
+    # Default outputPorts version to "v1" if missing
+    if "outputPorts" in enriched and isinstance(enriched["outputPorts"], list):
+        new_ports = []
+        for port in enriched["outputPorts"]:
+            if isinstance(port, dict):
+                p = dict(port)
+                p.setdefault("version", "v1")
+                new_ports.append(p)
+            else:
+                new_ports.append(port)
+        enriched["outputPorts"] = new_ports
+    
+    enriched["id"] = make_dp_id(
+        enriched.get("domain", ""),
+        enriched.get("name", ""),
+        enriched.get("version"),
+    )
+    return enriched
+
+
+def enrich_dc_spec(spec: dict[str, Any], dp_spec: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Return a new DataContract spec with defaults applied.
+
+    If a parent Data Product spec is provided, inherit common contextual values.
+    Does not mutate the input dict.
+    """
+    enriched = dict(spec)
+    enriched.setdefault("apiVersion", "v3.1.0")
+    enriched.setdefault("kind", "DataContract")
+    enriched.setdefault("version", "v1.0.0")
+    enriched.setdefault("status", "draft")
+
+    if dp_spec is not None:
+        if "dataProduct" not in enriched and dp_spec.get("name"):
+            enriched["dataProduct"] = dp_spec["name"]
+        if "domain" not in enriched and dp_spec.get("domain"):
+            enriched["domain"] = dp_spec["domain"]
+
+    return enriched

dmesh/sdk/core/id_generator.py

@@ -0,0 +1,44 @@
+"""Deterministic UUID5 generation for data products and data contracts."""
+import os
+import uuid
+
+# Fixed namespace for all dmesh IDs
+_NAMESPACE = uuid.UUID("6ba7b810-9dad-11d1-80b4-00c04fd430c8")  # NAMESPACE_DNS
+
+DEFAULT_DP_SCHEME = "DataProduct/{domain}/{name}/{version}"
+DEFAULT_DC_SCHEME = "DataContract/{domain}/{name}/{version}/{dc_index}"
+
+
+def _scheme(env_var: str, default: str) -> str:
+    return os.environ.get(env_var, default)
+
+
+def make_dp_id(domain: str, name: str, version: str) -> str:
+    """Generate a deterministic UUID5 for a data product."""
+    scheme = _scheme("DP_ID_SCHEME", DEFAULT_DP_SCHEME)
+    try:
+        key = scheme.format(domain=domain, name=name, version=version)
+    except KeyError as e:
+        # Fallback if scheme is malformed
+        key = f"DataProduct/{domain}/{name}/{version}"
+    return str(uuid.uuid5(_NAMESPACE, key))
+
+
+def make_dc_id(dp_domain: str, dp_name: str, dp_version: str, dc_index: int) -> str:
+    """Generate a deterministic UUID5 for a data contract.
+
+    dc_index is the count of existing data contracts for the parent DP
+    at creation time (0-based: first DC gets index 0).
+    """
+    scheme = _scheme("DC_ID_SCHEME", DEFAULT_DC_SCHEME)
+    try:
+        key = scheme.format(
+            domain=dp_domain,
+            name=dp_name,
+            version=dp_version,
+            dc_index=dc_index,
+        )
+    except KeyError as e:
+        # Fallback if scheme is malformed
+        key = f"DataContract/{dp_domain}/{dp_name}/{dp_version}/{dc_index}"
+    return str(uuid.uuid5(_NAMESPACE, key))

dmesh/sdk/core/validator.py

@@ -0,0 +1,102 @@
+import importlib.resources
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import jsonschema
+import requests
+
+SCHEMA_URLS = {
+    "DataProduct": "https://raw.githubusercontent.com/bitol-io/open-data-product-standard/refs/heads/main/schema/odps-json-schema-{api_version}.json",
+    "DataContract": "https://raw.githubusercontent.com/bitol-io/open-data-contract-standard/refs/heads/main/schema/odcs-json-schema-{api_version}.json",
+}
+
+SCHEMA_MAP = {
+    "DataProduct": "odps-{api_version}.json",
+    "DataContract": "odcs-{api_version}.json",
+}
+
+
+class SchemaFetchError(Exception):
+    """Raised when the Bitol JSON Schema cannot be fetched."""
+
+
+def validate_spec(spec: dict[str, Any]) -> None:
+    """Validate spec against the versioned Bitol JSON Schema.
+
+    Prioritizes local schemas in the package over external GitHub URLs.
+
+    Raises:
+        ValueError: if apiVersion or kind is missing from spec.
+        SchemaFetchError: if the schema cannot be fetched locally or remotely.
+        jsonschema.ValidationError: if the spec is invalid.
+    """
+    api_version = spec.get("apiVersion")
+    kind = spec.get("kind")
+    if not api_version:
+        raise ValueError("apiVersion is required for schema validation")
+    if not re.match(r"^v\d+\.\d+\.\d+$", api_version):
+        raise ValueError(f"invalid apiVersion input \"{api_version}\" expected format: vX.Y.Z")
+
+    if not kind:
+        # Heuristic to detect kind if not specified
+        if "domain" in spec or "payload" in spec:
+            kind = "DataProduct"
+        elif "specification" in spec or "info" in spec:
+            kind = "DataContract"
+        else:
+            # Default to DataProduct if ambiguous
+            kind = "DataProduct"
+
+    # Normalize version: strip 'v' prefix for local file lookup
+    clean_version = api_version[1:] if api_version.startswith("v") else api_version
+
+    # 1. Try local lookup
+    local_name_template = SCHEMA_MAP.get(kind, "odps-{api_version}.json")
+    local_filename = local_name_template.format(api_version=clean_version)
+    
+    try:
+        # Modern importlib.resources API (files() instead of is_resource/open_text)
+        pkg_path = importlib.resources.files("dmesh.sdk.schemas")
+        schema_file = pkg_path / local_filename
+        
+        if schema_file.is_file():
+            with schema_file.open("r", encoding="utf-8") as f:
+                schema = json.load(f)
+                # Successful local schema load
+                try:
+                    jsonschema.validate(spec, schema)
+                    return
+                except jsonschema.ValidationError:
+                    # If it's a validation error against a local schema, we STOP here.
+                    # This is the strict Source of Truth.
+                    raise
+    except jsonschema.ValidationError:
+        raise
+    except Exception:
+        # Only fallback if the file itself was missing or malformed
+        pass
+
+    # 2. Try remote lookup (fallback)
+    template = SCHEMA_URLS.get(kind, SCHEMA_URLS["DataProduct"])
+    versions_to_try = [api_version, clean_version]
+    if not api_version.startswith("v"):
+        versions_to_try.append(f"v{api_version}")
+
+    last_error = None
+    for v in versions_to_try:
+        url = template.format(api_version=v)
+        try:
+            response = requests.get(url, timeout=10)
+            if response.status_code == 200:
+                schema = response.json()
+                jsonschema.validate(spec, schema)
+                return
+            last_error = f"HTTP {response.status_code} at {url}"
+        except requests.RequestException as e:
+            last_error = str(e)
+
+    raise SchemaFetchError(
+        f"Schema not found for {kind} apiVersion={api_version} ({last_error})"
+    )

dmesh/sdk/models/__init__.py

@@ -0,0 +1,14 @@
+from .core import DataProduct, DataContract
+from .exceptions import (
+    OpenDataMeshError,
+    DataProductValidationError,
+    DataContractValidationError,
+)
+
+__all__ = [
+    "DataProduct",
+    "DataContract",
+    "OpenDataMeshError",
+    "DataProductValidationError",
+    "DataContractValidationError",
+]

dmesh/sdk/models/core.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class DataProduct:
+    id: str
+    specification: Dict[str, Any]
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+    @property
+    def domain(self) -> str:
+        return self.specification.get("domain", "")
+
+    @property
+    def name(self) -> str:
+        return self.specification.get("name", "")
+
+    @property
+    def version(self) -> str:
+        return self.specification.get("version", "v1.0.0")
+
+
+@dataclass
+class DataContract:
+    id: str
+    data_product_id: str
+    specification: Dict[str, Any]
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None

dmesh/sdk/models/exceptions.py

@@ -0,0 +1,13 @@
+class OpenDataMeshError(Exception):
+    """Base class for all SDK errors."""
+    pass
+
+
+class DataProductValidationError(OpenDataMeshError):
+    """Raised when a Data Product specification is invalid according to ODPS schema."""
+    pass
+
+
+class DataContractValidationError(OpenDataMeshError):
+    """Raised when a Data Contract specification is invalid according to ODCS schema."""
+    pass

dmesh/sdk/operations/data_contract.py

@@ -0,0 +1,63 @@
+from typing import Any, List, Optional, Union
+from dmesh.sdk.models import DataContract
+from dmesh.sdk.ports.repository import DataContractRepository, DataProductRepository
+from dmesh.sdk.sdk import AsyncSDK, _RepoWrapper
+
+async def create_dc(
+    repo: DataContractRepository, 
+    dp_repo: DataProductRepository,
+    spec: dict[str, Any], 
+    dp_id: str, 
+    include_metadata: Optional[bool] = False
+) -> Union[dict, DataContract]:
+    """Create a data contract for a given data product."""
+    return await AsyncSDK(_RepoWrapper(dp_repo=dp_repo, dc_repo=repo)).put_data_contract(
+        spec, dp_id=dp_id, include_metadata=include_metadata
+    )
+
+async def update_dc(
+    repo: DataContractRepository, 
+    spec: dict[str, Any], 
+    include_metadata: Optional[bool] = False
+) -> Union[dict, DataContract]:
+    """Update an existing data contract."""
+    if not spec.get("id"):
+        raise ValueError("Data contract id is required for update")
+    return await AsyncSDK(_RepoWrapper(dc_repo=repo)).put_data_contract(
+        spec, include_metadata=include_metadata
+    )
+
+async def patch_dc(
+    repo: DataContractRepository, 
+    spec: dict[str, Any], 
+    include_metadata: Optional[bool] = False
+) -> Union[dict, DataContract]:
+    """Patch an existing data contract."""
+    dc_id = spec.get("id")
+    if not dc_id:
+        raise ValueError("Data contract id is required for patch")
+    return await AsyncSDK(_RepoWrapper(dc_repo=repo)).patch_data_contract(
+        spec, id=dc_id, include_metadata=include_metadata
+    )
+
+async def get_dc(
+    repo: DataContractRepository, 
+    id: str, 
+    include_metadata: Optional[bool] = False
+) -> Optional[Union[dict, DataContract]]:
+    """Fetch a single data contract by ID."""
+    return await AsyncSDK(_RepoWrapper(dc_repo=repo)).get_data_contract(id, include_metadata=include_metadata)
+
+async def list_dcs(
+    repo: DataContractRepository, 
+    dp_id: Optional[str] = None, 
+    include_metadata: Optional[bool] = False
+) -> List[Union[dict, DataContract]]:
+    """List data contracts, optionally filtering by parent data product."""
+    return await AsyncSDK(_RepoWrapper(dc_repo=repo)).list_data_contracts(
+        dp_id=dp_id, include_metadata=include_metadata
+    )
+
+async def delete_dc(repo: DataContractRepository, id: str) -> bool:
+    """Delete a data contract by ID."""
+    return await AsyncSDK(_RepoWrapper(dc_repo=repo)).delete_data_contract(id)

dmesh/sdk/operations/data_product.py

@@ -0,0 +1,53 @@
+from typing import Any, List, Optional, Union
+from dmesh.sdk.models import DataProduct
+from dmesh.sdk.ports.repository import DataProductRepository
+from dmesh.sdk.sdk import AsyncSDK, _RepoWrapper
+
+async def create_dp(
+    repo: DataProductRepository, 
+    spec: dict[str, Any], 
+    domain: Optional[str] = None, 
+    name: Optional[str] = None, 
+    include_metadata: Optional[bool] = False
+) -> Union[dict, DataProduct]:
+    """Create a new data product."""
+    return await AsyncSDK(_RepoWrapper(dp_repo=repo)).put_data_product(
+        spec, domain=domain, name=name, include_metadata=include_metadata
+    )
+
+async def update_dp(
+    repo: DataProductRepository, 
+    spec: dict[str, Any], 
+    include_metadata: Optional[bool] = False
+) -> Union[dict, DataProduct]:
+    """Update an existing data product."""
+    if not spec.get("id"):
+        raise ValueError("Data product id is required for update")
+    return await AsyncSDK(_RepoWrapper(dp_repo=repo)).put_data_product(
+        spec, include_metadata=include_metadata
+    )
+
+async def get_dp(
+    repo: DataProductRepository, 
+    id: Optional[str] = None, 
+    include_metadata: bool = False
+) -> Optional[Union[dict, DataProduct]]:
+    """Fetch a single data product by ID."""
+    if not id:
+        return None
+    return await AsyncSDK(_RepoWrapper(dp_repo=repo)).get_data_product(id, include_metadata=include_metadata)
+
+async def list_dps(
+    repo: DataProductRepository, 
+    domain: Optional[str] = None, 
+    name: Optional[str] = None, 
+    include_metadata: Optional[bool] = False
+) -> List[Union[dict, DataProduct]]:
+    """List data products with optional filtering."""
+    return await AsyncSDK(_RepoWrapper(dp_repo=repo)).list_data_products(
+        domain=domain, name=name, include_metadata=include_metadata
+    )
+
+async def delete_dp(repo: DataProductRepository, id: str) -> bool:
+    """Delete a data product by ID."""
+    return await AsyncSDK(_RepoWrapper(dp_repo=repo)).delete_data_product(id)

dmesh/sdk/operations/discover.py

@@ -0,0 +1,20 @@
+from typing import Any, List, Optional, Union
+from dmesh.sdk.models import DataProduct, DataContract
+from dmesh.sdk.ports.repository import DataProductRepository, DataContractRepository
+from dmesh.sdk.sdk import AsyncSDK, _RepoWrapper
+
+async def discover(
+    dp_repo: DataProductRepository,
+    dc_repo: DataContractRepository,
+    dp_id: Optional[str] = None,
+    domain: Optional[str] = None,
+    name: Optional[str] = None,
+    include_metadata: Optional[bool] = False,
+    include_metadata_in_response: Optional[bool] = False
+) -> List[Union[dict, DataProduct, DataContract]]:
+    """Discovery by ID OR by domain and name. Returns a flat list of DataProduct and DataContract objects."""
+    # Note: include_metadata_in_response maps to include_metadata in SDK
+    incl = include_metadata or include_metadata_in_response
+    return await AsyncSDK(_RepoWrapper(dp_repo=dp_repo, dc_repo=dc_repo)).discover(
+        dp_id=dp_id, domain=domain, name=name, include_metadata=incl
+    )

dmesh/sdk/operations/utils.py

@@ -0,0 +1,6 @@
+from dmesh.sdk.ports.repository import DataProductRepository
+from dmesh.sdk.sdk import AsyncSDK, _RepoWrapper
+
+async def flush(repo: DataProductRepository) -> None:
+    """Flush the repository if supported."""
+    await AsyncSDK(_RepoWrapper(dp_repo=repo)).flush()