datablade 0.0.0__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/registry.py

@@ -0,0 +1,581 @@
+"""In-memory object registry for SQL-like dot notation."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field, replace
+from typing import Any, Callable, Iterable, Mapping, Optional, Union
+
+from .sql.dialects import Dialect
+from .sql.quoting import quote_identifier
+
+ObjectChild = Union["ObjectNode", "ObjectRef"]
+
+_ALLOWED_SEGMENTS = {"host", "catalog", "schema", "object"}
+_SEGMENT_ALIASES = {"collection": "object"}
+_NAME_POLICIES = {"preserve", "lower", "upper", "normalize"}
+
+
+@dataclass(frozen=True, slots=True)
+class DialectSpec:
+    """Defines how to qualify names for a dialect."""
+
+    qualifier: tuple[str, ...]
+    quote_style: Optional[str] = None
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectRef:
+    """Immutable object metadata reference."""
+
+    path: str
+    name: str
+    object_type: str
+    host: Optional[str] = None
+    catalog: Optional[str] = None
+    schema: Optional[str] = None
+    content: Any = None
+    aliases: tuple[str, ...] = ()
+    tags: Mapping[str, str] = field(default_factory=dict)
+    _dialects: Mapping[str, DialectSpec] = field(
+        default_factory=dict, repr=False, compare=False
+    )
+
+    def qualified(self, dialect: str = "sqlserver") -> str:
+        """Return a dialect-qualified name using available segments."""
+        spec = self._dialects.get(dialect)
+        if spec is None:
+            raise KeyError(f"Dialect not configured: {dialect}")
+        segments: list[str] = []
+        for segment in spec.qualifier:
+            value = _segment_value(self, segment)
+            if value:
+                segments.append(_quote_segment(value, spec.quote_style))
+        return ".".join(segments)
+
+    def with_content(self, content: Any) -> "ObjectRef":
+        """Return a new ObjectRef with updated content."""
+        return replace(self, content=content)
+
+
+class ObjectNode:
+    """Namespace container that supports dot and key access."""
+
+    __slots__ = ("name", "path", "_normalizer", "_children", "_raw_map", "_alias_map")
+
+    def __init__(self, name: str, path: str, normalizer: Callable[[str], str]) -> None:
+        self.name = name
+        self.path = path
+        self._normalizer = normalizer
+        self._children: dict[str, ObjectChild] = {}
+        self._raw_map: dict[str, str] = {}
+        self._alias_map: dict[str, str] = {}
+
+    def add_child(
+        self,
+        key: str,
+        child: ObjectChild,
+        *,
+        aliases: Iterable[str] = (),
+        strict: bool = True,
+    ) -> None:
+        normalized = self._normalizer(key)
+        if normalized in self._children:
+            raise ValueError(
+                f"Key collision at {self.path or '<root>'}: '{key}' conflicts after normalization."
+            )
+        self._children[normalized] = child
+        self._raw_map[key] = normalized
+        for alias in aliases:
+            normalized_alias = self._normalizer(alias)
+            if (
+                normalized_alias in self._children
+                or normalized_alias in self._alias_map
+            ):
+                if strict:
+                    raise ValueError(
+                        f"Alias collision at {self.path or '<root>'}: '{alias}' conflicts with a sibling."
+                    )
+                continue
+            self._alias_map[normalized_alias] = normalized
+
+    def _resolve(self, key: str) -> tuple[str, ObjectChild]:
+        if key in self._raw_map:
+            normalized = self._raw_map[key]
+            return normalized, self._children[normalized]
+        normalized = self._normalizer(key)
+        if normalized in self._alias_map:
+            canonical = self._alias_map[normalized]
+            return canonical, self._children[canonical]
+        if normalized in self._children:
+            return normalized, self._children[normalized]
+        raise KeyError(key)
+
+    def _replace_child(self, key: str, child: ObjectChild) -> None:
+        if key not in self._children:
+            raise KeyError(key)
+        self._children[key] = child
+
+    def __getattr__(self, key: str) -> ObjectChild:
+        if key.startswith("_"):
+            raise AttributeError(key)
+        try:
+            _, child = self._resolve(key)
+        except KeyError as exc:
+            raise AttributeError(key) from exc
+        return child
+
+    def __getitem__(self, key: str) -> ObjectChild:
+        _, child = self._resolve(key)
+        return child
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Safe lookup by key or alias."""
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def iter_objects(self) -> Iterable[ObjectRef]:
+        """Yield ObjectRef descendants."""
+        for child in self._children.values():
+            if isinstance(child, ObjectRef):
+                yield child
+            elif isinstance(child, ObjectNode):
+                yield from child.iter_objects()
+
+
+class ObjectRegistry:
+    """Root registry for object metadata and namespace lookup."""
+
+    __slots__ = ("root", "_dialects", "_name_policy")
+
+    def __init__(
+        self,
+        root: ObjectNode,
+        *,
+        dialects: Mapping[str, DialectSpec],
+        name_policy: str,
+    ) -> None:
+        self.root = root
+        self._dialects = dialects
+        self._name_policy = name_policy
+
+    def __getattr__(self, key: str) -> ObjectChild:
+        return getattr(self.root, key)
+
+    def __getitem__(self, key: str) -> ObjectChild:
+        return self.root[key]
+
+    @classmethod
+    def from_mapping(
+        cls,
+        mapping: Mapping[str, Any],
+        *,
+        dialects: Optional[Mapping[str, Any]] = None,
+        strict: bool = True,
+    ) -> "ObjectRegistry":
+        if not isinstance(mapping, Mapping):
+            raise ValueError("mapping must be a dict-like object")
+
+        version = mapping.get("version")
+        if not isinstance(version, int):
+            raise ValueError("version must be an integer")
+
+        defaults = mapping.get("defaults", {})
+        if defaults and not isinstance(defaults, Mapping):
+            raise ValueError("defaults must be a mapping when provided")
+
+        name_policy = defaults.get("name_policy", "preserve")
+        if name_policy not in _NAME_POLICIES:
+            raise ValueError(f"Unsupported name_policy: {name_policy}")
+
+        normalizer = _make_normalizer(name_policy)
+
+        config_dialects = mapping.get("dialects", {})
+        if config_dialects and not isinstance(config_dialects, Mapping):
+            raise ValueError("dialects must be a mapping when provided")
+
+        dialect_map = _build_dialects(config_dialects, override=dialects, strict=strict)
+
+        catalogs = mapping.get("catalogs")
+        hosts = mapping.get("hosts")
+        if not catalogs and not hosts:
+            raise ValueError("Either catalogs or hosts must be provided")
+
+        root = ObjectNode(name="root", path="", normalizer=normalizer)
+        if catalogs:
+            catalogs_node = ObjectNode(
+                name="catalogs", path="catalogs", normalizer=normalizer
+            )
+            root.add_child("catalogs", catalogs_node, strict=strict)
+            _build_catalogs(
+                catalogs_node,
+                catalogs,
+                defaults=defaults,
+                dialects=dialect_map,
+                normalizer=normalizer,
+                strict=strict,
+                host_context=defaults.get("host"),
+            )
+
+        if hosts:
+            hosts_node = ObjectNode(name="hosts", path="hosts", normalizer=normalizer)
+            root.add_child("hosts", hosts_node, strict=strict)
+            _build_hosts(
+                hosts_node,
+                hosts,
+                defaults=defaults,
+                dialects=dialect_map,
+                normalizer=normalizer,
+                strict=strict,
+            )
+
+        return cls(root, dialects=dialect_map, name_policy=name_policy)
+
+    @classmethod
+    def from_yaml(cls, path: str, **kwargs: Any) -> "ObjectRegistry":
+        try:
+            import yaml
+        except ImportError as exc:
+            raise ImportError(
+                "PyYAML is required for ObjectRegistry.from_yaml; install pyyaml."
+            ) from exc
+
+        with open(path, "r", encoding="utf-8") as handle:
+            data = yaml.safe_load(handle)
+        return cls.from_mapping(data, **kwargs)
+
+    def get(self, path: str) -> ObjectChild:
+        if not path or not isinstance(path, str):
+            raise ValueError("path must be a non-empty string")
+        current: ObjectChild = self.root
+        for part in path.split("."):
+            if not isinstance(current, ObjectNode):
+                raise KeyError(path)
+            _, current = current._resolve(part)
+        return current
+
+    def iter_objects(
+        self,
+        *,
+        object_type: Optional[str] = None,
+        tags: Optional[Mapping[str, str]] = None,
+    ) -> Iterable[ObjectRef]:
+        for obj in self.root.iter_objects():
+            if object_type and obj.object_type != object_type:
+                continue
+            if tags and any(obj.tags.get(k) != v for k, v in tags.items()):
+                continue
+            yield obj
+
+    def bind_content(self, path: str, content: Any) -> None:
+        parent, key, child = self._resolve_parent(path)
+        if not isinstance(child, ObjectRef):
+            raise ValueError(f"Path does not resolve to an object: {path}")
+        parent._replace_child(key, child.with_content(content))
+
+    def qualify(self, path: str, dialect: str = "sqlserver") -> str:
+        obj = self.get(path)
+        if not isinstance(obj, ObjectRef):
+            raise ValueError(f"Path does not resolve to an object: {path}")
+        return obj.qualified(dialect=dialect)
+
+    def _resolve_parent(self, path: str) -> tuple[ObjectNode, str, ObjectChild]:
+        if not path or not isinstance(path, str):
+            raise ValueError("path must be a non-empty string")
+        parts = path.split(".")
+        parent: ObjectNode = self.root
+        for part in parts[:-1]:
+            _, child = parent._resolve(part)
+            if not isinstance(child, ObjectNode):
+                raise KeyError(path)
+            parent = child
+        key, child = parent._resolve(parts[-1])
+        return parent, key, child
+
+
+def _build_dialects(
+    dialects: Mapping[str, Any],
+    *,
+    override: Optional[Mapping[str, Any]] = None,
+    strict: bool = True,
+) -> dict[str, DialectSpec]:
+    result = dict(_default_dialects())
+    for source in (dialects, override or {}):
+        for name, spec in source.items():
+            if not isinstance(spec, Mapping):
+                raise ValueError(f"dialects.{name} must be a mapping")
+            qualifier = spec.get("qualifier")
+            if not qualifier or not isinstance(qualifier, str):
+                raise ValueError(f"dialects.{name}.qualifier must be a string")
+            segments = []
+            for segment in qualifier.split("."):
+                segment = segment.strip()
+                if not segment:
+                    continue
+                segment = _SEGMENT_ALIASES.get(segment, segment)
+                if segment not in _ALLOWED_SEGMENTS:
+                    if strict:
+                        raise ValueError(
+                            f"dialects.{name}.qualifier uses unknown segment '{segment}'"
+                        )
+                    continue
+                segments.append(segment)
+            if not segments:
+                raise ValueError(f"dialects.{name}.qualifier has no valid segments")
+            quote_style = spec.get("quote_style")
+            if quote_style is not None and not isinstance(quote_style, str):
+                raise ValueError(f"dialects.{name}.quote_style must be a string")
+            result[name] = DialectSpec(tuple(segments), quote_style)
+    return result
+
+
+def _default_dialects() -> dict[str, DialectSpec]:
+    return {
+        "sqlserver": DialectSpec(("catalog", "schema", "object")),
+        "postgres": DialectSpec(("schema", "object")),
+        "mysql": DialectSpec(("schema", "object")),
+        "duckdb": DialectSpec(("schema", "object")),
+        "nosql": DialectSpec(("object",)),
+    }
+
+
+def _make_normalizer(policy: str) -> Callable[[str], str]:
+    if policy == "preserve":
+        return lambda value: value
+    if policy == "lower":
+        return lambda value: value.lower()
+    if policy == "upper":
+        return lambda value: value.upper()
+    if policy == "normalize":
+        return _normalize_identifier
+    raise ValueError(f"Unsupported name_policy: {policy}")
+
+
+def _normalize_identifier(value: str) -> str:
+    cleaned = re.sub(r"[^a-zA-Z0-9_]", "_", value.lower())
+    cleaned = re.sub(r"_+", "_", cleaned)
+    if not cleaned:
+        cleaned = "_"
+    if cleaned[0].isdigit():
+        cleaned = "_" + cleaned
+    return cleaned
+
+
+def _build_hosts(
+    hosts_node: ObjectNode,
+    hosts: Mapping[str, Any],
+    *,
+    defaults: Mapping[str, Any],
+    dialects: Mapping[str, DialectSpec],
+    normalizer: Callable[[str], str],
+    strict: bool,
+) -> None:
+    if not isinstance(hosts, Mapping):
+        raise ValueError("hosts must be a mapping")
+    for host_key, host_entry in hosts.items():
+        if not isinstance(host_key, str) or not host_key.strip():
+            raise ValueError("hosts keys must be non-empty strings")
+        if not isinstance(host_entry, Mapping):
+            raise ValueError(f"hosts.{host_key} must be a mapping")
+        host_value = host_entry.get("host") or defaults.get("host")
+        catalogs = host_entry.get("catalogs")
+        if not catalogs:
+            raise ValueError(f"hosts.{host_key}.catalogs must be provided")
+
+        host_path = _join_path(hosts_node.path, host_key, normalizer)
+        host_node = ObjectNode(name=host_key, path=host_path, normalizer=normalizer)
+        hosts_node.add_child(host_key, host_node, strict=strict)
+        _build_catalogs(
+            host_node,
+            catalogs,
+            defaults=defaults,
+            dialects=dialects,
+            normalizer=normalizer,
+            strict=strict,
+            host_context=host_value,
+        )
+
+
+def _build_catalogs(
+    parent_node: ObjectNode,
+    catalogs: Mapping[str, Any],
+    *,
+    defaults: Mapping[str, Any],
+    dialects: Mapping[str, DialectSpec],
+    normalizer: Callable[[str], str],
+    strict: bool,
+    host_context: Optional[str],
+) -> None:
+    if not isinstance(catalogs, Mapping):
+        raise ValueError(f"{parent_node.path}.catalogs must be a mapping")
+    for catalog_key, catalog_entry in catalogs.items():
+        if not isinstance(catalog_key, str) or not catalog_key.strip():
+            raise ValueError("catalog keys must be non-empty strings")
+        if not isinstance(catalog_entry, Mapping):
+            raise ValueError(f"{parent_node.path}.{catalog_key} must be a mapping")
+        catalog_name = catalog_entry.get("catalog") or catalog_key
+        catalog_path = _join_path(parent_node.path, catalog_key, normalizer)
+        catalog_node = ObjectNode(
+            name=catalog_key, path=catalog_path, normalizer=normalizer
+        )
+        parent_node.add_child(catalog_key, catalog_node, strict=strict)
+
+        _build_objects(
+            catalog_node,
+            catalog_entry.get("objects", {}),
+            defaults=defaults,
+            dialects=dialects,
+            normalizer=normalizer,
+            strict=strict,
+            host=host_context,
+            catalog=catalog_name,
+            schema=defaults.get("schema", "dbo"),
+        )
+
+        schemas = catalog_entry.get("schemas", {})
+        _build_schemas(
+            catalog_node,
+            schemas,
+            defaults=defaults,
+            dialects=dialects,
+            normalizer=normalizer,
+            strict=strict,
+            host=host_context,
+            catalog=catalog_name,
+        )
+
+
+def _build_schemas(
+    catalog_node: ObjectNode,
+    schemas: Mapping[str, Any],
+    *,
+    defaults: Mapping[str, Any],
+    dialects: Mapping[str, DialectSpec],
+    normalizer: Callable[[str], str],
+    strict: bool,
+    host: Optional[str],
+    catalog: Optional[str],
+) -> None:
+    if not isinstance(schemas, Mapping):
+        raise ValueError(f"{catalog_node.path}.schemas must be a mapping")
+    for schema_key, schema_entry in schemas.items():
+        if not isinstance(schema_key, str) or not schema_key.strip():
+            raise ValueError("schema keys must be non-empty strings")
+        if not isinstance(schema_entry, Mapping):
+            raise ValueError(f"{catalog_node.path}.{schema_key} must be a mapping")
+        schema_name = schema_entry.get("schema") or schema_key
+        schema_path = _join_path(catalog_node.path, schema_key, normalizer)
+        schema_node = ObjectNode(
+            name=schema_key, path=schema_path, normalizer=normalizer
+        )
+        catalog_node.add_child(schema_key, schema_node, strict=strict)
+
+        _build_objects(
+            schema_node,
+            schema_entry.get("objects", {}),
+            defaults=defaults,
+            dialects=dialects,
+            normalizer=normalizer,
+            strict=strict,
+            host=host,
+            catalog=catalog,
+            schema=schema_name,
+        )
+
+
+def _build_objects(
+    parent_node: ObjectNode,
+    objects: Mapping[str, Any],
+    *,
+    defaults: Mapping[str, Any],
+    dialects: Mapping[str, DialectSpec],
+    normalizer: Callable[[str], str],
+    strict: bool,
+    host: Optional[str],
+    catalog: Optional[str],
+    schema: Optional[str],
+) -> None:
+    if not isinstance(objects, Mapping):
+        raise ValueError(f"{parent_node.path}.objects must be a mapping")
+    for obj_key, obj_entry in objects.items():
+        if not isinstance(obj_key, str) or not obj_key.strip():
+            raise ValueError("object keys must be non-empty strings")
+        if not isinstance(obj_entry, Mapping):
+            raise ValueError(f"{parent_node.path}.{obj_key} must be a mapping")
+        name = obj_entry.get("name") or obj_key
+        object_type = obj_entry.get("object_type") or defaults.get(
+            "object_type", "table"
+        )
+        if not isinstance(object_type, str) or not object_type.strip():
+            raise ValueError(
+                f"{parent_node.path}.{obj_key}.object_type must be a string"
+            )
+        aliases = obj_entry.get("aliases", [])
+        if aliases is None:
+            aliases = []
+        if not isinstance(aliases, list):
+            raise ValueError(f"{parent_node.path}.{obj_key}.aliases must be a list")
+        content = obj_entry.get("content")
+        tags = obj_entry.get("tags", {})
+        if tags is None:
+            tags = {}
+        if not isinstance(tags, Mapping):
+            raise ValueError(f"{parent_node.path}.{obj_key}.tags must be a mapping")
+
+        obj_path = _join_path(parent_node.path, obj_key, normalizer)
+        obj_ref = ObjectRef(
+            path=obj_path,
+            name=name,
+            object_type=object_type,
+            host=host,
+            catalog=catalog,
+            schema=schema,
+            content=content,
+            aliases=tuple(aliases),
+            tags=dict(tags),
+            _dialects=dialects,
+        )
+        parent_node.add_child(obj_key, obj_ref, aliases=aliases, strict=strict)
+
+
+def _join_path(parent_path: str, key: str, normalizer: Callable[[str], str]) -> str:
+    normalized = normalizer(key)
+    return f"{parent_path}.{normalized}" if parent_path else normalized
+
+
+def _segment_value(obj: ObjectRef, segment: str) -> Optional[str]:
+    if segment == "host":
+        return obj.host
+    if segment == "catalog":
+        return obj.catalog
+    if segment == "schema":
+        return obj.schema
+    if segment == "object":
+        return obj.name
+    return None
+
+
+def _quote_segment(value: str, quote_style: Optional[str]) -> str:
+    if not quote_style or quote_style == "none":
+        return value
+    style = quote_style.lower()
+    mapping = {
+        "sqlserver": Dialect.SQLSERVER,
+        "postgres": Dialect.POSTGRES,
+        "mysql": Dialect.MYSQL,
+        "duckdb": Dialect.DUCKDB,
+    }
+    dialect = mapping.get(style)
+    if dialect is None:
+        return value
+    return quote_identifier(value, dialect)
+
+
+__all__ = [
+    "DialectSpec",
+    "ObjectRef",
+    "ObjectNode",
+    "ObjectRegistry",
+]

datablade/sql/__init__.py

@@ -0,0 +1,56 @@
+"""
+SQL utilities for datablade.
+
+Provides dialect-aware quoting, DDL generation, and bulk loading.
+Supports SQL Server, PostgreSQL, MySQL, and DuckDB.
+"""
+
+from .bulk_load import (
+    bulk_load,
+    bulk_load_duckdb,
+    bulk_load_mysql,
+    bulk_load_postgres,
+    bulk_load_sqlserver,
+    bulk_load_sqlserver_command,
+    bulk_load_sqlserver_commands,
+    bulk_load_sqlserver_many,
+    write_dataframe_and_load,
+)
+from .ddl import generate_create_table
+from .ddl_pyarrow import (
+    DroppedColumn,
+    FallbackColumn,
+    ParquetDDLMetadata,
+    generate_create_table_from_parquet,
+)
+from .dialects import Dialect
+from .quoting import quote_identifier
+from .sqlserver import (
+    sqlserver_bulk_insert_statements,
+    sqlserver_create_and_insert_from_parquet,
+    sqlserver_create_and_stage_from_parquets,
+    sqlserver_openrowset_parquet,
+)
+
+__all__ = [
+    "Dialect",
+    "quote_identifier",
+    "generate_create_table",
+    "generate_create_table_from_parquet",
+    "DroppedColumn",
+    "FallbackColumn",
+    "ParquetDDLMetadata",
+    "bulk_load",
+    "bulk_load_sqlserver",
+    "bulk_load_sqlserver_command",
+    "bulk_load_sqlserver_commands",
+    "bulk_load_sqlserver_many",
+    "bulk_load_postgres",
+    "bulk_load_mysql",
+    "bulk_load_duckdb",
+    "write_dataframe_and_load",
+    "sqlserver_openrowset_parquet",
+    "sqlserver_bulk_insert_statements",
+    "sqlserver_create_and_insert_from_parquet",
+    "sqlserver_create_and_stage_from_parquets",
+]