datalex-cli 0.1.1__py3-none-any.whl

datalex_core/datalex/project.py

@@ -0,0 +1,214 @@
+"""DataLexProject — the loaded, validated graph.
+
+Holds every kind in its own dict keyed by a stable ID (entity keys are
+`<layer>:<name>` because the same logical name can appear at each of the three
+layers). Provides convenience lookups and a `resolve()` pass that:
+  * Inlines snippet `use:` directives on columns.
+  * Validates `logical:` back-references from physical to logical entities.
+  * Flags dangling term/entity/source/model references.
+
+Kept as a thin orchestration layer over the dict-of-dict representation — dialect
+plugins and diff engine operate on dicts directly, so the Python object is a
+convenience, not a requirement.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+from datalex_core.datalex.errors import DataLexError, DataLexErrorBag, SourceLocation
+
+
+@dataclass
+class DataLexProject:
+    root: Path
+    manifest: Optional[Dict[str, Any]]
+    entities: Dict[str, Dict[str, Any]]
+    sources: Dict[str, Dict[str, Any]]
+    models: Dict[str, Dict[str, Any]]
+    terms: Dict[str, Dict[str, Any]]
+    domains: Dict[str, Dict[str, Any]]
+    policies: Dict[str, Dict[str, Any]]
+    snippets: Dict[str, Dict[str, Any]]
+    file_of: Dict[Tuple[str, str], str]
+    errors: DataLexErrorBag
+    # Phase C: imported packages. Each key is the package's alias; value is a
+    # loaded sub-project. Sub-projects are validated independently.
+    imports: Dict[str, "DataLexProject"] = field(default_factory=dict)
+
+    # ---------- lookups ----------
+
+    def entity(self, name: str, layer: str = "physical") -> Optional[Dict[str, Any]]:
+        return self.entities.get(f"{layer}:{name}")
+
+    def imported_entity(
+        self, alias: str, name: str, layer: str = "physical"
+    ) -> Optional[Dict[str, Any]]:
+        """Look up an entity inside an imported package by alias."""
+        sub = self.imports.get(alias)
+        if sub is None:
+            return None
+        return sub.entity(name, layer=layer)
+
+    def resolve_cross_package(
+        self, reference: str, layer: str = "physical"
+    ) -> Optional[Dict[str, Any]]:
+        """Resolve `@alias.entity_name` style references against imported packages.
+
+        Plain names without `@alias.` fall back to local entities so callers can
+        use a single lookup path.
+        """
+        if reference.startswith("@"):
+            try:
+                alias, name = reference[1:].split(".", 1)
+            except ValueError:
+                return None
+            return self.imported_entity(alias, name, layer=layer)
+        return self.entity(reference, layer=layer)
+
+    def iter_entities(self, layer: Optional[str] = None) -> Iterable[Dict[str, Any]]:
+        for key, ent in sorted(self.entities.items()):
+            if layer is None or key.startswith(f"{layer}:"):
+                yield ent
+
+    def physical_entities(self, dialect: Optional[str] = None) -> List[Dict[str, Any]]:
+        out = []
+        for ent in self.iter_entities(layer="physical"):
+            if dialect is None or ent.get("dialect") == dialect:
+                out.append(ent)
+        return out
+
+    # ---------- resolution ----------
+
+    def resolve(self) -> None:
+        """Run post-load resolution: snippet expansion, back-reference checks."""
+        self._expand_snippets()
+        self._check_logical_backrefs()
+        self._check_term_refs()
+        self._check_reference_targets()
+
+    def _expand_snippets(self) -> None:
+        """Inline `use: <snippet>` on columns with snippet.apply content.
+
+        Merge semantics: column keys win over snippet keys. Snippet fields fill in
+        missing keys only. This is conservative — users opt in explicitly.
+        """
+        for ent in self.entities.values():
+            for col in ent.get("columns", []) or []:
+                snippet_name = col.pop("use", None)
+                if not snippet_name:
+                    continue
+                snip = self.snippets.get(snippet_name)
+                if snip is None:
+                    self.errors.add(
+                        DataLexError(
+                            code="SNIPPET_NOT_FOUND",
+                            message=f"Column '{col.get('name')}' uses unknown snippet '{snippet_name}'",
+                            location=self._loc_for("entity", ent),
+                            suggested_fix=f"Create .datalex/snippets/{snippet_name}.yaml or remove the use: directive.",
+                        )
+                    )
+                    continue
+                apply = snip.get("apply", {}) or {}
+                for k, v in apply.items():
+                    if k not in col:
+                        col[k] = v
+
+    def _check_logical_backrefs(self) -> None:
+        for key, ent in self.entities.items():
+            if not key.startswith("physical:"):
+                continue
+            logical_name = ent.get("logical")
+            if not logical_name:
+                continue
+            if f"logical:{logical_name}" not in self.entities:
+                self.errors.add(
+                    DataLexError(
+                        code="LOGICAL_BACKREF",
+                        severity="warn",
+                        message=f"Physical entity '{ent.get('name')}' references logical '{logical_name}' which does not exist.",
+                        location=self._loc_for("entity", ent),
+                        suggested_fix=f"Create models/logical/{logical_name}.yaml or remove the logical: reference.",
+                    )
+                )
+
+    def _check_term_refs(self) -> None:
+        term_names = set(self.terms.keys())
+        for ent in self.entities.values():
+            for t in ent.get("terms", []) or []:
+                name = t.split(":", 1)[1] if t.startswith("term:") else t
+                if name not in term_names:
+                    self.errors.add(
+                        DataLexError(
+                            code="TERM_NOT_FOUND",
+                            severity="warn",
+                            message=f"Entity '{ent.get('name')}' references unknown term '{name}'",
+                            location=self._loc_for("entity", ent),
+                            suggested_fix=f"Create glossary/{name}.yaml or remove the term reference.",
+                        )
+                    )
+            for col in ent.get("columns", []) or []:
+                for t in col.get("terms", []) or []:
+                    name = t.split(":", 1)[1] if t.startswith("term:") else t
+                    if name not in term_names:
+                        self.errors.add(
+                            DataLexError(
+                                code="TERM_NOT_FOUND",
+                                severity="warn",
+                                message=f"Column '{ent.get('name')}.{col.get('name')}' references unknown term '{name}'",
+                                location=self._loc_for("entity", ent),
+                            )
+                        )
+
+    def _check_reference_targets(self) -> None:
+        for ent in self.entities.values():
+            for col in ent.get("columns", []) or []:
+                ref = col.get("references")
+                if not ref:
+                    continue
+                target_entity_name = ref.get("entity")
+                layer = ent.get("layer", "physical")
+                if not target_entity_name:
+                    continue
+                if f"{layer}:{target_entity_name}" not in self.entities:
+                    self.errors.add(
+                        DataLexError(
+                            code="REF_TARGET_MISSING",
+                            message=f"Column '{ent.get('name')}.{col.get('name')}' references missing entity '{target_entity_name}' at layer '{layer}'",
+                            location=self._loc_for("entity", ent),
+                            suggested_fix="Check the target entity name and layer.",
+                        )
+                    )
+
+    def _loc_for(self, kind: str, obj: Dict[str, Any]) -> SourceLocation:
+        name = obj.get("name", "")
+        layer = obj.get("layer", "physical") if kind == "entity" else ""
+        key = f"{layer}:{name}" if kind == "entity" else name
+        path = self.file_of.get((kind, key), str(self.root))
+        return SourceLocation(file=path)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a plain dict suitable for JSON serialization."""
+        return {
+            "root": str(self.root),
+            "manifest": self.manifest,
+            "entities": self.entities,
+            "sources": self.sources,
+            "models": self.models,
+            "terms": self.terms,
+            "domains": self.domains,
+            "policies": self.policies,
+            "snippets": self.snippets,
+            "imports": {
+                alias: {
+                    "root": str(sub.root),
+                    "entities": sorted(sub.entities.keys()),
+                    "terms": sorted(sub.terms.keys()),
+                }
+                for alias, sub in self.imports.items()
+            },
+            "errors": self.errors.to_list(),
+        }

datalex_core/datalex/types.py

@@ -0,0 +1,224 @@
+"""DataLex logical type system.
+
+Grammar:
+    type      := primitive | parameterized | composite
+    primitive := string | text | integer | bigint | float | boolean
+               | date | timestamp | timestamp_tz | interval
+               | uuid | json | binary | decimal
+    parameterized := primitive '(' INT [',' INT] ')'          e.g. decimal(18,4), string(255)
+    composite := 'array' '<' type '>'
+               | 'map'   '<' type ',' type '>'
+               | 'struct' '<' field (',' field)* '>'
+    field     := ident ':' type
+
+The parser is recursive-descent and deterministic; `str(parsed)` round-trips to a
+canonical form used by dialect plugins and the diff engine.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+
+from datalex_core.datalex.errors import DataLexError
+
+
+PRIMITIVES = frozenset({
+    "string", "text", "integer", "bigint", "float", "boolean",
+    "date", "timestamp", "timestamp_tz", "interval",
+    "uuid", "json", "binary", "decimal",
+})
+
+COMPOSITE_KEYWORDS = frozenset({"array", "map", "struct"})
+
+
+@dataclass(frozen=True)
+class LogicalType:
+    """In-memory representation of a parsed logical type.
+
+    `kind` is one of PRIMITIVES or COMPOSITE_KEYWORDS.
+    `params` is the tuple of numeric parameters (e.g. (18, 4) for decimal(18,4)).
+    `children` is the tuple of child types for array/map.
+    `fields` is the tuple of (name, type) pairs for struct.
+    """
+    kind: str
+    params: Tuple[int, ...] = ()
+    children: Tuple["LogicalType", ...] = ()
+    fields: Tuple[Tuple[str, "LogicalType"], ...] = ()
+
+    def render(self) -> str:
+        if self.kind == "array":
+            return f"array<{self.children[0].render()}>"
+        if self.kind == "map":
+            return f"map<{self.children[0].render()},{self.children[1].render()}>"
+        if self.kind == "struct":
+            inner = ",".join(f"{n}:{t.render()}" for n, t in self.fields)
+            return f"struct<{inner}>"
+        if self.params:
+            return f"{self.kind}({','.join(str(p) for p in self.params)})"
+        return self.kind
+
+    def is_composite(self) -> bool:
+        return self.kind in COMPOSITE_KEYWORDS
+
+    def __str__(self) -> str:
+        return self.render()
+
+
+class _Tokenizer:
+    """Tiny tokenizer for the logical type grammar."""
+
+    def __init__(self, text: str):
+        self.text = text
+        self.pos = 0
+        self.n = len(text)
+
+    def peek(self) -> str:
+        self._skip_ws()
+        return self.text[self.pos] if self.pos < self.n else ""
+
+    def consume(self, ch: str) -> bool:
+        self._skip_ws()
+        if self.pos < self.n and self.text[self.pos] == ch:
+            self.pos += 1
+            return True
+        return False
+
+    def expect(self, ch: str) -> None:
+        if not self.consume(ch):
+            raise DataLexError(
+                code="TYPE_PARSE",
+                message=f"Expected '{ch}' at position {self.pos} in type '{self.text}'",
+                suggested_fix=f"Check the type syntax near '{self.text[max(0, self.pos-8):self.pos+8]}'",
+            )
+
+    def read_ident(self) -> str:
+        self._skip_ws()
+        start = self.pos
+        while self.pos < self.n and (self.text[self.pos].isalnum() or self.text[self.pos] == "_"):
+            self.pos += 1
+        if start == self.pos:
+            raise DataLexError(
+                code="TYPE_PARSE",
+                message=f"Expected identifier at position {self.pos} in type '{self.text}'",
+            )
+        return self.text[start:self.pos]
+
+    def read_int(self) -> int:
+        self._skip_ws()
+        start = self.pos
+        while self.pos < self.n and self.text[self.pos].isdigit():
+            self.pos += 1
+        if start == self.pos:
+            raise DataLexError(code="TYPE_PARSE", message=f"Expected integer in type '{self.text}'")
+        return int(self.text[start:self.pos])
+
+    def eof(self) -> bool:
+        self._skip_ws()
+        return self.pos >= self.n
+
+    def _skip_ws(self) -> None:
+        while self.pos < self.n and self.text[self.pos] in " \t\n":
+            self.pos += 1
+
+
+def parse_type(text: str) -> LogicalType:
+    """Parse a DataLex logical type string into a LogicalType.
+
+    Raises DataLexError(code=TYPE_PARSE) on malformed input. Unknown primitive names
+    are accepted (returned as a raw kind with no params) so dialect plugins can accept
+    dialect-specific types as escape hatches; validation of known primitives happens
+    in the validator pass.
+    """
+    if not isinstance(text, str) or not text.strip():
+        raise DataLexError(code="TYPE_PARSE", message="Empty type string")
+    tok = _Tokenizer(text.strip())
+    parsed = _parse(tok)
+    if not tok.eof():
+        raise DataLexError(
+            code="TYPE_PARSE",
+            message=f"Trailing characters after type in '{text}'",
+        )
+    return parsed
+
+
+def _parse(tok: _Tokenizer) -> LogicalType:
+    ident = tok.read_ident().lower()
+
+    if ident == "array":
+        tok.expect("<")
+        inner = _parse(tok)
+        tok.expect(">")
+        return LogicalType(kind="array", children=(inner,))
+
+    if ident == "map":
+        tok.expect("<")
+        k = _parse(tok)
+        tok.expect(",")
+        v = _parse(tok)
+        tok.expect(">")
+        return LogicalType(kind="map", children=(k, v))
+
+    if ident == "struct":
+        tok.expect("<")
+        fields: List[Tuple[str, LogicalType]] = []
+        while True:
+            name = tok.read_ident()
+            tok.expect(":")
+            ftype = _parse(tok)
+            fields.append((name, ftype))
+            if tok.consume(","):
+                continue
+            break
+        tok.expect(">")
+        return LogicalType(kind="struct", fields=tuple(fields))
+
+    # primitive or parameterized
+    params: Tuple[int, ...] = ()
+    if tok.peek() == "(":
+        tok.expect("(")
+        params_list: List[int] = [tok.read_int()]
+        while tok.consume(","):
+            params_list.append(tok.read_int())
+        tok.expect(")")
+        params = tuple(params_list)
+
+    return LogicalType(kind=ident, params=params)
+
+
+def is_known_primitive(kind: str) -> bool:
+    return kind in PRIMITIVES
+
+
+def validate_type_string(text: str) -> Optional[DataLexError]:
+    """Return a DataLexError if the type string is malformed or uses unknown primitives
+    in a shape that is clearly wrong (e.g. composite keyword without generics)."""
+    try:
+        t = parse_type(text)
+    except DataLexError as e:
+        return e
+
+    return _validate_tree(t)
+
+
+def _validate_tree(t: LogicalType) -> Optional[DataLexError]:
+    if t.kind in COMPOSITE_KEYWORDS:
+        for c in t.children:
+            err = _validate_tree(c)
+            if err:
+                return err
+        for _, ft in t.fields:
+            err = _validate_tree(ft)
+            if err:
+                return err
+        return None
+
+    if t.kind not in PRIMITIVES:
+        # allow as pass-through so dialects can accept native types, but flag a warning
+        return DataLexError(
+            code="TYPE_UNKNOWN_PRIMITIVE",
+            severity="warn",
+            message=f"Unknown logical primitive '{t.kind}' — will be passed through to the dialect verbatim",
+            suggested_fix=f"Use one of: {', '.join(sorted(PRIMITIVES))} — or provide a per-dialect physical override.",
+        )
+    return None

datalex_core/dbt/__init__.py

@@ -0,0 +1,18 @@
+"""DataLex <-> dbt integration: emit dbt YAML, import manifest.json, sync live warehouse."""
+
+from datalex_core.dbt.emit import emit_dbt, build_sources_yaml, build_models_yaml, EmitReport
+from datalex_core.dbt.manifest import import_manifest, write_import_result, ImportResult
+from datalex_core.dbt.sync import sync_dbt_project, SyncReport, TableSyncRecord
+
+__all__ = [
+    "emit_dbt",
+    "build_sources_yaml",
+    "build_models_yaml",
+    "EmitReport",
+    "import_manifest",
+    "write_import_result",
+    "ImportResult",
+    "sync_dbt_project",
+    "SyncReport",
+    "TableSyncRecord",
+]