metricflow-sql 0.1.1__py3-none-any.whl

metricflow_sql/objects/base.py

@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+from urllib.parse import quote, unquote
+
+from sqlglot import exp
+from sqlglot.dialects.dialect import NormalizationStrategy
+
+from metricflow_sql.utils.normalize import read_identifier, read_table
+from metricflow_sql.utils.ordered_set import OrderedSet
+from metricflow_sql.utils.sql import column_paths
+
+DEFAULT_NAMESPACE = "public"
+
+
+class Prefix(StrEnum):
+    # Datasets, entities and constellations share one, as they share names.
+    TABLE = "t"
+    FIELD = "f"
+    SOURCE = "s"
+    ASSIGNMENT = "a"
+    PAGE = "p"
+
+    @property
+    def qualified(self) -> bool:
+        """Whether names are qualified by their parent, e.g. `f:<dataset>:<field>`."""
+        return self in _QUALIFIED_PREFIXES
+
+    @classmethod
+    def parse(cls, text: str) -> Prefix:
+        """The prefix `text` names, case-insensitively: a prefix (`f`) or its alias (`metric`)."""
+        text = text.lower()
+        prefix = _PREFIX_ALIASES.get(text)
+        return prefix if prefix is not None else cls(text)
+
+
+# How many parts an id of each prefix has, tables and fields.
+_PART_COUNTS = {
+    Prefix.TABLE: 1,
+    Prefix.FIELD: 2,
+    Prefix.SOURCE: 2,
+    Prefix.ASSIGNMENT: 3,
+    Prefix.PAGE: 1,
+}
+
+_QUALIFIED_PREFIXES = frozenset({Prefix.FIELD, Prefix.SOURCE, Prefix.ASSIGNMENT})
+
+# A table after the first is in the first's namespace, as the objects resolve it.
+_TABLE_PARTS = {
+    Prefix.TABLE: (0,),
+    Prefix.FIELD: (0,),
+    Prefix.SOURCE: (0, 1),
+    Prefix.ASSIGNMENT: (0, 2),
+    Prefix.PAGE: (),
+}
+
+# Human-friendly names for prefixes, e.g. `metric:sales:revenue` for `f:sales:revenue`.
+# Kinds sharing a prefix (e.g. `metric` and `role`) are interchangeable; they document intent.
+_PREFIX_ALIASES: dict[str, Prefix] = {
+    "dataset": Prefix.TABLE,
+    "entity": Prefix.TABLE,
+    "constellation": Prefix.TABLE,
+    "field": Prefix.FIELD,
+    "column": Prefix.FIELD,
+    "calculation": Prefix.FIELD,
+    "relationship": Prefix.FIELD,
+    "attribute": Prefix.FIELD,
+    "metric": Prefix.FIELD,
+    "role": Prefix.FIELD,
+    "source": Prefix.SOURCE,
+    "assignment": Prefix.ASSIGNMENT,
+    "page": Prefix.PAGE,
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Id:
+    """
+    Object identifier.
+
+    Should be unique within an environment.
+    """
+
+    prefix: Prefix
+    parts: tuple[exp.Table | exp.Identifier, ...]
+
+    def __post_init__(self) -> None:
+        for i in _TABLE_PARTS[self.prefix]:
+            match self.parts[i] if i < len(self.parts) else None:
+                case exp.Table(db="") | exp.Identifier():
+                    raise ValueError(f"{self!r} names a table without its namespace")
+
+    def names(self) -> tuple[str, ...]:
+        names: list[str] = []
+        for part in self.parts:
+            match part:
+                case exp.Table():
+                    names.extend((part.db, part.name))
+                case exp.Identifier():
+                    names.append(part.name)
+        return tuple(names)
+
+    def text(self) -> str:
+        """
+        `<prefix>:<name>:...`, where a table is its namespace and name, so a field is
+        `f:<namespace>:<table>:<field>`. Names are the registry's, normalized, so they're written
+        percent-encoded as a URL component, e.g. `f:SALES:Net%20Orders:AMOUNT` on Snowflake.
+        """
+        return ":".join((self.prefix.value, *map(_escape, self.names())))
+
+    @classmethod
+    def parse(cls, text: str) -> Id:
+        """
+        Parse an id as `text` writes it, e.g. `f:SALES:Net%20Orders:AMOUNT`, where every table
+        names its namespace. Names are read exactly: they're already the registry's. Escapes are
+        decoded leniently, so only `:` and `%` need them.
+        """
+        prefix, texts = _split(text)
+        if len(texts) != _PART_COUNTS[prefix] + len(_TABLE_PARTS[prefix]):
+            raise ValueError(f"Invalid id '{text}': wrong number of names for '{prefix.value}'")
+        return cls(prefix, _read_exact(prefix, texts))
+
+    @classmethod
+    def resolve(cls, text: str, dialect: str, strategy: NormalizationStrategy) -> Id:
+        """
+        Resolve a reference written by hand, e.g. `metric:sales:revenue`, whose prefix may be an
+        alias. Names are read as the registry reads a query's, and a table is in the namespace it
+        gives, e.g. `finance.sales`, else the first table's, else the default one, so the example
+        is `f:public:sales:revenue`. An id as
+        `text` writes it, which names every namespace, is read as `parse` does.
+        """
+        prefix, texts = _split(text)
+        tables = _TABLE_PARTS[prefix]
+        if len(texts) == _PART_COUNTS[prefix] + len(tables):
+            return cls(prefix, _read_exact(prefix, texts))
+        if len(texts) != _PART_COUNTS[prefix]:
+            raise ValueError(f"Invalid id '{text}': wrong number of names for '{prefix.value}'")
+        namespace = read_identifier(DEFAULT_NAMESPACE, dialect, strategy)
+        parts: list[exp.Table | exp.Identifier] = []
+        for i, part in enumerate(texts):
+            if i in tables:
+                table = in_namespace(read_table(part, dialect, strategy), namespace)
+                if not parts:
+                    namespace = table.args["db"]
+                parts.append(table)
+            else:
+                parts.append(read_identifier(part, dialect, strategy))
+        return cls(prefix, tuple(parts))
+
+
+def _split(text: str) -> tuple[Prefix, list[str]]:
+    """The prefix of `text`, which may be an alias, and its names, unescaped."""
+    prefix_text, separator, name = text.partition(":")
+    try:
+        prefix = Prefix.parse(prefix_text)
+    except ValueError:
+        raise ValueError(f"Invalid id '{text}': unknown prefix '{prefix_text}'") from None
+    if prefix is Prefix.PAGE:
+        if not name:
+            raise ValueError(f"Invalid id '{text}': expected '{prefix_text}:<name>'")
+        return prefix, [_unescape(name)]
+    texts = [_unescape(part) for part in name.split(":")]
+    if prefix.qualified:
+        if len(texts) < 2 or not all(texts):
+            raise ValueError(f"Invalid id '{text}': expected '{prefix_text}:<parent>:<name>'")
+    elif not (separator and name):
+        raise ValueError(f"Invalid id '{text}': expected '{prefix_text}:<name>'")
+    return prefix, texts
+
+
+def _escape(name: str) -> str:
+    return quote(name, safe="")
+
+
+def _unescape(text: str) -> str:
+    return unquote(text, errors="strict")
+
+
+def _read_exact(prefix: Prefix, texts: list[str]) -> tuple[exp.Table | exp.Identifier, ...]:
+    """The parts of an id as `text` writes it: a namespace and name for each table."""
+    parts: list[exp.Table | exp.Identifier] = []
+    remaining = iter(texts)
+    for i in range(_PART_COUNTS[prefix]):
+        if i in _TABLE_PARTS[prefix]:
+            namespace, table = next(remaining), next(remaining)
+            parts.append(
+                exp.table_(
+                    exp.to_identifier(table, quoted=True),
+                    db=exp.to_identifier(namespace, quoted=True),
+                )
+            )
+        else:
+            parts.append(exp.to_identifier(next(remaining), quoted=True))
+    return tuple(parts)
+
+
+def in_namespace(name: exp.Table, namespace: exp.Identifier) -> exp.Table:
+    """A reference without a namespace is in the referring object's, as SQL resolves a table."""
+    return name if name.db else exp.table_(name.this, db=namespace)
+
+
+@dataclass(frozen=True, slots=True)
+class Path:
+    base: Id
+    path: tuple[exp.Identifier, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class Link:
+    """An outgoing reference from one object to `target`, e.g. a markdown link in a wiki page."""
+
+    target: Id
+    label: str
+
+
+DEPENDENCY_LINK = "dependency"
+PARENT_LINK = "parent"
+TYPE_LINK = "type"
+
+
+@dataclass(frozen=True, slots=True)
+class Backlink:
+    """An incoming reference to an object, from `source` - the reverse of a `Link`."""
+
+    source: Id
+    label: str
+
+
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    """An independently indexable piece of an object's text, for search."""
+
+    text: str
+    # Heading trail locating the chunk within its object, e.g. ("Setup", "Install").
+    section: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, kw_only=True)
+class BaseObject:
+    """
+    Abstract base class for all objects.
+    """
+
+    meta: dict[str, Any] = field(default_factory=dict)
+    label: str | None = None
+    description: str | None = None
+
+    @property
+    def id(self) -> Id:
+        raise NotImplementedError
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        """
+        References to other objects that this object depends on for compilation
+        """
+        return OrderedSet()
+
+    def links(self) -> OrderedSet[Link]:
+        links: OrderedSet[Link] = OrderedSet()
+        for dependency in self.dependencies():
+            match dependency:
+                case Id():
+                    links.add(Link(target=dependency, label=DEPENDENCY_LINK))
+                case Path():
+                    # TODO: link paths too, e.g. to the relationship a path starts from.
+                    pass
+        return links
+
+    def chunks(self) -> tuple[Chunk, ...]:
+        """Indexable text for search"""
+        parts = (" ".join(self.id.names()), self.label, self.description)
+        return (Chunk(text="\n".join(part for part in parts if part)),)
+
+
+def dataset_paths(
+    parts: tuple[exp.Identifier, ...], namespace: exp.Identifier
+) -> tuple[tuple[exp.Table, tuple[exp.Identifier, ...]], ...]:
+    """Which reading of `a.b.c` applies depends on what's in scope, as in SQL: a dataset `a` in
+    `namespace` wins over `a.b`. Dependencies can't tell, so they depend on both."""
+    own = (exp.table_(parts[0], db=namespace), parts[1:])
+    if len(parts) < 3:
+        return (own,)
+    return own, (exp.table_(parts[1], db=parts[0]), parts[2:])
+
+
+def resolve_dataset_path(
+    parts: tuple[exp.Identifier, ...],
+    namespace: exp.Identifier,
+    exists: Callable[[exp.Table], bool],
+) -> tuple[exp.Table, tuple[exp.Identifier, ...]]:
+    readings = dataset_paths(parts, namespace)
+    return next((reading for reading in readings if exists(reading[0])), readings[0])
+
+
+def deps_from_dataset_expr(dataset: exp.Table, expr: exp.Expr) -> OrderedSet[Id | Path]:
+    """
+    Get dependencies from a dataset SQL expression, using `dataset` as the base.
+    """
+    deps: OrderedSet[Id | Path] = OrderedSet()
+    for col in expr.find_all(exp.Column):
+        if not col.table:
+            deps.add(Id(Prefix.FIELD, (dataset, col.this.copy())))
+    for parts in column_paths(expr):
+        deps.add(Path(base=Id(Prefix.TABLE, (dataset,)), path=parts))
+    return deps

metricflow_sql/objects/constellation.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass
+from dataclasses import field as dcfield
+
+from sqlglot import expressions as exp
+
+from metricflow_sql.objects.base import (
+    PARENT_LINK,
+    TYPE_LINK,
+    BaseObject,
+    Id,
+    Link,
+    Path,
+    Prefix,
+    dataset_paths,
+    deps_from_dataset_expr,
+    in_namespace,
+)
+from metricflow_sql.utils.ordered_set import OrderedSet
+from metricflow_sql.utils.sql import column_paths, ref_target
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class IncludeAll:
+    """Include all fields in the import"""
+
+
+@dataclass(frozen=True)
+class Include:
+    """Include `field` in the import"""
+
+    field: exp.Identifier
+
+
+@dataclass(frozen=True)
+class Exclude:
+    """Exclude `field` from the import"""
+
+    field: exp.Identifier
+
+
+ImportRule = IncludeAll | Include | Exclude
+
+
+def import_rule_from_str(rule: str, name: Callable[[str], exp.Identifier]) -> ImportRule:
+    if rule == "*":
+        return IncludeAll()
+    if rule.startswith("!"):
+        return Exclude(field=name(rule[1:]))
+    return Include(field=name(rule))
+
+
+@dataclass(frozen=True)
+class Import:
+    """
+    Specification for how one constellation imports another
+    """
+
+    constellation: exp.Table
+    fields: tuple[ImportRule, ...]
+
+    def includes(self, field: exp.Identifier) -> bool:
+        """Returns whether `field` is imported. Only exclusions, e.g. `!refunds`, imply `*`."""
+        if Exclude(field=field) in self.fields:
+            return False
+        return (
+            IncludeAll() in self.fields
+            or Include(field=field) in self.fields
+            or (bool(self.fields) and all(isinstance(rule, Exclude) for rule in self.fields))
+        )
+
+
+@dataclass(frozen=True)
+class Constellation(BaseObject):
+    """
+    A Constellation is a group of fact tables connected by conformed dimensions.
+    Constellations capture higher-order business domains and enable multi-fact queries.
+
+    Constellations can include configuration from other constellations.
+    """
+
+    name: exp.Table
+    imports: list[Import] = dcfield(default_factory=list)
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.TABLE, (self.name,))
+
+
+@dataclass(frozen=True)
+class _ConstellationField(BaseObject):
+    constellation: exp.Table
+    name: exp.Identifier
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.FIELD, (self.constellation, self.name))
+
+    @property
+    def namespace(self) -> exp.Identifier:
+        namespace: exp.Identifier = self.constellation.args["db"]
+        return namespace
+
+    def links(self) -> OrderedSet[Link]:
+        links = super().links()
+        links.add(Link(target=Id(Prefix.TABLE, (self.constellation,)), label=PARENT_LINK))
+        return links
+
+
+@dataclass(frozen=True)
+class Role(_ConstellationField):
+    """
+    A Role is a qualitative property that describes `Metrics` - metrics can be sliced
+    and diced by roles.
+
+    Roles are scoped to `Constellation`s.
+
+    Each table contributing to a constellation can assign columns to a Role.
+
+    `datatype` can be a physical data type or a REF to an entity or dataset, in which case
+    datasets must implement the role with a relationship.
+    """
+
+    datatype: exp.DataType | None = None
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        deps: OrderedSet[Id | Path] = OrderedSet()
+        deps.add(Id(Prefix.TABLE, (self.constellation,)))
+        return deps
+
+    def links(self) -> OrderedSet[Link]:
+        links = super().links()
+        reference = ref_target(self.datatype)
+        if reference is not None:
+            target = Id(Prefix.TABLE, (in_namespace(reference, self.namespace),))
+            links.add(Link(target=target, label=TYPE_LINK))
+        return links
+
+
+@dataclass(frozen=True)
+class Metric(_ConstellationField):
+    """
+    A Metric is an aggregatable measurement.
+
+    Metrics are typically quantitative, but they can be any aggregatable value.
+    Metrics can span multiple datasets.
+
+    Metrics are scoped to `Constellation`s.
+
+    `expr` is a SQL expression, where qualified columns reference fields from datasets
+    and unqualified columns reference other metrics or roles.
+
+    `datatype` should be resolved from dependencies.
+    """
+
+    expr: exp.Expr
+    datatype: exp.DataType | None = None
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        deps: OrderedSet[Id | Path] = OrderedSet()
+        deps.add(Id(Prefix.TABLE, (self.constellation,)))
+        for col in self.expr.find_all(exp.Column):
+            # Unqualified references are other metrics or roles.
+            if not col.table:
+                deps.add(Id(Prefix.FIELD, (self.constellation, col.this.copy())))
+        for parts in column_paths(self.expr):
+            for dataset, path in dataset_paths(parts, self.namespace):
+                deps.add(Path(base=Id(Prefix.TABLE, (dataset,)), path=path))
+        return deps
+
+
+@dataclass(frozen=True)
+class Assignment(BaseObject):
+    """
+    An Assignment defines how a `Dataset` provides a `Role`.
+
+    This is akin to "dimension conforming" in traditional dimensional modeling, allowing
+    cross-table analysis - queries reference roles, and the physical location
+    of roles are resolved per dataset.
+
+    Assignments are unique per `constellation`/`dataset`/`role`.
+
+    `expr` is a sql expression, where columns are resolved relative to `dataset`.
+    """
+
+    constellation: exp.Table
+    dataset: exp.Table
+    role: exp.Identifier
+    expr: exp.Expr
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.ASSIGNMENT, (self.constellation, self.role, self.dataset))
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        deps = deps_from_dataset_expr(self.dataset, self.expr)
+        # A constant like `'store'` reads no column, but the branch still reads the dataset.
+        deps.add(Id(Prefix.TABLE, (self.dataset,)))
+        return deps
+
+    def links(self) -> OrderedSet[Link]:
+        links = super().links()
+        links.add(Link(target=Id(Prefix.TABLE, (self.constellation,)), label=PARENT_LINK))
+        links.add(Link(target=Id(Prefix.FIELD, (self.constellation, self.role)), label=PARENT_LINK))
+        return links