metricflow-sql 0.1.1__py3-none-any.whl

metricflow_sql/objects/dataset.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Protocol, runtime_checkable
+
+from sqlglot import expressions as exp
+from sqlglot import parse_one
+
+from metricflow_sql.objects.base import (
+    PARENT_LINK,
+    BaseObject,
+    Id,
+    Link,
+    Path,
+    Prefix,
+    deps_from_dataset_expr,
+)
+from metricflow_sql.utils.normalize import resolve_name
+from metricflow_sql.utils.ordered_set import OrderedSet
+from metricflow_sql.utils.sql import ref_type
+from metricflow_sql.utils.template import render_sql
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class _DatasetField(BaseObject):
+    dataset: exp.Table
+    name: exp.Identifier
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.FIELD, (self.dataset, self.name))
+
+    def links(self) -> OrderedSet[Link]:
+        links = super().links()
+        links.add(Link(target=Id(Prefix.TABLE, (self.dataset,)), label=PARENT_LINK))
+        return links
+
+
+@dataclass(frozen=True)
+class Column(_DatasetField):
+    """
+    Concrete column in the underlying database.
+    """
+
+    datatype: exp.DataType | None = None
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        return OrderedSet([Id(Prefix.TABLE, (self.dataset,))])
+
+
+@dataclass(frozen=True)
+class Calculation(_DatasetField):
+    """
+    Derived column.
+
+    `expr` is a SQL expression, where columns reference other `Column`s, `Calculation`s, or
+    `Relationship`s.
+
+    `datatype` is an optional declaration. The type resolver infers undeclared types
+    from dependencies without changing this object.
+    """
+
+    expr: exp.Expr
+    datatype: exp.DataType | None = None
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        return deps_from_dataset_expr(self.dataset, self.expr)
+
+
+class Cardinality(StrEnum):
+    """Relationship cardinality"""
+
+    MANY_TO_ONE = "many_to_one"
+    ONE_TO_MANY = "one_to_many"
+
+
+@dataclass(frozen=True)
+class Relationship(_DatasetField):
+    """
+    Relationship to its `target`, an `Entity` or a `Dataset`.
+
+    Unique by `dataset`/`name`.
+
+    `key` maps the target's fields to expressions on this dataset. For an entity, the fields are
+    its `Entity.key` columns or its attributes. For a dataset, they are its `DatasetKey` parts or
+    any of its fields. A key that binds all of the target's key is many-to-one, and any other is
+    one-to-many.
+
+    `cardinality` optionally declares which, and must then agree with the key.
+    """
+
+    target: exp.Table
+    key: dict[exp.Identifier, exp.Expr]
+    cardinality: Cardinality | None = None
+
+    @property
+    def datatype(self) -> exp.DataType:
+        """A reference to the target, e.g. `REF(customer)`; registries resolve its fields."""
+        return ref_type(self.target)
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        deps: OrderedSet[Id | Path] = OrderedSet()
+        deps.add(Id(Prefix.TABLE, (self.target,)))
+        for name in self.key:
+            deps.add(Id(Prefix.FIELD, (self.target, name)))
+        for expr in self.key.values():
+            deps |= deps_from_dataset_expr(self.dataset, expr)
+        return deps
+
+
+@dataclass(frozen=True)
+class Parameter:
+    """
+    A value a query passes on its root table function, e.g. `store_sales(min_quantity => 90)`,
+    to every dataset the query reaches through relationships.
+    """
+
+    datatype: exp.DataType | None = None
+    default: exp.Expr | None = None
+
+
+def parameter_value(parameter: Parameter, argument: exp.Expr | None) -> exp.Expr:
+    """What the query passed, else the parameter's default, else NULL."""
+    return argument or parameter.default or exp.null()
+
+
+def argument(arguments: dict[str, exp.Expr], parameter: exp.Identifier) -> exp.Expr | None:
+    """
+    What the query passed for `parameter`, by its name as `resolve_name` resolves it: sqlglot
+    doesn't keep an argument name's quotes, e.g. `"MinAmount" => 90`.
+    """
+    name = resolve_name(parameter.name, arguments)
+    return None if name is None else arguments[name]
+
+
+@dataclass(frozen=True)
+class KeyParameter:
+    """A placeholder of a `KeyCondition`, e.g. `:as_of`."""
+
+    datatype: exp.DataType | None = None
+
+
+@dataclass(frozen=True)
+class KeyCondition:
+    """
+    A part of a `DatasetKey` that identifies rows by a condition rather than a field's value,
+    e.g. `:as_of >= valid_from AND :as_of < valid_to` for versions valid over a period.
+
+    `on` reads the dataset's fields, and the values relationships bind as `:name` placeholders.
+    `parameters` are those placeholders, by name.
+    """
+
+    on: exp.Expr
+    parameters: dict[exp.Identifier, KeyParameter]
+
+
+@dataclass(frozen=True)
+class DatasetKey:
+    """
+    What identifies a row of a dataset, e.g. `[user_id, item_id]`.
+
+    Each part is the name of a field, whose value identifies rows, or a `KeyCondition`.
+    """
+
+    parts: tuple[exp.Identifier | KeyCondition, ...]
+
+    def fields(self) -> tuple[exp.Identifier, ...]:
+        """The parts that are fields, e.g. `user_id`."""
+        fields: list[exp.Identifier] = []
+        for part in self.parts:
+            match part:
+                case exp.Identifier():
+                    fields.append(part)
+        return tuple(fields)
+
+    def conditions(self) -> tuple[KeyCondition, ...]:
+        """The parts that are conditions."""
+        conditions: list[KeyCondition] = []
+        for part in self.parts:
+            match part:
+                case KeyCondition():
+                    conditions.append(part)
+        return tuple(conditions)
+
+    def names(self) -> tuple[exp.Identifier, ...]:
+        """
+        The names relationships bind to the key, in order: fields, and condition parameters,
+        e.g. `user_id`, `as_of`.
+        """
+        names: list[exp.Identifier] = []
+        for part in self.parts:
+            match part:
+                case exp.Identifier():
+                    names.append(part)
+                case KeyCondition(parameters=parameters):
+                    names.extend(parameters)
+        return tuple(names)
+
+    def dependencies(self, dataset: exp.Table) -> OrderedSet[Id | Path]:
+        """
+        The fields of `dataset` the key reads, e.g. a calculation it names, which joins and fan-out
+        grouping expand like a `Source`'s `on`.
+        """
+        deps: OrderedSet[Id | Path] = OrderedSet()
+        for name in self.fields():
+            deps.add(Id(Prefix.FIELD, (dataset, name)))
+        for condition in self.conditions():
+            deps |= deps_from_dataset_expr(dataset, condition.on)
+        return deps
+
+
+@runtime_checkable
+class Dataset(Protocol):
+    @property
+    def name(self) -> exp.Table: ...
+
+    @property
+    def parameters(self) -> dict[exp.Identifier, Parameter]: ...
+
+    @property
+    def key(self) -> DatasetKey | None: ...
+
+    def from_(self, arguments: dict[str, exp.Expr]) -> exp.Expr:
+        """What the dataset reads from, given the query's table function arguments."""
+        ...
+
+
+@dataclass(frozen=True)
+class Table(BaseObject):
+    name: exp.Table
+    location: exp.Table
+    parameters: dict[exp.Identifier, Parameter] = field(default_factory=dict)
+    key: DatasetKey | None = None
+
+    def from_(self, arguments: dict[str, exp.Expr]) -> exp.Expr:
+        return self.location.copy()
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.TABLE, (self.name,))
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        return self.key.dependencies(self.name) if self.key is not None else OrderedSet()
+
+
+@dataclass(frozen=True)
+class View(BaseObject):
+    """
+    The rows of `sql`, a Jinja template over the view's `parameters`, written in `dialect`.
+    """
+
+    name: exp.Table
+    sql: str
+    dialect: str | None = None
+    parameters: dict[exp.Identifier, Parameter] = field(default_factory=dict)
+    key: DatasetKey | None = None
+
+    def from_(self, arguments: dict[str, exp.Expr]) -> exp.Expr:
+        values = {
+            name.name: parameter_value(parameter, argument(arguments, name))
+            for name, parameter in self.parameters.items()
+        }
+        rendered = render_sql(self.sql, values, dialect=self.dialect)
+        return exp.Subquery(this=parse_one(rendered, dialect=self.dialect))
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.TABLE, (self.name,))
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        return self.key.dependencies(self.name) if self.key is not None else OrderedSet()

metricflow_sql/objects/entity.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+from sqlglot import exp
+
+from metricflow_sql.objects.base import (
+    PARENT_LINK,
+    BaseObject,
+    Id,
+    Link,
+    Path,
+    Prefix,
+    dataset_paths,
+    deps_from_dataset_expr,
+)
+from metricflow_sql.utils.ordered_set import OrderedSet
+from metricflow_sql.utils.sql import column_paths
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class KeyColumn:
+    """
+    Specification for an individual `Entity.key`.
+
+    `optional` means that `Dataset` relationships don't need to reference it,
+    but then they don't get access to `Attributes` from `Source`s that implement it.
+    """
+
+    datatype: exp.DataType | None = None
+    optional: bool = False
+
+
+@dataclass(frozen=True)
+class Entity(BaseObject):
+    """
+    An Entity is an identifiable, referenceable business object.
+
+    Entities are like virtual datasets, composed from many datasets.
+    `Attribute`s act like an entity's columns.
+
+    Entities are unique by `name`.
+
+    `key` is an interface that `Source`s must implement and `Dataset` relationships reference.
+    """
+
+    name: exp.Table
+    key: dict[exp.Identifier, KeyColumn]
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.TABLE, (self.name,))
+
+
+@dataclass(frozen=True)
+class Source(BaseObject):
+    """
+    `Entity`s have `Attribute`s that are sourced from many `Dataset`s.
+    A Source defines such a source.
+
+    Sources are unique on `dataset`/`entity` combinations.
+
+    `on` is a SQL expression that defines how the `Dataset` implements the `Entity.key`.
+    It is used as the ON clause of JOIN expressions, where columns are relative to the `dataset`,
+    and entity key values, provided by the other side of the join, are referencable with
+    `:name` placeholders.
+    """
+
+    dataset: exp.Table
+    entity: exp.Table
+    on: exp.Expr
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.SOURCE, (self.entity, self.dataset))
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        deps: OrderedSet[Id | Path] = OrderedSet()
+        deps |= deps_from_dataset_expr(self.dataset, self.on)
+        deps.add(Id(Prefix.TABLE, (self.entity,)))
+        deps.add(Id(Prefix.TABLE, (self.dataset,)))
+        return deps
+
+    def links(self) -> OrderedSet[Link]:
+        links = super().links()
+        links.add(Link(target=Id(Prefix.TABLE, (self.entity,)), label=PARENT_LINK))
+        return links
+
+
+@dataclass(frozen=True)
+class Attribute(BaseObject):
+    """
+    An Attribute is a property of an `Entity`.
+
+    Attributes are unique on `entity`/`name` combinations.
+
+    `expr` is a SQL expression, where qualified columns reference fields from a `Source`
+    and unqualified columns reference other Attributes.
+
+    `datatype` should be resolved from dependencies.
+    """
+
+    entity: exp.Table
+    name: exp.Identifier
+    expr: exp.Expr
+    datatype: exp.DataType | None = None
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.FIELD, (self.entity, self.name))
+
+    def dependencies(self) -> OrderedSet[Id | Path]:
+        deps: OrderedSet[Id | Path] = OrderedSet()
+        for col in self.expr.find_all(exp.Column):
+            if not col.table:
+                deps.add(Id(Prefix.FIELD, (self.entity, col.this.copy())))
+        for parts in column_paths(self.expr):
+            # TODO - there's some ambiguity on whether an expression is <source>.<field> or <attribute>.<path>
+            for source_dataset, field_parts in dataset_paths(parts, self.entity.args["db"]):
+                deps.add(Id(Prefix.SOURCE, (self.entity, source_dataset)))
+                deps.add(Path(base=Id(Prefix.TABLE, (source_dataset,)), path=field_parts))
+                deps.add(Id(Prefix.FIELD, (source_dataset, self.name)))
+        return deps
+
+    def links(self) -> OrderedSet[Link]:
+        links = super().links()
+        links.add(Link(target=Id(Prefix.TABLE, (self.entity,)), label=PARENT_LINK))
+        return links

metricflow_sql/objects/reference.py

@@ -0,0 +1,35 @@
+"""
+References to objects in wiki links, e.g. `[revenue](metric:sales:revenue)`.
+
+A reference is an id, whose prefix may be an alias - see `Id.resolve`.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from sqlglot.dialects.dialect import NormalizationStrategy
+
+from metricflow_sql.objects.base import Id, Prefix
+
+logger = logging.getLogger(__name__)
+
+
+def parse_reference(url: str, dialect: str, strategy: NormalizationStrategy) -> Id | None:
+    """
+    Resolve `url` to the id it references, or None if it isn't a reference (e.g. `https:`).
+
+    Names are percent-decoded by `Id.resolve` (as ids escape `:` and `%`, and markdown parsers
+    encode URLs), and any `#fragment` is dropped, e.g. `Metric:sales:net%20revenue#notes` names the field `net revenue`.
+
+    Raises `ValueError` for a reference with a known prefix but a malformed name, so typos
+    surface instead of silently dropping the link.
+    """
+    prefix_text, separator, rest = url.partition(":")
+    try:
+        Prefix.parse(prefix_text)
+    except ValueError:
+        return None
+    if not separator:
+        return None
+    return Id.resolve(f"{prefix_text}:{rest.partition('#')[0]}", dialect, strategy)

metricflow_sql/objects/wiki.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Literal
+
+from sqlglot import exp
+from sqlglot.dialects.dialect import NormalizationStrategy
+from wenmode import Wenmode
+from wenmode.ast import plain_text, walk
+from wenmode.nodes import Heading, Root
+from wenmode.nodes import Link as MarkdownLink
+from wenmode.presets import github
+
+from metricflow_sql.objects.base import BaseObject, Chunk, Id, Link, Prefix
+from metricflow_sql.objects.reference import parse_reference
+from metricflow_sql.utils.ordered_set import OrderedSet
+
+logger = logging.getLogger(__name__)
+
+# Sections longer than this are split into several chunks, on block boundaries.
+MAX_CHUNK_CHARS = 2000
+
+_MARKDOWN = Wenmode(rules=github())
+
+
+@dataclass(frozen=True)
+class Page(BaseObject):
+    """
+    A markdown wiki page, stored alongside the semantic layer.
+
+    Pages are unique by `name`, which may contain `/` to organize pages hierarchically. Unlike
+    the semantic objects, pages aren't in namespaces, so a name may contain `.`, e.g.
+    `guides/v1.2`.
+
+    Links to other objects are regular markdown links with a reference as the URL, e.g.
+    `[revenue](metric:sales:revenue)` - see `metricflow_sql.objects.reference`.
+    """
+
+    name: str
+    content: str
+    # The fragment's, which its links are written in
+    dialect: str
+    # The registry's, so links read names as queries do. It's not the page's, so not compared.
+    normalization_strategy: NormalizationStrategy = field(compare=False)
+    content_type: Literal["text/markdown"] = "text/markdown"
+
+    @property
+    def id(self) -> Id:
+        return Id(Prefix.PAGE, (exp.to_identifier(self.name, quoted=True),))
+
+    def links(self) -> OrderedSet[Link]:
+        """
+        Links to other objects, in order of appearance.
+
+        Inline, reference-style and autolinks all count; links in code spans and blocks
+        don't, nor do links that aren't references (e.g. `https:`). Pages aren't in namespaces,
+        so a bare name in a reference is in the default one.
+        """
+        links: OrderedSet[Link] = OrderedSet()
+        for node in walk(self._parse()):
+            match node:
+                case MarkdownLink(url=url):
+                    target = parse_reference(url, self.dialect, self.normalization_strategy)
+                    if target is not None:
+                        links.add(Link(target=target, label=plain_text(node)))
+        return links
+
+    def chunks(self) -> tuple[Chunk, ...]:
+        """
+        The default name/label/description chunk, then the content split by heading.
+
+        Each chunk's `section` is its heading trail, e.g. `("Setup", "Install")` for text
+        under a `###` heading within a `#` heading, skipping the missing `##` level.
+        """
+        chunks = list(super().chunks())
+        section: tuple[str, ...] = ()
+        blocks: list[str] = []
+
+        def flush() -> None:
+            if blocks:
+                chunks.append(Chunk(text="\n\n".join(blocks), section=section))
+                blocks.clear()
+
+        for node in self._parse().children:
+            match node:
+                case Heading(depth=depth):
+                    flush()
+                    section = (*section[: depth - 1], plain_text(node))
+                case _:
+                    text = plain_text(node)
+                    if not text:
+                        continue
+                    if blocks and sum(map(len, blocks)) + len(text) > MAX_CHUNK_CHARS:
+                        flush()
+                    blocks.append(text)
+        flush()
+        return tuple(chunks)
+
+    def _parse(self) -> Root:
+        return _MARKDOWN.parse(self.content)