model2data 1.16.0__tar.gz → 1.16.2__tar.gz

{model2data-1.16.0/model2data.egg-info → model2data-1.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 1.16.0
+Version: 1.16.2
 Summary: Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values
 Author: JB Analytica
 License-Expression: MIT

{model2data-1.16.0 → model2data-1.16.2}/model2data/metrics/sql.py

@@ -5,11 +5,11 @@ Two dialects of the same thing:
 - **warehouse SQL** (`query`, `scalar`), which the known values run on DuckDB
   and the generated dbt tests run on DuckDB or Postgres. It reads relations a
   caller names (a DuckDB table, a dbt `ref()`), joins the tables a metric's
-  filter and time are on along their many-to-one path with `left join` (a row
-  whose parent is missing stays, and fails any filter on the parent, as a null
-  does), and casts every column it reads to the type its kind is read as
-  (`model2data.metrics.columns.SQL_TYPES`), so the answer does not depend on
-  how a CSV loader guessed the column's type.
+  filter is on (and, by month, its time) along their many-to-one path with
+  `left join` (a row whose parent is missing stays, and fails any filter on the
+  parent, as a null does), and casts every column it reads to the type its kind
+  is read as (`model2data.metrics.columns.SQL_TYPES`), so the answer does not
+  depend on how a CSV loader guessed the column's type.
 - **ANSI SQL for Ossie** (`ansi`), over `dataset.field` names, with no joins:
   an Ossie consumer joins along the relationships the document declares.
 
@@ -149,19 +149,27 @@ def aggregate_sql(
 # ---------------------------------------------------------------------------
 # Warehouse SQL
 # ---------------------------------------------------------------------------
-def _cast(alias: str, column: str, kind: ColumnKind) -> str:
-    return f"cast({alias}.{quote(column)} as {SQL_TYPES[kind]})"
+class _From:
+    """The `from` and `join` clauses of a simple metric, and the alias of each table in them.
 
+    Joins only the tables in `needed` (of `metric.joins`, in its order): a table the
+    query reads no column of stays out, so a parent with a duplicated key cannot repeat
+    the metric's rows in a query that does not need it.
 
-class _From:
-    """The `from` and `join` clauses of a simple metric, and the alias of each table in them."""
+    Every column the query reads is written by `column`, which also records it in `read`
+    (by table key), so `columns_read` knows them from the same code that writes them.
+    """
 
-    def __init__(self, metric: ResolvedMetric, relation: Relation):
+    def __init__(self, metric: ResolvedMetric, relation: Relation, needed: set[str]):
         assert metric.table is not None
         self.lines = [f"from {relation(metric.table)} as t0"]
         self.alias = {metric.table: "t0"}
+        self.read: dict[str, set[str]] = {metric.table: set()}
         by_prefix: dict[tuple, str] = {(): "t0"}
+        table_of = {"t0": metric.table}
         for table, path in metric.joins.items():
+            if table not in needed:
+                continue
             for length in range(1, len(path) + 1):
                 prefix = path[:length]
                 if prefix in by_prefix:
@@ -169,46 +177,91 @@ class _From:
                 alias = f"t{len(by_prefix)}"
                 previous = by_prefix[path[: length - 1]]
                 step = path[length - 1]
+                table_of[alias] = step.parent
                 on = " and ".join(
-                    f"cast({previous}.{quote(child)} as varchar) = cast({alias}.{quote(parent)} as varchar)"
+                    f"cast({self._at(previous, table_of[previous], child)} as varchar)"
+                    f" = cast({self._at(alias, step.parent, parent)} as varchar)"
                     for child, parent in zip(step.child_columns, step.parent_columns, strict=True)
                 )
                 self.lines.append(f"left join {relation(step.parent)} as {alias} on {on}")
                 by_prefix[prefix] = alias
             self.alias[table] = by_prefix[path]
 
+    def _at(self, alias: str, table: str, column: str) -> str:
+        self.read.setdefault(table, set()).add(column)
+        return f"{alias}.{quote(column)}"
+
+    def column(self, table: str, column: str) -> str:
+        """`column` of `table` (a table key) as the query reads it, through its alias."""
+        return self._at(self.alias[table], table, column)
+
+    def cast(self, table: str, column: str, kind: ColumnKind) -> str:
+        return f"cast({self.column(table, column)} as {SQL_TYPES[kind]})"
+
 
 def _parts(
-    semantic: SemanticModel, metric: ResolvedMetric, relation: Relation
+    semantic: SemanticModel, metric: ResolvedMetric, relation: Relation, by_month: bool
 ) -> tuple[str, _From]:
-    clauses = _From(metric, relation)
+    # The total joins what its filter reads; by month, the time column's table as well.
+    needed = (
+        {condition.column.rpartition(".")[0] for condition in metric.where.conditions()}
+        if metric.where is not None
+        else set()
+    )
+    if by_month:
+        assert metric.time is not None
+        needed.add(metric.time.rpartition(".")[0])
+    clauses = _From(metric, relation, needed)
 
     def column_of(path: str) -> tuple[str, ColumnKind]:
         table, column = find_column(semantic.model, path)
         assert table is not None and column is not None
         kind = column_kind(semantic.model, column)
-        return _cast(clauses.alias[table], path.rpartition(".")[2], kind), kind
+        return clauses.cast(table, path.rpartition(".")[2], kind), kind
 
     condition = where_sql(metric.where, column_of) if metric.where is not None else None
     value = None
     if metric.kind == "simple":
+        assert metric.table is not None
         assert metric.column is not None and metric.column_kind is not None
-        value = _cast("t0", metric.column, metric.column_kind)
+        value = clauses.cast(metric.table, metric.column, metric.column_kind)
     return aggregate_sql(metric, value, condition), clauses
 
 
-def query(semantic: SemanticModel, name: str, relation: Relation, *, by_month: bool = False) -> str:
-    """A simple metric or row count as a query: one row with its `value`, or, `by_month`, a
-    row per calendar month of its time column (`month` the month's first day)."""
+def _query(
+    semantic: SemanticModel, name: str, relation: Relation, by_month: bool
+) -> tuple[str, _From]:
     metric = semantic.metrics[name]
-    aggregate, clauses = _parts(semantic, metric, relation)
+    aggregate, clauses = _parts(semantic, metric, relation, by_month)
     joins = "\n".join(clauses.lines)
     if not by_month:
-        return f"select {aggregate} as value\n{joins}"
+        return f"select {aggregate} as value\n{joins}", clauses
     assert metric.time is not None
     table, column = metric.time.rpartition(".")[0], metric.time.rpartition(".")[2]
-    month = f"cast(date_trunc('month', cast({clauses.alias[table]}.{quote(column)} as timestamp)) as date)"
-    return f"select {month} as month, {aggregate} as value\n{joins}\ngroup by 1\norder by 1"
+    month = f"cast(date_trunc('month', cast({clauses.column(table, column)} as timestamp)) as date)"
+    return (
+        f"select {month} as month, {aggregate} as value\n{joins}\ngroup by 1\norder by 1",
+        clauses,
+    )
+
+
+def query(semantic: SemanticModel, name: str, relation: Relation, *, by_month: bool = False) -> str:
+    """A simple metric or row count as a query: one row with its `value`, or, `by_month`, a
+    row per calendar month of its time column (`month` the month's first day)."""
+    return _query(semantic, name, relation, by_month)[0]
+
+
+def columns_read(semantic: SemanticModel, name: str) -> dict[str, set[str]]:
+    """Every column `query` reads for a simple metric or row count, by table key, its
+    `by_month` query's time column included: the columns its aggregate, filter, time and
+    joins name. Every table the query reads is a key, the metric's own table always, even
+    when no column of it is named (a row count with no filter: `count(*)`).
+
+    Recorded while the query is written rather than found in its text, so it cannot
+    miss a column however the column is quoted or aliased.
+    """
+    metric = semantic.metrics[name]
+    return _query(semantic, name, str, by_month=metric.time is not None)[1].read
 
 
 def scalar(semantic: SemanticModel, name: str, relation: Relation) -> str:

{model2data-1.16.0 → model2data-1.16.2}/model2data/metrics/values.py

@@ -13,6 +13,10 @@ How the values are computed:
   any case) being null as dbt's seed loader reads them. Each column a metric
   touches is then cast to the SQL type of its kind, in the same SQL the dbt
   tests run (`model2data.metrics.sql`). DuckDB runs it, on one thread.
+- Only the columns some metric's query reads are read (`columns_read`), so a
+  wide table costs what its few metric columns cost. That gives the same
+  values: a column's CSV text depends on that column alone, and a query cannot
+  see a column it does not name.
 - Simple metrics and row counts are SQL aggregates. A ratio is its
   numerator's total over its denominator's, null when the denominator is null
   or zero; a derived metric is its expression over its inputs' totals, with
@@ -44,7 +48,7 @@ import pandas as pd
 from model2data.metrics import expression as expr
 from model2data.metrics.check import SPEC_VERSION
 from model2data.metrics.semantic import ResolvedMetric, SemanticModel
-from model2data.metrics.sql import query, quote
+from model2data.metrics.sql import columns_read, query, quote
 
 Number = Union[int, Decimal]
 _PLACES = Decimal("0.000001")
@@ -144,15 +148,42 @@ def _empty(metric: ResolvedMetric) -> Optional[Number]:
     return 0 if metric.kind == "count" or metric.agg in ("count", "count_distinct") else None
 
 
-def _as_text(frame: pd.DataFrame) -> pd.DataFrame:
-    """`frame` as its seed file reads back: every column text, `""` and `null` null."""
-    buffer = io.StringIO(frame.to_csv(index=False))
-    text = pd.read_csv(buffer, dtype=str, keep_default_na=False, na_values=[], na_filter=False)
-    return text.apply(lambda column: column.map(_null_or_text)).astype(object)
+def _text_columns(frame: pd.DataFrame, columns: list[Any]) -> pd.DataFrame:
+    """`columns` of `frame` as its seed file reads them back: every value the text of its
+    CSV field, nulls left as `""` (`_NULL_OR_TEXT` makes them null in DuckDB).
+
+    pandas writes each column of a CSV by that column's own type and values, and quotes
+    each field on its own, so a column's text is the same whether the other columns are
+    written beside it or not. One column with an empty field is written as `""`, not as
+    a blank line, so no row is lost when a single column is written. Read as `object`
+    rather than `str`: with no NA filtering every field is a Python string either way, and
+    pandas 3 reads `str` as its own string type, which DuckDB cannot scan.
+    """
+    buffer = io.StringIO(frame[columns].to_csv(index=False))
+    return pd.read_csv(buffer, dtype=object, keep_default_na=False, na_values=[], na_filter=False)
+
+
+# What dbt's seed loader reads as null: an empty field, or `null` in any case. The letters
+# are spelled out rather than compared with `lower()`, which folds non-ASCII letters too:
+# in Python only `n`, `u`, `l` and their capitals lower to those letters, so these are
+# exactly the fields for which `field.lower() == "null"`.
+_NULL_OR_TEXT = (
+    "case when {value} = '' or regexp_full_match({value}, '[Nn][Uu][Ll][Ll]') "
+    "then null else {value} end"
+)
+
 
+def _kept(frame: pd.DataFrame, read: set[str]) -> list[Any]:
+    """The columns of `frame` to load, in its order: those the queries read.
 
-def _null_or_text(value: Any) -> Any:
-    return None if value == "" or str(value).lower() == "null" else value
+    DuckDB matches identifiers without regard to case, so a column is kept when any name
+    read folds to its name; keeping one more than needed costs only memory. A table whose
+    rows a metric counts without naming any of its columns (`count(*)`) keeps its first
+    column, which has a value, or a null, on every row, so the count is the same.
+    """
+    folded = {name.casefold() for name in read}
+    kept = [column for column in frame.columns if str(column).casefold() in folded]
+    return kept or list(frame.columns[:1])
 
 
 def known_values(
@@ -165,35 +196,41 @@ def known_values(
     """Every metric of `semantic` over `frames` (by table key: what the seeds hold)."""
     import duckdb
 
+    computed = [
+        name for name, metric in semantic.metrics.items() if metric.kind in ("simple", "count")
+    ]
+    read: dict[str, set[str]] = {}
+    for name in computed:
+        for table, columns in columns_read(semantic, name).items():
+            read.setdefault(table, set()).update(columns)
+    keys = [table for table in semantic.entities if table in read]
+    for key in keys:
+        if key not in frames:
+            raise ValueError(f"no generated rows for table {key!r}, which a metric reads")
+    names = {key: f"m2d_{index}" for index, key in enumerate(keys)}
+    relation = names.__getitem__
+
     connection = duckdb.connect(":memory:")
     try:
         connection.execute("set threads = 1")
-        names: dict[str, str] = {}
-        used = {
-            table
-            for metric in semantic.metrics.values()
-            if metric.table is not None
-            for table in (metric.table, *metric.joins)
-        }
-        for index, key in enumerate(table for table in semantic.entities if table in used):
-            if key not in frames:
-                raise ValueError(f"no generated rows for table {key!r}, which a metric reads")
-            names[key] = f"m2d_{index}"
-            text = _as_text(frames[key])
+        for key in keys:
+            # One table's text at a time: it is freed once DuckDB holds its own copy.
+            text = _text_columns(frames[key], _kept(frames[key], read[key]))
             connection.register(f"{names[key]}_frame", text)
-            columns = ", ".join(f"cast({quote(c)} as varchar) as {quote(c)}" for c in text.columns)
+            columns = ", ".join(
+                f"{_NULL_OR_TEXT.format(value=f'cast({quote(c)} as varchar)')} as {quote(c)}"
+                for c in text.columns
+            )
             connection.execute(
                 f"create table {names[key]} as select {columns} from {names[key]}_frame"
             )
             connection.unregister(f"{names[key]}_frame")
+            del text
         raw: dict[str, tuple[Any, Optional[dict[str, Any]]]] = {}
-        for name, metric in semantic.metrics.items():
-            if metric.kind not in ("simple", "count"):
-                continue
-            relation = names.__getitem__
+        for name in computed:
             total = connection.execute(query(semantic, name, relation)).fetchone()
             by_month = None
-            if metric.time is not None:
+            if semantic.metrics[name].time is not None:
                 rows = connection.execute(query(semantic, name, relation, by_month=True)).fetchall()
                 by_month = {
                     month.strftime("%Y-%m"): value for month, value in rows if month is not None

{model2data-1.16.0 → model2data-1.16.2/model2data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 1.16.0
+Version: 1.16.2
 Summary: Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values
 Author: JB Analytica
 License-Expression: MIT

{model2data-1.16.0 → model2data-1.16.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "model2data"
-version = "1.16.0"
+version = "1.16.2"
 description = "Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values"
 readme = "README_PYPI.md"
 requires-python = ">=3.10"