model2data 1.10.1__tar.gz → 1.10.4__tar.gz

{model2data-1.10.1/model2data.egg-info → model2data-1.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 1.10.1
+Version: 1.10.4
 Summary: Generate analytics-ready datasets from DBML models
 Author: JB Analytica
 License-Expression: MIT

{model2data-1.10.1 → model2data-1.10.4}/README.md

@@ -49,9 +49,9 @@ access required.
   `sku ean13`) to pick its generator outright when the name is wrong for the data.
 - **Relationship-preserving.** Foreign keys resolve to real parent rows; tables are generated in
   dependency order.
-- **Deterministic.** Pass `--seed` and the same schema always produces the same data — safe to
-  commit fixtures, safe to diff across CI runs. Add `--as-of` to pin the date the data is anchored
-  on, and the run reproduces on any later day rather than only on the day it first ran.
+- **Deterministic, byte for byte.** The same model, `--seed` and `--as-of` (and the same options)
+  give the same files, byte for byte, on any machine and in every 1.x release — safe to commit
+  fixtures, safe to diff across CI runs and upgrades. See [The determinism promise](#the-determinism-promise).
 - **Re-rollable one table at a time.** `--table-seed orders=7` regenerates a single table and
   leaves every other table byte-identical, so you can keep the four tables that look right.
 - **A real dbt project, not just CSVs.** Seeds, staging models that `ref()` them, schema tests,
@@ -222,14 +222,14 @@ jobs:
 ```
 
 Inputs: `files` (glob, default `**/*.model2data.yml`), `version` (default: the release the tag
-points at, e.g. `1.10.1`; a specifier such as `>=1.10,<2` also works), `python-version` (3.12).
+points at, e.g. `1.10.4`; a specifier such as `>=1.10,<2` also works), `python-version` (3.12).
 
 **pre-commit**
 
 ```yaml
 repos:
   - repo: https://github.com/JB-Analytica/model2data
-    rev: v1.10.1
+    rev: v1.10.4
     hooks:
       - id: model2data-validate
 ```
@@ -243,7 +243,7 @@ model2data:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes: ["**/*.model2data.yml"]
   script:
-    - pip install model2data==1.10.1
+    - pip install model2data==1.10.4
     - model2data validate --glob "**/*.model2data.yml"
 ```
 
@@ -309,6 +309,21 @@ fixture safe to commit:
 model2data --file examples/ecommerce.model2data.yml --rows 200 --seed 42 --as-of 2026-01-31
 ```
 
+### The determinism promise
+
+The same model, seed and `--as-of` — with the same options (`--rows`, `--rows-for`,
+`--table-seed`, `--locale`, `--defects`, `--days`, …) — produce the same files, byte for byte:
+
+- on any machine and operating system, and on every supported Python version;
+- with any Faker or pandas version model2data accepts;
+- in every 1.x release. A release may make generation faster or add options, but never changes
+  what an existing model, seed and set of options produce. A fix that cannot keep that waits for
+  a new major version, and the changelog says so.
+
+`tests/test_determinism.py` holds the file hashes of a spread of models, presets, multi-day runs
+and locales, and runs on every pull request — against the newest Faker and pandas too — so a
+change to a single output byte fails the build rather than reaching a release.
+
 If one table comes out wrong and the rest looks right, `--table-seed` re-rolls just that table.
 Every other table's seed CSV stays byte-identical, and children of the re-rolled table still
 reference rows that exist, so there's nothing to re-check but the table you asked to change:

{model2data-1.10.1 → model2data-1.10.4}/model2data/defects/apply.py

@@ -64,7 +64,7 @@ from model2data import __version__
 from model2data.dbt.hint_tests import DEFAULT_TOLERANCE
 from model2data.dbt.naming import dbt_names, for_dbt
 from model2data.dbt.tests import DbtTest, dbt_tests
-from model2data.defects.checks import as_seeded, failing
+from model2data.defects.checks import Columns, as_seeded, failing, fails
 from model2data.defects.report import AppliedDefect, DefectsReport, ExpectedFailure
 from model2data.generate import kinds
 from model2data.generate.days import DayResult, TableDay
@@ -123,6 +123,8 @@ class _Table:
         # so no defect undoes another and every count holds.
         self.claimed: dict[str, set[int]] = {}
         self.patches: list[_Patch] = []
+        # What `referenced` found, by the columns asked about: the clean frames never change.
+        self._referenced: dict[tuple[str, ...], frozenset[int]] = {}
         rows = len(self.frame)
         if results is not None and key not in results[0].tables:
             results = None  # a table the days do not hold: a history table
@@ -162,15 +164,20 @@ class _Table:
 
     def referenced(self, columns: Sequence[str]) -> set[int]:
         """The rows whose value in one of `columns` some child row holds."""
-        found: set[int] = set()
-        for ref in self.refs:
-            if ref["target_table"] != self.key or ref["target_column"] not in columns:
-                continue
-            child = self.frames[ref["source_table"]][ref["source_column"]].tolist()
-            held = {_comparable(v) for v in child if _present(v)}
-            values = self.frame[ref["target_column"]].tolist()
-            found.update(p for p, v in enumerate(values) if _present(v) and _comparable(v) in held)
-        return found
+        asked = tuple(columns)
+        if asked not in self._referenced:
+            found: set[int] = set()
+            for ref in self.refs:
+                if ref["target_table"] != self.key or ref["target_column"] not in asked:
+                    continue
+                child = self.frames[ref["source_table"]][ref["source_column"]].tolist()
+                held = {_comparable(v) for v in child if _present(v)}
+                values = self.frame[ref["target_column"]].tolist()
+                found.update(
+                    p for p, v in enumerate(values) if _present(v) and _comparable(v) in held
+                )
+            self._referenced[asked] = frozenset(found)
+        return set(self._referenced[asked])
 
     def free(self, candidates: list[int], columns: Sequence[str]) -> list[int]:
         """`candidates` without the rows an earlier defect broke one of `columns` in."""
@@ -218,6 +225,9 @@ class _Table:
 
 
 def _present(value: Any) -> bool:
+    kind = type(value)
+    if kind is str or kind is int:
+        return True
     return (
         value is not None
         and not (isinstance(value, float) and math.isnan(value))
@@ -227,6 +237,9 @@ def _present(value: Any) -> bool:
 
 def _comparable(value: Any) -> Any:
     """A value as it compares to the warehouse: 7, 7.0 and numpy's 7 are one value."""
+    kind = type(value)
+    if kind is str or kind is int:
+        return value
     item = getattr(value, "item", None)
     if callable(item):
         value = item()
@@ -290,7 +303,9 @@ def _duplicate_keys(table: _Table, defect: Defect, rng: random.Random, count: in
     if not columns:
         return _Outcome(None, note=f"{table.key} has no primary key to repeat")
     values = {name: table.frame[name].tolist() for name in columns}
-    complete = [p for p in range(len(table.frame)) if all(_present(values[c][p]) for c in columns)]
+    complete = [
+        p for p, row in enumerate(zip(*values.values(), strict=True)) if all(map(_present, row))
+    ]
     avoid = table.referenced(columns)
     wanted = min(count, max(len(complete) - 1, 0))
     targets = _choose(rng, table.free(complete, columns), wanted, avoid)
@@ -668,15 +683,54 @@ def requested_rows(defect: Defect, rows: int) -> int:
 def _patched(
     frame: pd.DataFrame, patches: Sequence[_Patch], day: Optional[int] = None
 ) -> pd.DataFrame:
-    """A copy of `frame` with `patches` set, those from day `day` on when it is given."""
+    """A copy of `frame` with `patches` set, those from day `day` on when it is given.
+
+    A column of objects is set as a list, in one go; any other cell by cell, as
+    `_set` does, for what pandas makes of a value its column's type cannot hold.
+    """
     out = frame.copy()
+    by_column: dict[str, list[_Patch]] = {}
     for patch in patches:
         if patch.position >= len(out) or (day is not None and patch.since > day):
             continue
-        _set(out, patch.position, patch.column, patch.value)
+        by_column.setdefault(patch.column, []).append(patch)
+    for column, cells in by_column.items():
+        if out[column].dtype == object:
+            values = out[column].tolist()
+            for patch in cells:
+                values[patch.position] = patch.value
+            out[column] = pd.Series(values, index=out.index, dtype=object)
+            continue
+        if isinstance(out[column].dtype, _NULLABLE_INTEGERS) and all(
+            type(patch.value) is int for patch in cells
+        ):
+            # What `_set` does to each cell, done to all of them at once.
+            final = {patch.position: patch.value for patch in cells}
+            array = out[column].array.copy()
+            try:
+                array[list(final)] = list(final.values())
+            except (TypeError, ValueError):
+                pass  # too large for the column: cell by cell, as before
+            else:
+                out[column] = pd.Series(array, index=out.index)
+                continue
+        for patch in cells:
+            _set(out, patch.position, column, patch.value)
     return out
 
 
+_NULLABLE_INTEGERS = (
+    pd.Int8Dtype,
+    pd.Int16Dtype,
+    pd.Int32Dtype,
+    pd.Int64Dtype,
+    pd.UInt8Dtype,
+    pd.UInt16Dtype,
+    pd.UInt32Dtype,
+    pd.UInt64Dtype,
+)
+
+
 def _set(frame: pd.DataFrame, position: int, column: str, value: Any) -> None:
     index = frame.columns.get_loc(column)
     try:
@@ -823,18 +877,41 @@ def _report(
     dbt_tables, dbt_refs = for_dbt(inputs.tables, inputs.refs, names)
     tests = dbt_tests(dbt_tables, dbt_refs, hint_tests=hint_tests, test_tolerance=test_tolerance)
     keys = {name: key for key, name in names.items()}
+    cache = Columns()
     seeded_clean = {names[key]: as_seeded(frame) for key, frame in clean.items()}
-    seeded_broken = {names[key]: as_seeded(frame) for key, frame in broken.items()}
-    failing_clean = failing(tests, seeded_clean)
-    failing_broken = failing(tests, seeded_broken)
+    seeded_broken = dict(seeded_clean)
+    done: dict[str, tuple[pd.DataFrame, Sequence[_Patch]]] = {}
+    for key, frame in broken.items():
+        if frame is not clean.get(key):
+            patches = [p for k, _, _, o in applied if k == key for p in o.patches]
+            seeded = _seeded_patched(clean[key], seeded_clean[names[key]], patches, cache)
+            seeded_broken[names[key]] = seeded
+            done[key] = (seeded, patches)
+    fails_clean = [fails(test, seeded_clean, cache=cache) for test in tests]
+    failing_clean = {test.name for test, failed in zip(tests, fails_clean, strict=True) if failed}
+    failing_broken = failing(tests, seeded_broken, cache=cache)
     alone: list[set[str]] = []
     for key, _, _, outcome in applied:
         if not outcome.patches:
             alone.append(set())
             continue
+        # A test that does not read this table fails as it does on the clean data.
+        name = names[key]
         seeded = dict(seeded_clean)
-        seeded[names[key]] = as_seeded(_patched(clean[key], outcome.patches))
-        alone.append(failing(tests, seeded))
+        seeded[name] = _seeded_patched(
+            clean[key], seeded_clean[name], outcome.patches, cache, done.get(key)
+        )
+        alone.append(
+            {
+                test.name
+                for test, failed in zip(tests, fails_clean, strict=True)
+                if (
+                    fails(test, seeded, cache=cache)
+                    if test.table == name or (test.parent is not None and test.parent[0] == name)
+                    else failed
+                )
+            }
+        )
 
     expected: list[ExpectedFailure] = []
     by_defect: list[list[str]] = [[] for _ in applied]
@@ -871,6 +948,52 @@ def _report(
     return report
 
 
+def _seeded_patched(
+    frame: pd.DataFrame,
+    base: pd.DataFrame,
+    patches: Sequence[_Patch],
+    cache: Columns,
+    done: Optional[tuple[pd.DataFrame, Sequence[_Patch]]] = None,
+) -> pd.DataFrame:
+    """`as_seeded(_patched(frame, patches))`, given `base`, `as_seeded(frame)`.
+
+    Only the columns `patches` set are written out and read back, as a column
+    reads back the same whichever columns stand beside it; the others, and what
+    `cache` knows of them, are `base`'s. `done` is another seeded frame of
+    `frame` and the patches it was made with: a column those patches set exactly
+    as `patches` do is taken from it rather than written out again.
+    """
+    mine = _by_column(patches)
+    theirs = _by_column(done[1]) if done is not None else {}
+    reuse = [
+        column
+        for column, own in mine.items()
+        if len(own) == len(theirs.get(column, ()))
+        and all(a is b for a, b in zip(own, theirs[column], strict=True))
+    ]
+    fresh = [column for column in mine if column not in reuse]
+    seeded = base.copy()
+    if fresh:
+        part = as_seeded(
+            _patched(frame[fresh], [patch for patch in patches if patch.column in fresh])
+        )
+        for column in fresh:
+            seeded[column] = part[column]
+    cache.share(seeded, base, [column for column in base.columns if column not in mine])
+    if done is not None and reuse:
+        for column in reuse:
+            seeded[column] = done[0][column]
+        cache.share(seeded, done[0], reuse)
+    return seeded
+
+
+def _by_column(patches: Sequence[_Patch]) -> dict[str, list[_Patch]]:
+    by_column: dict[str, list[_Patch]] = {}
+    for patch in patches:
+        by_column.setdefault(patch.column, []).append(patch)
+    return by_column
+
+
 def _culprits(
     test: DbtTest,
     alone: list[set[str]],
@@ -932,15 +1055,17 @@ def _recount(defect: Defect, outcome: _Outcome, frame: Optional[pd.DataFrame]) -
     by_row: dict[int, list[_Patch]] = {}
     for patch in outcome.patches:
         by_row.setdefault(patch.position, []).append(patch)
+    columns = sorted({patch.column for patch in outcome.patches})
+    cells = {column: frame[column].tolist() for column in columns}
     held = [
         p
         for p in outcome.positions
-        if all(_same(frame[c.column].iloc[p], c.value) for c in by_row.get(p, []))
+        if all(_same(cells[c.column][p], c.value) for c in by_row.get(p, []))
     ]
     if defect.type == "duplicate_keys":
-        columns = sorted({patch.column for patch in outcome.patches})
         keys = [
-            tuple(_comparable(v) for v in row) for row in frame[columns].itertuples(index=False)
+            tuple(map(_comparable, row))
+            for row in zip(*(cells[column] for column in columns), strict=True)
         ]
         counts: dict[tuple, int] = {}
         for key in keys:
@@ -998,7 +1123,8 @@ def _applied(
     # A defect with no table to break (a history not kept) broke no row.
     primary = primary_key(inputs.tables[key]) if key in inputs.tables else []
     numbers = [position + 1 for position in outcome.positions]
-    keys = [[_native(frame[name].iloc[p]) for name in primary] for p in outcome.positions]
+    held = {name: frame[name].tolist() for name in primary} if outcome.positions else {}
+    keys = [[_native(held[name][p]) for name in primary] for p in outcome.positions]
     rows: list[Any] = [values[0] if len(values) == 1 else values for values in keys]
     row_key: Optional[list[str]] = list(primary) or None
     if not primary: