model2data 1.4.0__tar.gz → 1.6.0__tar.gz

{model2data-1.4.0/model2data.egg-info → model2data-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 1.4.0
+Version: 1.6.0
 Summary: Generate analytics-ready datasets from DBML models
 Author: JB Analytica
 License-Expression: MIT

{model2data-1.4.0 → model2data-1.6.0}/README.md

@@ -165,6 +165,73 @@ it's a per-run setting, so a table can't end up holding one Belgian and one Amer
 model2data --file examples/ecommerce.dbml --rows 200 --seed 42 --locale nl_BE
 ```
 
+### Shape when things happen
+
+By default, every date and timestamp is drawn uniformly across its window. `--business-hours`,
+`--growth`, and `--seasonality` shape that instead — weekdays and working hours, a trend across
+the window, and an annual cycle peaking in Q4:
+
+```bash
+model2data --file examples/ecommerce.dbml --rows 200 --seed 42 \
+  --business-hours --growth 0.5 --seasonality 0.3
+```
+
+Within a row, created/updated/deleted-style columns are ordered automatically — `updated_at` never
+lands before its own `created_at` — under any profile, uniform included. A column whose name
+doesn't say what it depends on can say so explicitly with an `after` note:
+
+```dbml
+shipped_at timestamp [note: '{"after": "ordered_at"}']
+```
+
+The flags above shape every date and timestamp column the same way, run-wide. `business_hours`,
+`growth`, and `seasonality` column note hints override that for one column at a time — the whole
+point being a run can be uniform everywhere except the one column that needs shaping, or shaped
+everywhere except the one column that shouldn't be:
+
+```dbml
+Table orders {
+  id int [pk]
+  created_at timestamp [note: '{"business_hours": true, "growth": 0.4}']
+  refunded_at timestamp [note: '{"growth": 0}']
+}
+```
+
+Here `created_at` gets business hours and growth even on an otherwise-uniform run, while
+`refunded_at` stays flat even under `--growth 0.5` — each hint only replaces the fields it names,
+so a partial hint like `{"growth": 0}` leaves that column's `business_hours`/`seasonality` at
+whatever the run-level flags set.
+
+### Shape how the data is spread
+
+By default every parent row is equally likely to be picked for a child row, and every column
+gets the same generic null rate, value spread, and true/false split. `--skew` changes the first
+part: `0.0` is that uniform default, `1.0` means a handful of parents hold most of the children —
+"a fifth of the customers place most of the orders":
+
+```bash
+model2data --file examples/ecommerce.dbml --rows 200 --seed 42 --skew 0.8
+```
+
+Column note hints shape the rest, per column:
+
+```dbml
+Table orders {
+  id int [pk]
+  customer_id int [ref: > customers.id, note: '{"skew": 0.9}']
+  status order_status [note: '{"weights": {"delivered": 20, "cancelled": 2}}']
+  is_paid boolean [note: '{"true_rate": 0.9}']
+  discount_code varchar [note: '{"null_rate": 0.8}']
+  shipping_city varchar [note: '{"distinct": 12}']
+}
+```
+
+`skew` on a foreign key overrides `--skew` for just that column. `weights` biases an enum column
+toward the values named (unnamed values still appear, at weight 1). `true_rate` is the fraction of
+non-null rows a boolean column comes back `true`. `null_rate` replaces the column's default null
+fraction outright. `distinct` draws the column's values from a fixed-size pool instead of a fresh
+value per row — a `shipping_city` most warehouses only ever see a handful of.
+
 Run dbt to load, transform, and test the data:
 
 ```bash

{model2data-1.4.0 → model2data-1.6.0}/model2data/cli.py

@@ -24,6 +24,7 @@ from model2data.generate.faker import (
     get_unmapped_columns,
     reset_stats,
 )
+from model2data.generate.options import TimeProfile
 from model2data.parse.dbml import get_parse_warnings, parse_dbml
 from model2data.utils import normalize_identifier
 
@@ -186,6 +187,43 @@ def main(
             "Pin it and a --seed run reproduces on any later day, not just the day it first ran."
         ),
     ),
+    business_hours: bool = typer.Option(
+        False,
+        "--business-hours",
+        help=(
+            "Weight generated timestamps toward weekdays and working hours,\n"
+            "instead of spreading them evenly over every hour of every day."
+        ),
+    ),
+    growth: float = typer.Option(
+        0.0,
+        "--growth",
+        min=-1.0,
+        help=(
+            "Relative change in activity across the generated window: 0.5 means the end\n"
+            "is half again as busy as the start, -0.3 means it tailed off. Default: flat."
+        ),
+    ),
+    seasonality: float = typer.Option(
+        0.0,
+        "--seasonality",
+        min=0.0,
+        max=1.0,
+        help=(
+            "Strength of an annual cycle in generated timestamps, 0 (none) to 1,\n"
+            "peaking in the fourth quarter."
+        ),
+    ),
+    skew: float = typer.Option(
+        0.0,
+        "--skew",
+        min=0.0,
+        max=1.0,
+        help=(
+            "How unevenly child rows are spread over their parents: 0 (every parent equally\n"
+            "likely, the default) to 1 (a few parents hold most of the children)."
+        ),
+    ),
     locale: Optional[str] = typer.Option(
         None,
         "--locale",
@@ -251,6 +289,28 @@ def main(
     if as_of is not None:
         typer.echo(f"📅 Anchoring generated dates on: {as_of.date()}")
 
+    # Same guard again for the shaping options: a direct call leaves them as
+    # `OptionInfo` objects, which means "not supplied", i.e. the uniform draw.
+    time_profile = TimeProfile(
+        business_hours=business_hours if isinstance(business_hours, bool) else False,
+        growth=growth if isinstance(growth, (int, float)) else 0.0,
+        seasonality=seasonality if isinstance(seasonality, (int, float)) else 0.0,
+    )
+    skew = skew if isinstance(skew, (int, float)) else 0.0
+    if not time_profile.is_uniform:
+        shaped = [
+            label
+            for label, active in (
+                ("business hours", time_profile.business_hours),
+                (f"growth {time_profile.growth:+.0%}", time_profile.growth != 0.0),
+                (f"seasonality {time_profile.seasonality:.0%}", time_profile.seasonality != 0.0),
+            )
+            if active
+        ]
+        typer.echo(f"🕒 Shaping timestamps: {', '.join(shaped)}")
+    if skew:
+        typer.echo(f"📈 Skewing child rows over their parents: {skew:.0%}")
+
     # -------------------------
     # Parse DBML (names untouched)
     # -------------------------
@@ -296,16 +356,26 @@ def main(
     # -------------------------
     typer.echo("🧮 Generating synthetic datasets from DBML definitions...")
     reset_stats()
-    generated_tables = generate_data_from_dbml(
-        tables=tables,
-        refs=refs,
-        base_rows=rows,
-        seed=seed,
-        row_overrides=row_overrides,
-        locale=locale,
-        as_of=as_of,
-        table_seeds=table_seeds,
-    )
+    try:
+        generated_tables = generate_data_from_dbml(
+            tables=tables,
+            refs=refs,
+            base_rows=rows,
+            seed=seed,
+            row_overrides=row_overrides,
+            locale=locale,
+            as_of=as_of,
+            table_seeds=table_seeds,
+            time_profile=time_profile,
+            skew=skew,
+        )
+    except ValueError as exc:
+        # A bad column-note hint (an unknown enum value in `weights`, `distinct`
+        # on a foreign key...) is a mistake in the schema, not a bug -- it
+        # deserves the same one-line, non-traceback treatment as every other
+        # validation error this command already reports.
+        typer.echo(f"❌ {exc}")
+        raise typer.Exit(1) from None
 
     # -------------------------
     # Write dbt seeds (normalized names)

{model2data-1.4.0 → model2data-1.6.0}/model2data/generate/core.py

@@ -17,10 +17,13 @@ from model2data.generate.faker import (
     reset_row_pools,
     set_locale,
 )
+from model2data.generate.hints import validate_hints
+from model2data.generate.options import UNIFORM, TimeProfile, validate_skew
 from model2data.generate.relationships import (
     build_fk_lookup,
     classify_refs,
 )
+from model2data.generate.timeline import order_row_times
 from model2data.parse.dbml import TableDef
 
 # Tables the most recent generate_data_from_dbml() call found stuck in an
@@ -72,6 +75,8 @@ def generate_data_from_dbml(
     locale: Optional[str] = None,
     as_of: AsOf = None,
     table_seeds: Optional[Mapping[str, int]] = None,
+    time_profile: Optional[TimeProfile] = None,
+    skew: float = 0.0,
 ) -> dict[str, pd.DataFrame]:
     """
     Generate synthetic datasets from parsed DBML definitions.
@@ -107,10 +112,23 @@ def generate_data_from_dbml(
     `seed` to work off -- with none, every table is already different on every
     run -- and unknown table names are an error rather than a silent no-op.
 
+    `time_profile` shapes *when* generated timestamps and dates fall: toward
+    business hours and weekdays, along a growth trend, with a seasonal peak.
+    See `TimeProfile`. None is the uniform profile, which draws every second of
+    the window with equal probability the way earlier releases did.
+
+    `skew` is how unevenly a child table's rows are spread over its parents,
+    from `0.0` (every parent equally likely, the earlier behaviour) to `1.0` (a
+    few parents hold most of the children). A column can override it with a
+    `{"skew": ...}` hint in its note.
+
     This function is deterministic if a seed is provided (and, with `as_of`,
     on any day). It performs no filesystem I/O and returns pandas DataFrames.
     """
     _validate_table_seeds(tables, table_seeds, seed)
+    validate_hints(tables, refs)
+    profile = time_profile or UNIFORM
+    skew = validate_skew(skew)
 
     # Locale first, then the seed: switching locale builds a new Faker, and the
     # seed has to be the last word on the generator that actually runs.
@@ -210,14 +228,35 @@ def generate_data_from_dbml(
                 force_not_null=column.name in composite_pk_columns,
                 table_name=table_name,
                 as_of=as_of,
+                time_profile=profile,
+                skew=skew,
             )
 
         df = pd.DataFrame(data)
+        # Ordering runs before FK resolution/dedup so a self-ref repair or a
+        # composite-key retry regenerates a temporal column's value into a
+        # frame that already respects created/updated/closed ordering, rather
+        # than one where only the untouched columns do.
+        df = order_row_times(df, table_def, as_of=as_of)
         df = _resolve_self_referencing_fks(
-            df, table_def, table_name, fk_lookup, row_count, as_of=as_of
+            df,
+            table_def,
+            table_name,
+            fk_lookup,
+            row_count,
+            as_of=as_of,
+            time_profile=profile,
+            skew=skew,
         )
         df = _deduplicate_composite_keys(
-            df, table_def, table_name, fk_lookup, generated, as_of=as_of
+            df,
+            table_def,
+            table_name,
+            fk_lookup,
+            generated,
+            as_of=as_of,
+            time_profile=profile,
+            skew=skew,
         )
 
         # -----------------------------------------------------
@@ -299,6 +338,8 @@ def _deduplicate_composite_keys(
     generated: dict[str, pd.DataFrame],
     max_attempts: int = 20,
     as_of: AsOf = None,
+    time_profile: Optional[TimeProfile] = None,
+    skew: float = 0.0,
 ) -> pd.DataFrame:
     """
     Regenerate colliding rows for any pk/unique composite key declared via an
@@ -350,7 +391,11 @@ def _deduplicate_composite_keys(
                         df.at[idx, col_name] = random.choice(fk_pools[col_name])
                     else:
                         df.at[idx, col_name] = generate_column_values(
-                            col_def, row_count=1, as_of=as_of
+                            col_def,
+                            row_count=1,
+                            as_of=as_of,
+                            time_profile=time_profile,
+                            skew=skew,
                         )[0]
                 combo = tuple(df.at[idx, c] for c in key_columns)
                 attempts += 1
@@ -377,6 +422,8 @@ def _resolve_self_referencing_fks(
     fk_lookup: dict[tuple[str, str], tuple[str, str]],
     row_count: int,
     as_of: AsOf = None,
+    time_profile: Optional[TimeProfile] = None,
+    skew: float = 0.0,
 ) -> pd.DataFrame:
     """
     Re-generate any FK column that references its own table (e.g. a
@@ -415,6 +462,8 @@ def _resolve_self_referencing_fks(
             force_not_null=column.name in composite_pk_columns,
             table_name=table_name,
             as_of=as_of,
+            time_profile=time_profile,
+            skew=skew,
         )
 
     return df

{model2data-1.4.0 → model2data-1.6.0}/model2data/generate/faker.py

@@ -1,16 +1,19 @@
 from __future__ import annotations
 
+import math
 import random
 import re
 import unicodedata
 import uuid
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import date, datetime, timedelta
 from typing import Callable, Optional, Union
 
 import pandas as pd
 from faker import Faker
 
+from model2data.generate.options import UNIFORM, TimeProfile
+from model2data.generate.timeline import weighted_dates, weighted_timestamps
 from model2data.parse.dbml import ColumnDef
 
 # ---------------------------------------------------------
@@ -502,6 +505,68 @@ def _infer_by_type(base_type: str) -> Optional[_Provider]:
     return lambda: fake.format(base_type)
 
 
+def _column_time_profile(
+    column: ColumnDef, time_profile: Optional[TimeProfile]
+) -> Optional[TimeProfile]:
+    """The run-level `time_profile` with this column's note hints applied on top.
+
+    Mirrors the `skew` override on the FK branch, one level up: a note doesn't
+    replace the run's profile, it patches only the fields it names (via
+    `dataclasses.replace`), so `{"growth": 0}` flattens one column of a
+    growing run while `business_hours`/`seasonality` stay exactly what the
+    run set. `validate_hints` has already confirmed the column is a date or
+    timestamp column and that each hint present is well-typed, so this does
+    no validation of its own -- a column with no hint gets `time_profile`
+    back unchanged, including a bare `None`, so the uniform path stays
+    untouched.
+    """
+    note = column.note or {}
+    overrides = {
+        key: note[key] for key in ("business_hours", "growth", "seasonality") if key in note
+    }
+    if not overrides:
+        return time_profile
+    base = time_profile if time_profile is not None else UNIFORM
+    return replace(base, **overrides)
+
+
+def _generate_dates(row_count: int, as_of: AsOf, time_profile: Optional[TimeProfile]) -> list:
+    """The date branch's values: shaped by `time_profile` when it isn't uniform.
+
+    A bare `-> list` return, matching `generate_column_values` itself: `values`
+    there is reassigned by every branch of one big if/elif chain (int, float,
+    uuid, this one...) and then written into by the nullability pass below with
+    an arbitrary `column.default`, so none of those branches can commit to a
+    concrete element type without the type checker flagging the later write as
+    unsound. Keeping `weighted_dates`'s real `list[date]` return type contained
+    to this one call, instead of leaking it into `values`, is what lets
+    `weighted_dates` itself stay properly typed for its own callers and tests.
+    """
+    anchor = _anchor_date(as_of)
+    if time_profile is not None and not time_profile.is_uniform:
+        return weighted_dates(
+            row_count, time_profile, start=_years_before(anchor, _DATE_WINDOW_YEARS), end=anchor
+        )
+    # Explicit endpoints rather than Faker's "-2y"/"today" shorthand: those
+    # strings are resolved against `date.today()` inside Faker, which is
+    # precisely the hidden dependency on the wall clock `as_of` removes.
+    return [
+        fake.date_between(start_date=_years_before(anchor, _DATE_WINDOW_YEARS), end_date=anchor)
+        for _ in range(row_count)
+    ]
+
+
+def _generate_timestamps(row_count: int, as_of: AsOf, time_profile: Optional[TimeProfile]) -> list:
+    """The timestamp branch's values: shaped by `time_profile` when it isn't uniform.
+
+    See `_generate_dates` for why this returns a bare `list` rather than
+    `weighted_timestamps`'s own `list[str]`.
+    """
+    if time_profile is not None and not time_profile.is_uniform:
+        return weighted_timestamps(row_count, time_profile, anchor=_anchor_date(as_of))
+    return [_random_datetime(as_of=as_of).isoformat(sep=" ") for _ in range(row_count)]
+
+
 # ---------------------------------------------------------
 # Public API
 # ---------------------------------------------------------
@@ -513,11 +578,19 @@ def generate_column_values(
     force_not_null: bool = False,
     table_name: Optional[str] = None,
     as_of: AsOf = None,
+    time_profile: Optional[TimeProfile] = None,
+    skew: float = 0.0,
 ) -> list:
     """
     Generate synthetic values for a single column.
     Respects FKs, uniqueness, and optional min/max hints in column notes.
 
+    `time_profile` is read by the date and timestamp branches, `skew` by the
+    foreign-key branch; both default to the uniform behaviour of earlier
+    releases. They are accepted here rather than in a separate pass so that
+    every path that draws a value -- the main pass, the self-referencing FK
+    repair, the composite-key retry -- draws it the same way.
+
     `as_of` is the date every generated date and timestamp is placed relative
     to, defaulting to today. Pass it to make a seeded run reproduce on any
     later day rather than only on the day it first ran.
@@ -534,12 +607,57 @@ def generate_column_values(
     # (an `id` on each of two tables) stay distinguishable in the report.
     unique_label = f"{table_name}.{column.name}" if table_name else column.name
 
+    # A `distinct` hint means "draw from a small pool", which is orthogonal to
+    # every type-specific branch below: generate the pool through this same
+    # function (so a pooled `city` still reads the row-identity pool, a pooled
+    # `int` still respects its own min/max), then repeat pool entries to fill
+    # row_count. validate_hints has already refused this on an FK/pk/unique/
+    # enum column, so there is no interaction with those branches to worry
+    # about. Handled before anything else so every other branch stays exactly
+    # what it was for a column with no `distinct` hint.
+    column_note = column.note or {}
+    distinct = column_note.get("distinct")
+    if distinct is not None:
+        pool_note = {key: value for key, value in column_note.items() if key != "distinct"}
+        pool = generate_column_values(
+            column=replace(column, note=pool_note or None),
+            row_count=distinct,
+            fk_series=None,
+            ensure_unique=False,
+            force_not_null=True,
+            table_name=table_name,
+            as_of=as_of,
+            time_profile=time_profile,
+            skew=skew,
+        )
+        values = random.choices(pool, k=row_count)
+        if not force_not_null and "not null" not in column.settings and "pk" not in column.settings:
+            _null_out(values, _null_fraction_for(column, row_count), column.default, row_count)
+        return values
+
     if column.enum_values:
-        return [random.choice(column.enum_values) for _ in range(row_count)]
+        note = column.note or {}
+        weights = note.get("weights")
+        null_rate_hint = "null_rate" in note
+        if weights is None and not null_rate_hint:
+            return [random.choice(column.enum_values) for _ in range(row_count)]
+
+        if weights is not None:
+            # Values the hint doesn't mention default to weight 1, so naming
+            # only the ones that matter (`{"delivered": 20}`) doesn't silently
+            # drop the rest of the enum.
+            enum_weights = [float(weights.get(value, 1)) for value in column.enum_values]
+            values = random.choices(column.enum_values, weights=enum_weights, k=row_count)
+        else:
+            values = [random.choice(column.enum_values) for _ in range(row_count)]
+
+        if null_rate_hint and not force_not_null:
+            _null_out(values, note["null_rate"], column.default, row_count)
+        return values
 
     dtype = column.data_type.lower()
     base_type = dtype.split("(")[0].strip()
-    values: list = []
+    values = []
 
     # Extract min/max from note if present
     min_val = None
@@ -556,7 +674,34 @@ def generate_column_values(
         # other branch here already respects `not null`/`pk` via the
         # nullability pass below.
         fk_values = fk_series.tolist()
-        values = [random.choice(fk_values) for _ in range(row_count)]
+        effective_skew = column.note.get("skew") if column.note else None
+        if effective_skew is None:
+            effective_skew = skew
+
+        if effective_skew == 0.0:
+            # Unchanged from every release before skew existed.
+            values = [random.choice(fk_values) for _ in range(row_count)]
+        else:
+            # Distinct parents, in their original order, shuffled so *which*
+            # ones end up popular is randomized under the seed rather than
+            # always being the first ones inserted.
+            seen: set = set()
+            parents = []
+            for value in fk_values:
+                if value not in seen:
+                    seen.add(value)
+                    parents.append(value)
+            random.shuffle(parents)
+
+            # Geometric decay by rank: w_i = exp(-lam * i / n), lam = 8 *
+            # skew**2. At skew 0.8 (lam=5.12) the top 20% of parents hold
+            # roughly 65-75% of the children; at skew 1.0 (lam=8) they hold
+            # roughly 80%; at skew 0 every parent is equally likely, handled
+            # above. Both ranges are pinned in tests/test_shaping.py.
+            n = len(parents)
+            lam = 8 * effective_skew**2
+            weights = [math.exp(-lam * i / n) for i in range(n)]
+            values = random.choices(parents, weights=weights, k=row_count)
 
     # -----------------------------------------------------
     # UUIDs / hashes
@@ -620,26 +765,23 @@ def generate_column_values(
     # Booleans
     # -----------------------------------------------------
     elif "boolean" in base_type or "bool" in base_type:
-        values = [random.choice([True, False]) for _ in range(row_count)]
+        true_rate = column.note.get("true_rate") if column.note else None
+        if true_rate is None:
+            values = [random.choice([True, False]) for _ in range(row_count)]
+        else:
+            values = [random.random() < true_rate for _ in range(row_count)]
 
     # -----------------------------------------------------
     # Dates
     # -----------------------------------------------------
     elif "date" in base_type and "time" not in base_type:
-        # Explicit endpoints rather than Faker's "-2y"/"today" shorthand: those
-        # strings are resolved against `date.today()` inside Faker, which is
-        # precisely the hidden dependency on the wall clock `as_of` removes.
-        anchor = _anchor_date(as_of)
-        values = [
-            fake.date_between(start_date=_years_before(anchor, _DATE_WINDOW_YEARS), end_date=anchor)
-            for _ in range(row_count)
-        ]
+        values = _generate_dates(row_count, as_of, _column_time_profile(column, time_profile))
 
     elif "time" in base_type and "stamp" not in base_type:
         values = [fake.time() for _ in range(row_count)]
 
     elif any(key in base_type for key in ["timestamp", "datetime"]):
-        values = [_random_datetime(as_of=as_of).isoformat(sep=" ") for _ in range(row_count)]
+        values = _generate_timestamps(row_count, as_of, _column_time_profile(column, time_profile))
 
     # -----------------------------------------------------
     # Untyped / generic string columns: honour a type that names
@@ -674,11 +816,7 @@ def generate_column_values(
     # Nullability
     # -----------------------------------------------------
     if not force_not_null and "not null" not in column.settings and "pk" not in column.settings:
-        null_fraction = max(0, min(0.2, 1 - (row_count / (row_count + 50))))
-        sample_size = int(row_count * null_fraction)
-        if sample_size:
-            for idx in random.sample(range(row_count), k=sample_size):
-                values[idx] = column.default
+        _null_out(values, _null_fraction_for(column, row_count), column.default, row_count)
 
     return values
 
@@ -686,6 +824,33 @@ def generate_column_values(
 # ---------------------------------------------------------
 # Internal helpers
 # ---------------------------------------------------------
+def _null_fraction_for(column: ColumnDef, row_count: int) -> float:
+    """The fraction of `row_count` rows this column should turn null.
+
+    A `null_rate` hint replaces the default outright; with none, the same
+    formula that has applied since before hints existed -- up to a fifth of
+    the rows, tapering off for very small tables so a 3-row lookup table
+    doesn't lose a third of itself to nulls.
+    """
+    note_null_rate = column.note.get("null_rate") if column.note else None
+    if note_null_rate is not None:
+        return note_null_rate
+    return max(0, min(0.2, 1 - (row_count / (row_count + 50))))
+
+
+def _null_out(values: list, fraction: float, default: object, row_count: int) -> None:
+    """Overwrite `fraction` of `values`, in place, with `default`.
+
+    A fixed count drawn once via `random.sample` rather than a per-row coin
+    flip, so the null count is exactly `round(row_count * fraction)` instead
+    of only approximately so.
+    """
+    sample_size = int(row_count * fraction)
+    if sample_size:
+        for idx in random.sample(range(row_count), k=sample_size):
+            values[idx] = default
+
+
 def _deduplicate(
     values: list,
     generator: Callable[[], object],