sqllocks-shape-simulation 0.9.1__py3-none-any.whl

shape_simulation/__init__.py

@@ -0,0 +1,157 @@
+"""Shape plugin: simulation scenarios (P6-04).
+
+Each simulator lives in its own module and is also importable from here, lazily (importing the
+package loads no simulator, and none of Arrow or NumPy):
+
+* ``file_drop``: :class:`FileDropSimulator`, an upstream source landing files over a date range;
+* ``scd2_file_drops``: :class:`SCD2FileDropSimulator`, a full load and daily versioned deltas;
+* ``stream_emit``: :class:`StreamEmitter`, tables as enveloped events on the emit runtime;
+* ``hybrid``: :class:`HybridSimulator`, a file drop and a stream of the same tables, linked;
+* ``state_machine``: :class:`WorkflowSimulator`, business-process events with dwell times;
+* ``clickstream_patterns``, ``financial_patterns``, ``iot_patterns``,
+  ``operational_log_patterns``, ``pulse_patterns``: the pattern simulators (sessions and funnels,
+  reversals and fraud bursts, sensor drift and alert storms, service logs and traces, rideshare
+  telemetry and marts), which take a configuration and, where they layer on existing data, Arrow
+  tables, and return Arrow tables and summary statistics;
+* ``cli``, ``cli_batch``, ``cli_patterns``: ``shape simulate``, the command that runs them.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import TYPE_CHECKING, Any
+
+SHAPE_API = "1.0"
+"""The plugin targets the plugin API major version this Shape provides."""
+
+# name -> module that defines it (one line per name; each simulator adds its own lines)
+_EXPORTS: dict[str, str] = {
+    "ClickstreamConfig": "shape_simulation.clickstream_patterns",
+    "ClickstreamResult": "shape_simulation.clickstream_patterns",
+    "ClickstreamSimulator": "shape_simulation.clickstream_patterns",
+    "FileDropConfig": "shape_simulation.file_drop",
+    "FileDropResult": "shape_simulation.file_drop",
+    "FileDropSimulator": "shape_simulation.file_drop",
+    "FinancialStreamConfig": "shape_simulation.financial_patterns",
+    "FinancialStreamResult": "shape_simulation.financial_patterns",
+    "FinancialStreamSimulator": "shape_simulation.financial_patterns",
+    "HybridConfig": "shape_simulation.hybrid",
+    "HybridResult": "shape_simulation.hybrid",
+    "HybridSimulator": "shape_simulation.hybrid",
+    "IoTTelemetryConfig": "shape_simulation.iot_patterns",
+    "IoTTelemetryResult": "shape_simulation.iot_patterns",
+    "IoTTelemetrySimulator": "shape_simulation.iot_patterns",
+    "OperationalLogConfig": "shape_simulation.operational_log_patterns",
+    "OperationalLogResult": "shape_simulation.operational_log_patterns",
+    "OperationalLogSimulator": "shape_simulation.operational_log_patterns",
+    "PulseDemandConfig": "shape_simulation.pulse_patterns",
+    "PulseDemandSimulator": "shape_simulation.pulse_patterns",
+    "PulseSimResult": "shape_simulation.pulse_patterns",
+    "SCD2FileDropConfig": "shape_simulation.scd2_file_drops",
+    "SCD2FileDropResult": "shape_simulation.scd2_file_drops",
+    "SCD2FileDropSimulator": "shape_simulation.scd2_file_drops",
+    "SimulateCommand": "shape_simulation.cli",
+    "StateDefinition": "shape_simulation.state_machine",
+    "StreamEmitConfig": "shape_simulation.stream_emit",
+    "StreamEmitResult": "shape_simulation.stream_emit",
+    "StreamEmitter": "shape_simulation.stream_emit",
+    "TransitionRule": "shape_simulation.state_machine",
+    "WorkflowConfig": "shape_simulation.state_machine",
+    "WorkflowResult": "shape_simulation.state_machine",
+    "WorkflowSimulator": "shape_simulation.state_machine",
+    "get_preset_workflow": "shape_simulation.state_machine",
+}
+
+__all__ = [
+    "ClickstreamConfig",
+    "ClickstreamResult",
+    "ClickstreamSimulator",
+    "FileDropConfig",
+    "FileDropResult",
+    "FileDropSimulator",
+    "FinancialStreamConfig",
+    "FinancialStreamResult",
+    "FinancialStreamSimulator",
+    "HybridConfig",
+    "HybridResult",
+    "HybridSimulator",
+    "IoTTelemetryConfig",
+    "IoTTelemetryResult",
+    "IoTTelemetrySimulator",
+    "OperationalLogConfig",
+    "OperationalLogResult",
+    "OperationalLogSimulator",
+    "PulseDemandConfig",
+    "PulseDemandSimulator",
+    "PulseSimResult",
+    "SCD2FileDropConfig",
+    "SCD2FileDropResult",
+    "SCD2FileDropSimulator",
+    "SimulateCommand",
+    "StateDefinition",
+    "StreamEmitConfig",
+    "StreamEmitResult",
+    "StreamEmitter",
+    "TransitionRule",
+    "WorkflowConfig",
+    "WorkflowResult",
+    "WorkflowSimulator",
+    "get_preset_workflow",
+]
+
+if TYPE_CHECKING:
+    from shape_simulation.cli import SimulateCommand
+    from shape_simulation.clickstream_patterns import (
+        ClickstreamConfig,
+        ClickstreamResult,
+        ClickstreamSimulator,
+    )
+    from shape_simulation.file_drop import FileDropConfig, FileDropResult, FileDropSimulator
+    from shape_simulation.financial_patterns import (
+        FinancialStreamConfig,
+        FinancialStreamResult,
+        FinancialStreamSimulator,
+    )
+    from shape_simulation.hybrid import HybridConfig, HybridResult, HybridSimulator
+    from shape_simulation.iot_patterns import (
+        IoTTelemetryConfig,
+        IoTTelemetryResult,
+        IoTTelemetrySimulator,
+    )
+    from shape_simulation.operational_log_patterns import (
+        OperationalLogConfig,
+        OperationalLogResult,
+        OperationalLogSimulator,
+    )
+    from shape_simulation.pulse_patterns import (
+        PulseDemandConfig,
+        PulseDemandSimulator,
+        PulseSimResult,
+    )
+    from shape_simulation.scd2_file_drops import (
+        SCD2FileDropConfig,
+        SCD2FileDropResult,
+        SCD2FileDropSimulator,
+    )
+    from shape_simulation.state_machine import (
+        StateDefinition,
+        TransitionRule,
+        WorkflowConfig,
+        WorkflowResult,
+        WorkflowSimulator,
+        get_preset_workflow,
+    )
+    from shape_simulation.stream_emit import StreamEmitConfig, StreamEmitResult, StreamEmitter
+
+
+def __getattr__(name: str) -> Any:
+    module = _EXPORTS.get(name)
+    if module is None:
+        raise AttributeError(f"module 'shape_simulation' has no attribute {name!r}")
+    value = getattr(importlib.import_module(module), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted([*globals(), *_EXPORTS])

shape_simulation/_patterns.py

@@ -0,0 +1,321 @@
+"""Shared helpers of the pattern simulators (clickstream, financial, IoT, operational log, pulse).
+
+The simulators work on Arrow tables (inputs and outputs) with numpy, so the plugin needs nothing
+beyond Shape's own dependencies. Everything random comes from one ``numpy`` generator seeded by
+the configuration, ids included: the same configuration and seed give the same tables.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, ClassVar
+
+import numpy as np
+import pyarrow as pa  # type: ignore[import-untyped]
+import pyarrow.compute as pc  # type: ignore[import-untyped]
+
+FORMATS = ("parquet", "csv", "jsonl")
+
+
+# ---- ids ------------------------------------------------------------------------------------
+
+
+def uuid_strings(rng: np.random.Generator, n: int) -> list[str]:
+    """``n`` version-4 style UUID strings drawn from ``rng`` (so a seed reproduces them)."""
+    if n <= 0:
+        return []
+    raw = rng.integers(0, 256, size=(n, 16), dtype=np.uint8)
+    raw[:, 6] = (raw[:, 6] & 0x0F) | 0x40
+    raw[:, 8] = (raw[:, 8] & 0x3F) | 0x80
+    hexa = raw.tobytes().hex()
+    return [
+        f"{h[0:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:32]}"
+        for h in (hexa[i * 32 : (i + 1) * 32] for i in range(n))
+    ]
+
+
+def hex_strings(rng: np.random.Generator, n: int, length: int) -> list[str]:
+    """``n`` random lowercase hex strings of ``length`` characters."""
+    if n <= 0:
+        return []
+    nbytes = (length + 1) // 2
+    raw = rng.integers(0, 256, size=(n, nbytes), dtype=np.uint8).tobytes().hex()
+    width = nbytes * 2
+    return [raw[i * width : i * width + length] for i in range(n)]
+
+
+# ---- weighted draws -------------------------------------------------------------------------
+
+
+def pick(
+    rng: np.random.Generator,
+    values: list[Any] | np.ndarray,
+    size: int,
+    p: list[float] | np.ndarray | None = None,
+) -> np.ndarray:
+    """``size`` draws from ``values`` (uniform, or with probabilities ``p``, normalised)."""
+    if p is not None:
+        weights = np.asarray(p, dtype=np.float64)
+        weights = weights / weights.sum()
+        return np.asarray(values)[rng.choice(len(values), size=size, p=weights)]
+    return np.asarray(values)[rng.integers(0, len(values), size=size)]
+
+
+# ---- Arrow in -------------------------------------------------------------------------------
+
+
+def as_table(obj: Any) -> pa.Table:
+    """An Arrow table from a table, a record batch, a mapping of columns or a pandas frame."""
+    if isinstance(obj, pa.Table):
+        return obj
+    if isinstance(obj, pa.RecordBatch):
+        return pa.Table.from_batches([obj])
+    if isinstance(obj, Mapping):
+        return pa.table(dict(obj))
+    to_arrow = getattr(obj, "to_arrow", None)
+    if callable(to_arrow):
+        out = to_arrow()
+        if isinstance(out, pa.Table):
+            return out
+    if hasattr(obj, "__arrow_c_stream__"):
+        return pa.table(obj)
+    if hasattr(obj, "columns") and hasattr(obj, "dtypes"):  # a pandas frame
+        return pa.Table.from_pandas(obj, preserve_index=False)
+    raise TypeError(f"cannot read {type(obj).__name__} as an Arrow table")
+
+
+def table_mapping(tables: Any) -> dict[str, pa.Table]:
+    """``tables`` (a mapping of tables, or a generation result with ``.tables``) as a dict."""
+    inner = getattr(tables, "tables", tables)
+    if not isinstance(inner, Mapping):
+        raise TypeError("tables must map table names to tables")
+    return {str(k): as_table(v) for k, v in inner.items()}
+
+
+def _is_number(value: Any) -> bool:
+    return not isinstance(value, bool) and isinstance(value, int | float | np.integer | np.floating)
+
+
+def check_settings(
+    config: Any,
+    *,
+    positive: tuple[str, ...] = (),
+    non_negative: tuple[str, ...] = (),
+    probabilities: tuple[str, ...] = (),
+    non_empty: tuple[str, ...] = (),
+) -> None:
+    """Refuse settings a simulator cannot use, with a ``ValueError`` that names the setting.
+
+    Every field whose default is a number must be a number (``--set users=abc`` gives text);
+    ``positive`` ones are above 0 (they divide), ``non_negative`` ones 0 or more,
+    ``probabilities`` in [0, 1] and ``non_empty`` lists hold at least one item. ``None`` is
+    accepted where it is the default (an unset optional)."""
+    import dataclasses
+
+    for f in dataclasses.fields(config):
+        default = f.default
+        if isinstance(default, bool) or not isinstance(default, int | float):
+            continue
+        value = getattr(config, f.name)
+        if not _is_number(value):
+            raise ValueError(f"{f.name} must be a number, got {value!r}")
+    for names, strict, high, what in (
+        (positive, True, None, "above 0"),
+        (non_negative, False, None, "0 or more"),
+        (probabilities, False, 1.0, "between 0 and 1"),
+    ):
+        for name in names:
+            value = getattr(config, name)
+            if value is None:
+                continue
+            if not _is_number(value):
+                raise ValueError(f"{name} must be a number, got {value!r}")
+            number = float(value)
+            # written so that NaN fails every rule
+            inside = number > 0 if strict else number >= 0
+            if high is not None:
+                inside = inside and number <= high
+            if not inside:
+                raise ValueError(f"{name} must be {what}, got {value!r}")
+    for name in non_empty:
+        if not getattr(config, name):
+            raise ValueError(f"{name} must hold at least one item")
+
+
+def timestamp_us(col: pa.ChunkedArray | pa.Array) -> tuple[np.ndarray, np.ndarray, str | None]:
+    """A date, timestamp or ISO-8601 string column as ``(microseconds, valid, timezone)``.
+
+    ``microseconds`` is int64 since the epoch (0 where null), ``valid`` marks the non-null
+    cells and ``timezone`` is the column's time zone, if it had one.
+    """
+    if isinstance(col, pa.ChunkedArray):
+        col = col.combine_chunks() if col.num_chunks else pa.array([], type=col.type)
+    tz: str | None = None
+    if pa.types.is_timestamp(col.type):
+        tz = col.type.tz
+        col = col.cast(pa.timestamp("us", tz))
+    elif pa.types.is_date(col.type):
+        col = col.cast(pa.timestamp("us"))
+    elif pa.types.is_string(col.type) or pa.types.is_large_string(col.type):
+        col = _text_times(col)
+    else:
+        raise TypeError(f"{col.type} is not a date or timestamp type")
+    valid = ~np.asarray(col.is_null().to_numpy(zero_copy_only=False), dtype=bool)
+    ints = col.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False)
+    return ints.astype(np.int64), valid, tz
+
+
+def _text_times(col: pa.Array) -> pa.Array:
+    """ISO-8601 text as naive ``timestamp[us]``: a time with ``Z`` or an offset becomes its UTC
+    wall time, one without is taken as it is. Text that is not a time raises ``ValueError``."""
+    try:
+        return pc.cast(col, pa.timestamp("us"))
+    except pa.ArrowInvalid:
+        pass
+    try:  # every time carries a zone
+        return pc.cast(col, pa.timestamp("us", "UTC")).cast(pa.timestamp("us"))
+    except pa.ArrowInvalid:
+        pass
+    values: list[datetime | None] = []
+    for text in col.to_pylist():  # some with a zone, some without
+        if text is None:
+            values.append(None)
+            continue
+        try:
+            value = datetime.fromisoformat(text)
+        except ValueError:
+            raise ValueError(f"{text!r} is not an ISO-8601 date or time") from None
+        if value.tzinfo is not None:
+            value = value.astimezone(UTC).replace(tzinfo=None)
+        values.append(value)
+    return pa.array(values, pa.timestamp("us"))
+
+
+def timestamps(us: np.ndarray, tz: str | None = None, valid: np.ndarray | None = None) -> pa.Array:
+    """A ``timestamp[us]`` array (``tz`` if given) from microseconds, null where not ``valid``."""
+    mask = None if valid is None else ~np.asarray(valid, dtype=bool)
+    return pa.array(
+        np.asarray(us, dtype=np.int64).astype("datetime64[us]"),
+        type=pa.timestamp("us", tz),
+        mask=mask,
+    )
+
+
+def parse_start(text: str) -> int:
+    """An ISO-8601 instant as microseconds since the epoch (a missing zone means UTC)."""
+    moment = datetime.fromisoformat(text)
+    if moment.tzinfo is None:
+        moment = moment.replace(tzinfo=UTC)
+    return int(moment.timestamp() * 1_000_000)
+
+
+def float_values(col: pa.ChunkedArray | pa.Array) -> tuple[np.ndarray, np.ndarray]:
+    """A numeric column as ``(float64 values, valid)``; null cells are NaN."""
+    arr = pc.cast(col, pa.float64())
+    if isinstance(arr, pa.ChunkedArray):
+        arr = arr.combine_chunks() if arr.num_chunks else pa.array([], type=pa.float64())
+    valid = ~np.asarray(arr.is_null().to_numpy(zero_copy_only=False), dtype=bool)
+    values = arr.fill_null(float("nan")).to_numpy(zero_copy_only=False).astype(np.float64)
+    return values, valid
+
+
+def float_array(values: np.ndarray) -> pa.Array:
+    """A float64 Arrow array from ``values``, null where the value is NaN."""
+    values = np.asarray(values, dtype=np.float64)
+    return pa.array(values, type=pa.float64(), mask=np.isnan(values))
+
+
+def empty_table(schema: pa.Schema) -> pa.Table:
+    return schema.empty_table()
+
+
+def combine(tables: list[pa.Table]) -> pa.Table:
+    """The rows of ``tables`` stacked, with the union of their columns (missing cells are null).
+
+    Where the same column has different types (decimal and float amounts, say) numeric
+    columns become float64 and anything else text.
+    """
+    names: list[str] = []
+    types: dict[str, pa.DataType] = {}
+    for table in tables:
+        for field in table.schema:
+            if field.name not in types:
+                names.append(field.name)
+                types[field.name] = field.type
+            elif types[field.name] != field.type:
+                a, b = types[field.name], field.type
+                numeric = all(
+                    pa.types.is_integer(t) or pa.types.is_floating(t) or pa.types.is_decimal(t)
+                    for t in (a, b)
+                )
+                types[field.name] = pa.float64() if numeric else pa.string()
+    schema = pa.schema([pa.field(n, types[n]) for n in names])
+    parts = []
+    for table in tables:
+        cols = []
+        for name in names:
+            if name in table.column_names:
+                cols.append(pc.cast(table.column(name), types[name]))
+            else:
+                cols.append(pa.nulls(table.num_rows, type=types[name]))
+        parts.append(pa.Table.from_arrays(cols, schema=schema))
+    return pa.concat_tables(parts)
+
+
+def value_counts(values: np.ndarray) -> dict[str, int]:
+    """``{value: count}`` ordered by count (largest first; ties by first appearance)."""
+    if len(values) == 0:
+        return {}
+    uniq, first, counts = np.unique(values, return_index=True, return_counts=True)
+    order = np.lexsort((first, -counts))
+    return {str(uniq[i]): int(counts[i]) for i in order}
+
+
+# ---- results --------------------------------------------------------------------------------
+
+
+class TablesResult:
+    """What every simulator's result offers: its tables, as flat events, and on disk."""
+
+    TABLES: ClassVar[tuple[str, ...]] = ()
+    stats: dict[str, Any]
+
+    def table_map(self) -> dict[str, pa.Table]:
+        """Every output table by name."""
+        return {name: getattr(self, name) for name in self.TABLES}
+
+    def events(self, table: str, *, seq_start: int = 0) -> pa.RecordBatch:
+        """The rows of ``table`` as flat stream events: its columns plus ``_shape_table``,
+        ``_shape_seq`` and (when it has a date or timestamp column) ``_shape_event_time``.
+        Feed them to ``shape.streaming.emit`` sinks or an emitter."""
+        from shape.streaming.emit.formats import with_event_fields
+
+        source = self.table_map().get(table)
+        if source is None:
+            raise KeyError(f"no table {table!r}; the result has {', '.join(self.table_map())}")
+        batch = source.combine_chunks().to_batches()
+        merged = batch[0] if batch else pa.RecordBatch.from_pylist([], schema=source.schema)
+        events: pa.RecordBatch = with_event_fields(merged, table, seq_start)
+        return events
+
+    def write(self, directory: str | Path, fmt: str = "parquet") -> dict[str, Path]:
+        """Write every table to ``directory`` as ``<table>.<fmt>`` (parquet, csv or jsonl)
+        and return the paths."""
+        if fmt not in FORMATS:
+            raise ValueError(f"unknown format {fmt!r}; choose from {', '.join(FORMATS)}")
+        from shape.builtins.sinks.files import CsvSink, JsonlSink, ParquetSink
+
+        sink: Any = {"parquet": ParquetSink, "csv": CsvSink, "jsonl": JsonlSink}[fmt]()
+        out = Path(directory)
+        out.mkdir(parents=True, exist_ok=True)
+        written: dict[str, Path] = {}
+        for name, table in self.table_map().items():
+            sink.write(str(out) + "/", name, table.to_batches(), schema=table.schema)
+            written[name] = out / f"{name}.{fmt}"
+        (out / "stats.json").write_text(
+            json.dumps(self.stats, indent=2, default=str) + "\n", encoding="utf-8", newline="\n"
+        )
+        return written

shape_simulation/_tables.py

@@ -0,0 +1,156 @@
+"""Arrow helpers for the file-drop, SCD2, stream and workflow simulators (P6-04a).
+
+The simulators take generated tables as Arrow tables: a ``dict[str, pa.Table]``, a generation
+result (its ``tables``), or pandas frames (converted when a frame is given; pandas stays
+optional). Files are written through the ``shape.sinks`` plugins, so a file drop writes the
+same Parquet, CSV and JSON Lines as ``shape generate``.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pyarrow as pa  # type: ignore[import-untyped]
+import pyarrow.compute as pc  # type: ignore[import-untyped]
+
+FILE_FORMATS = ("parquet", "csv", "jsonl")
+"""The formats a drop can write (the sink names of the same files)."""
+
+DATE_WORDS = ("date", "timestamp", "created", "updated")
+"""A column named with one of these is a candidate time column when no column is a timestamp."""
+
+CADENCES: dict[str, dt.timedelta] = {
+    "daily": dt.timedelta(days=1),
+    "hourly": dt.timedelta(hours=1),
+    "every_15m": dt.timedelta(minutes=15),
+}
+
+
+def as_tables(source: Any) -> dict[str, pa.Table]:
+    """``source`` as ``{name: Arrow table}``: a mapping of tables or frames, or an object with a
+    ``tables`` mapping (a generation result)."""
+    tables = (
+        source.tables if hasattr(source, "tables") and not isinstance(source, Mapping) else source
+    )
+    if not isinstance(tables, Mapping):
+        raise TypeError("expected a mapping of table name to table, or a generation result")
+    return {str(name): as_table(table) for name, table in tables.items()}
+
+
+def as_table(table: Any) -> pa.Table:
+    """One Arrow table from an Arrow table, record batch or pandas frame."""
+    if isinstance(table, pa.Table):
+        return table
+    if isinstance(table, pa.RecordBatch):
+        return pa.Table.from_batches([table])
+    if hasattr(table, "to_arrow") and callable(table.to_arrow):  # a generation table wrapper
+        return as_table(table.to_arrow())
+    if hasattr(table, "iloc"):  # a pandas frame
+        return pa.Table.from_pandas(table, preserve_index=False)
+    raise TypeError(f"cannot read {type(table).__name__} as a table")
+
+
+def check_format(fmt: str) -> str:
+    if fmt not in FILE_FORMATS:
+        raise ValueError(f"Unsupported file format: {fmt!r} (use one of {', '.join(FILE_FORMATS)})")
+    return fmt
+
+
+def check_cadence(cadence: str) -> dt.timedelta:
+    if cadence not in CADENCES:
+        raise ValueError(f"unknown cadence {cadence!r}; use one of {', '.join(CADENCES)}")
+    return CADENCES[cadence]
+
+
+def parse_day(text: str, name: str) -> dt.datetime:
+    """``YYYY-MM-DD`` as a datetime at midnight; the error names the setting."""
+    try:
+        return dt.datetime.strptime(text, "%Y-%m-%d")
+    except (TypeError, ValueError):
+        raise ValueError(f"{name} must be a date as YYYY-MM-DD, not {text!r}") from None
+
+
+def write_table(table: pa.Table, path: Path, fmt: str) -> None:
+    """Write ``table`` to ``path`` as ``fmt`` (parquet, csv or jsonl) with the ``shape.sinks``
+    plugin of that name; the folder is created."""
+    from shape.plugins.host import default_host
+
+    check_format(fmt)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    sink = default_host().get("shape.sinks", fmt)
+    sink.write(str(path), path.stem, iter(table.to_batches()), schema=table.schema)
+
+
+def is_temporal(t: pa.DataType) -> bool:
+    return bool(pa.types.is_timestamp(t) or pa.types.is_date(t))
+
+
+def to_timestamps(column: pa.ChunkedArray | pa.Array) -> pa.Array:
+    """``column`` as naive microsecond timestamps (UTC wall time for a zoned type); a value that
+    is not a date becomes null. Dates, timestamps and ISO-8601 strings are understood."""
+    col = column.combine_chunks() if isinstance(column, pa.ChunkedArray) else column
+    t = col.type
+    if pa.types.is_dictionary(t):
+        col = col.cast(t.value_type)
+        t = col.type
+    if pa.types.is_timestamp(t):
+        return col.cast(pa.timestamp("us"), safe=False)
+    if pa.types.is_date(t):
+        return col.cast(pa.timestamp("ms")).cast(pa.timestamp("us"))
+    if pa.types.is_string(t) or pa.types.is_large_string(t):
+        try:
+            return col.cast(pa.timestamp("us"))
+        except pa.ArrowInvalid:
+            values: list[dt.datetime | None] = []
+            for v in col.to_pylist():
+                try:
+                    values.append(None if v is None else dt.datetime.fromisoformat(v))
+                except ValueError:
+                    values.append(None)
+            return pa.array(values, pa.timestamp("us"))
+    return pa.nulls(len(col), pa.timestamp("us"))
+
+
+def temporal_column(table: pa.Table) -> str | None:
+    """The column that says when a row happened: the first timestamp column; else the first
+    column named like a date (``date``, ``timestamp``, ``created``, ``updated``) whose first
+    values are dates, ISO-8601 text or date types. A number is not a date."""
+    for f in table.schema:
+        if pa.types.is_timestamp(f.type):
+            return str(f.name)
+    for f in table.schema:
+        name = str(f.name).lower()
+        if not any(w in name for w in DATE_WORDS):
+            continue
+        t = f.type
+        if pa.types.is_date(t):
+            return str(f.name)
+        if pa.types.is_string(t) or pa.types.is_large_string(t):
+            head = table.column(f.name).drop_null().slice(0, 5)
+            if len(head) and to_timestamps(head).null_count == 0:
+                return str(f.name)
+    return None
+
+
+def window_mask(stamps: pa.Array, start: dt.datetime, end: dt.datetime) -> pa.Array:
+    """Rows with ``start <= time < end`` (a null time is outside every window)."""
+    lo = pa.scalar(start, pa.timestamp("us"))
+    hi = pa.scalar(end, pa.timestamp("us"))
+    return pc.fill_null(pc.and_(pc.greater_equal(stamps, lo), pc.less(stamps, hi)), False)
+
+
+def take_mask(table: pa.Table, mask: pa.Array | np.ndarray[Any, Any]) -> pa.Table:
+    """The rows where ``mask`` is true, in table order."""
+    return table.filter(mask if isinstance(mask, pa.Array) else pa.array(mask))
+
+
+def concat(tables: list[pa.Table]) -> pa.Table:
+    return pa.concat_tables(tables, promote_options="default")
+
+
+def iso_utc_now() -> str:
+    return dt.datetime.now(dt.UTC).isoformat()

shape_simulation/cli.py

@@ -0,0 +1,57 @@
+"""``shape simulate``: run a simulator on a domain's generated tables (P6-04).
+
+::
+
+    shape simulate file-drop retail --from 2024-01-01 --to 2024-01-31 -o landing/
+    shape simulate stream retail --table order --sink file -o events.jsonl
+    shape simulate workflow --preset order_fulfillment --entities 1000 -o out/
+
+Each simulator is a sub-command, registered by a handler module listed in ``HANDLER_MODULES``;
+a handler module has ``register(sub)``, which adds its parsers and gives each one a ``run``
+default (a function of the parsed arguments returning the exit code). The target of a simulator
+that works on tables is a domain name or a generation schema file; the tables are generated at
+``--scale`` and ``--seed`` and handed to the simulator.
+
+Nothing heavy loads at import time: the handlers import Arrow, the engine and the simulators
+when they run.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+SHAPE_API = "1.0"
+
+HANDLER_MODULES = [
+    "shape_simulation.cli_batch",
+    "shape_simulation.cli_patterns",
+]
+"""The modules that add simulators to ``shape simulate``, one line each."""
+
+
+class SimulateCommand:
+    """``shape simulate KIND ...``."""
+
+    name = "simulate"
+    help = (
+        "run a simulator: file drop, SCD2 drop, stream, hybrid, workflow, clickstream, financial, "
+        "iot, operational-log, pulse"
+    )
+
+    def configure(self, parser: Any) -> None:
+        sub = parser.add_subparsers(dest="simulate_cmd", required=True, metavar="KIND")
+        for module in HANDLER_MODULES:
+            importlib.import_module(module).register(sub)
+
+    def run(self, args: Any) -> int:
+        return int(args.run(args))
+
+
+def generate_tables(target: str, scale: str | None, seed: int | None) -> dict[str, Any]:
+    """The tables of a domain (or schema file) at ``scale`` and ``seed``, as Arrow tables."""
+    from shape.cli.generation import load_target
+    from shape.generation.engine import Engine
+
+    schema = load_target(target)
+    return dict(Engine(schema, scale=scale, seed=seed).generate().tables)