trendify2 2.0.4__py3-none-any.whl

trendify/generator/__init__.py

@@ -0,0 +1,38 @@
+"""Pipeline stages: generating records from run directories, then rendering plots and tables."""
+
+from trendify.generator import generate
+from trendify.generator import histogrammer
+from trendify.generator import render
+from trendify.generator import table_builder
+from trendify.generator import xy_data_plotter
+
+from trendify.generator.generate import (
+    generate_records,
+    get_sorted_dirs,
+)
+from trendify.generator.histogrammer import (
+    Histogrammer,
+)
+from trendify.generator.render import (
+    render_assets,
+)
+from trendify.generator.table_builder import (
+    TableBuilder,
+)
+from trendify.generator.xy_data_plotter import (
+    XYDataPlotter,
+)
+
+__all__ = [
+    "Histogrammer",
+    "TableBuilder",
+    "XYDataPlotter",
+    "generate",
+    "generate_records",
+    "get_sorted_dirs",
+    "histogrammer",
+    "render",
+    "render_assets",
+    "table_builder",
+    "xy_data_plotter",
+]

trendify/generator/generate.py

@@ -0,0 +1,167 @@
+"""
+Generate pipeline: maps a user-supplied `RecordGenerator` over raw-data run directories and
+writes the resulting records straight to a `RecordStore`-backed `.db` file.
+
+There is no separate "sort" step: tags are indexed at write time, so retrieving records by
+tag is an indexed query against `RecordStore`, not a physical grouping step that has to run
+before rendering can start.
+
+Worker processes only ever *compute* records; exactly one connection (in this process) ever
+writes. SQLite allows a single writer at a time no matter how many connections are open, so
+having every worker process open its own writing connection (the earlier design) doesn't buy
+any write throughput; it only adds lock contention, which gets worse, not better, as
+`n_procs` goes up. Splitting the pool into N compute workers feeding one writer lets
+`n_procs` control the part that actually parallelizes (running `record_generator`) without
+touching the part that can't (writing to the shared `.db` file).
+"""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+from trendify.base.record import RecordGenerator, RecordList
+from trendify.log import create_queue_listener
+from trendify.log import worker_init as _init_worker_logging
+from trendify.progress import ProgressCallback, ProgressEvent
+from trendify.store.record_store import RecordStore
+
+__all__ = ["generate_records", "get_sorted_dirs"]
+
+logger = logging.getLogger(__name__)
+
+# Per-process global set by `_init_worker`: the one thing a compute worker needs. Workers
+# never open a `RecordStore`; only the caller of `generate_records` writes.
+_worker_generator: RecordGenerator | None = None
+
+
+def get_sorted_dirs(dirs: list[Path]) -> list[Path]:
+    """
+    Sorts dirs numerically if possible, else alphabetically.
+
+    Args:
+        dirs (list[Path]): Directories to sort
+
+    Returns:
+        (list[Path]): Sorted list of directories
+
+    """
+    dirs = list(dirs)
+    try:
+        dirs.sort(key=lambda p: int(p.name))
+    except ValueError:
+        dirs.sort()
+    return dirs
+
+
+def _init_worker(record_generator: RecordGenerator) -> None:
+    global _worker_generator
+    _worker_generator = record_generator
+
+
+def _init_worker_with_logging(
+    record_generator: RecordGenerator,
+    log_queue: Any,
+    log_level: int,
+) -> None:
+    # Route this worker's logging through the parent's QueueListener before anything else
+    # runs, so no worker ever opens a file/console handler directly. See trendify.log for why
+    # that matters (Windows spawn semantics, and fork-inherited-handler races on POSIX).
+    _init_worker_logging(log_queue, log_level)
+    _init_worker(record_generator)
+
+
+def _compute(run_dir: Path) -> tuple[Path, RecordList]:
+    assert _worker_generator is not None
+    logger.info(f"Processing run_dir = '{run_dir}'")
+    return run_dir, _worker_generator(run_dir)
+
+
+def generate_records(
+    record_generator: RecordGenerator,
+    data_dirs: list[Path],
+    db_path: Path,
+    n_procs: int = 1,
+    on_progress: ProgressCallback | None = None,
+) -> int:
+    """
+    Maps `record_generator` over `data_dirs`, writing each directory's records to the
+    `RecordStore` at `db_path` (one `write_run` transaction per directory). There is no
+    intermediate file and no separate sort step, since tag-based retrieval from `RecordStore`
+    is an indexed query.
+
+    Args:
+        record_generator (RecordGenerator): A callable that returns a list of records
+            given a working directory. Must be picklable (e.g. a module-level function) when
+            `n_procs > 1`, since it is sent to worker processes.
+        data_dirs (list[Path]): Directories over which to map `record_generator`.
+        db_path (Path): Path to the trendify output directory's `.db` file. Created if it
+            doesn't already exist.
+        n_procs (int): Number of worker processes computing records in parallel.
+            `n_procs == 1` runs sequentially in this process (easier to debug with full
+            tracebacks). `n_procs > 1` uses a `ProcessPoolExecutor` for computing records
+            only; writing always happens through the single connection this function opens,
+            regardless of `n_procs`, since SQLite only ever allows one writer at a time.
+        on_progress (ProgressCallback | None): called once per run directory finished
+            (after its records are written), always from this process -- never a worker, so
+            it never needs to be picklable. Left to raise: a failing callback aborts
+            generation rather than being silently swallowed.
+
+    Returns:
+        (int): total number of records written across all directories.
+
+    """
+    sorted_dirs = get_sorted_dirs(data_dirs)
+    db_path = Path(db_path)
+    total_dirs = len(sorted_dirs)
+
+    def _report(run_dir: Path, completed: int) -> None:
+        if on_progress is not None:
+            on_progress(
+                ProgressEvent(
+                    stage="generate",
+                    completed=completed,
+                    total=total_dirs,
+                    detail=str(run_dir),
+                )
+            )
+
+    logger.info(f"Generating and writing Records for {total_dirs} run(s)...")
+    total = 0
+    with RecordStore.open(db_path) as store:
+        if n_procs > 1:
+            root_logger = logging.getLogger()
+            log_queue, listener = create_queue_listener(*root_logger.handlers)
+            listener.start()
+            try:
+                with ProcessPoolExecutor(
+                    max_workers=n_procs,
+                    initializer=_init_worker_with_logging,
+                    initargs=(record_generator, log_queue, root_logger.level),
+                ) as executor:
+                    futures = [
+                        executor.submit(_compute, run_dir) for run_dir in sorted_dirs
+                    ]
+                    for completed, future in enumerate(as_completed(futures), start=1):
+                        run_dir, records = future.result()
+                        total += store.write_run(run_dir, records)
+                        logger.info(f"Wrote records for run_dir = '{run_dir}'")
+                        _report(run_dir, completed)
+            finally:
+                # Blocks until every already-queued log record has been dispatched, so worker
+                # log output isn't lost or interleaved with what follows.
+                listener.stop()
+        else:
+            _init_worker(record_generator)
+            for completed, run_dir in enumerate(sorted_dirs, start=1):
+                _, records = _compute(run_dir)
+                total += store.write_run(run_dir, records)
+                logger.info(f"Wrote records for run_dir = '{run_dir}'")
+                _report(run_dir, completed)
+
+    logger.info(
+        f"Finished generating Records: {total} records written across {total_dirs} run(s)"
+    )
+    return total

trendify/generator/histogrammer.py

@@ -0,0 +1,62 @@
+"""
+Draws `HistogramEntry` records onto a matplotlib figure. Fed directly by `RecordStore` query
+results, so there's no directory-loading state to manage.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from pydantic import BaseModel
+
+from trendify.base.helpers import Tag
+from trendify.plotting.figure import SingleAxisFigure
+from trendify.plotting.histogram import HistogramEntry
+
+__all__ = ["Histogrammer"]
+
+logger = logging.getLogger(__name__)
+
+
+class Histogrammer(BaseModel):
+    """
+    Draws `HistogramEntry` records sharing a tag onto a matplotlib axes, grouped by style.
+    """
+
+    @classmethod
+    def handle_histogram_entries(
+        cls,
+        tag: Tag,
+        histogram_entries: list[HistogramEntry],
+        saf: SingleAxisFigure | None = None,
+    ) -> SingleAxisFigure:
+        """
+        Histograms the provided entries onto `saf` (creating a new figure if not given).
+
+        Args:
+            tag (Tag): tag these entries belong to (used only if a new figure is created)
+            histogram_entries (list[HistogramEntry]): entries to histogram, grouped by
+                `HistogramStyle` (each distinct style becomes one `ax.hist` call/series)
+            saf (SingleAxisFigure | None): figure to draw onto; a new one is created if `None`
+
+        Returns:
+            (SingleAxisFigure): the figure drawn onto
+
+        """
+        if saf is None:
+            saf = SingleAxisFigure.new(tag=tag)
+
+        histogram_styles = set(h.style for h in histogram_entries)
+        logger.debug(
+            f"Histogramming {len(histogram_entries)} entries into {len(histogram_styles)} "
+            f"style(s) for {tag = }"
+        )
+        for style in histogram_styles:
+            matching_entries = [e for e in histogram_entries if e.style == style]
+            values = [e.value for e in matching_entries]
+            if style is not None:
+                saf.ax.hist(values, **style.as_plot_kwargs())
+            else:
+                saf.ax.hist(values)
+
+        return saf

trendify/generator/render.py

@@ -0,0 +1,195 @@
+"""
+Static rendering pipeline: renders CSV tables and matplotlib figures directly from a
+`RecordStore`.
+
+Design note: every record type sharing a tag (`Point2D`/`Scatter2D`/`Trace2D`/`AxLine`/
+`HistogramEntry`) is drawn onto one shared `SingleAxisFigure` and saved once to `<tag>.jpg`,
+since histogram data isn't special-cased onto its own figure/file; a tag mixing record types
+renders the same way any other tag does.
+
+Rendering only ever reads, so worker processes each open their own read-only `RecordStore`
+connection and render disjoint tags in parallel with zero write-lock contention, since SQLite's
+WAL mode allows any number of concurrent readers, unlike `generate_records`, where only one
+process can ever write at a time.
+"""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+import matplotlib.pyplot as plt
+
+from trendify.base.helpers import Tag
+from trendify.formats.format2d import Format2D, Rastered
+from trendify.generator.histogrammer import Histogrammer
+from trendify.generator.table_builder import TableBuilder
+from trendify.generator.xy_data_plotter import XYDataPlotter
+from trendify.log import create_queue_listener
+from trendify.log import worker_init as _init_worker_logging
+from trendify.plotting.axline import AxLine
+from trendify.plotting.histogram import HistogramEntry
+from trendify.plotting.point import Point2D
+from trendify.plotting.scatter import Scatter2D
+from trendify.plotting.trace import Trace2D
+from trendify.progress import ProgressCallback, ProgressEvent
+from trendify.store.record_store import RecordStore
+from trendify.store.tags import tag_to_path_parts
+
+__all__ = ["render_assets"]
+
+logger = logging.getLogger(__name__)
+
+# Per-process global set by `_init_worker`: the one read-only connection a render worker needs.
+_worker_store: RecordStore | None = None
+
+
+def _init_worker(db_path: str) -> None:
+    global _worker_store
+    _worker_store = RecordStore.open(Path(db_path), readonly=True)
+
+
+def _init_worker_with_logging(
+    db_path: str,
+    log_queue: Any,
+    log_level: int,
+) -> None:
+    _init_worker_logging(log_queue, log_level)
+    _init_worker(db_path)
+
+
+def _render_tag(
+    tag: Tag,
+    output_dir: str,
+) -> Tag:
+    # Returns `tag` (not just None) so the parent process's as_completed loop knows which tag
+    # a given future was for, to report progress -- `future.result()` is the only thing that
+    # survives the process boundary back to the parent.
+    assert _worker_store is not None
+    _render_tag_assets(
+        _worker_store,
+        tag,
+        Path(output_dir),
+    )
+    return tag
+
+
+def _render_tag_assets(
+    store: RecordStore,
+    tag: Tag,
+    output_dir: Path,
+) -> None:
+    melted = store.get_table_entries(tag)
+    if melted.height > 0:
+        logger.info(f"Making tables for {tag = }")
+        TableBuilder.process_table_entries(tag=tag, melted=melted, out_dir=output_dir)
+        logger.info(f"Finished tables for {tag = }")
+
+    format2d_records = store.get_records_of_type(Format2D, tag=tag)
+    format2d = format2d_records[0] if format2d_records else None
+
+    points = store.get_records_of_type(Point2D, tag=tag)
+    traces = store.get_records_of_type(Trace2D, tag=tag)
+    scatters = store.get_records_of_type(Scatter2D, tag=tag)
+    axlines = store.get_records_of_type(AxLine, tag=tag)
+    histogram_entries = store.get_records_of_type(HistogramEntry, tag=tag)
+
+    if not (points or traces or scatters or axlines or histogram_entries):
+        return
+
+    # Points/traces/scatters/axlines and histogram entries all draw onto the same
+    # `SingleAxisFigure`/axes and get saved once, rather than forcing histogram data onto a
+    # separate figure: a tag mixing record types is treated the same as any other tag.
+    logger.info(f"Making plot for {tag = }")
+    saf = XYDataPlotter.handle_points_and_traces(
+        tag=tag,
+        points=points,
+        traces=traces,
+        axlines=axlines,
+        scatters=scatters,
+    )
+    if histogram_entries:
+        Histogrammer.handle_histogram_entries(
+            tag=tag, histogram_entries=histogram_entries, saf=saf
+        )
+    if format2d is not None:
+        saf.apply_format(format2d)
+
+    renderer = format2d.renderer if format2d is not None else Rastered()
+    save_path = output_dir.joinpath(*tag_to_path_parts(tag)).with_suffix(
+        renderer.filetype
+    )
+    save_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.info(f"Saving to '{save_path}'")
+    saf.savefig(save_path, dpi=renderer.dpi if isinstance(renderer, Rastered) else None)
+    plt.close(saf.fig)
+    logger.info(f"Finished plot for {tag = }")
+
+
+def render_assets(
+    db_path: Path,
+    output_dir: Path,
+    n_procs: int = 1,
+    on_progress: ProgressCallback | None = None,
+) -> None:
+    """
+    Renders CSV tables and matplotlib figures for every tag in the `RecordStore` at
+    `db_path`, writing them under `output_dir` (nested per tag).
+
+    Args:
+        db_path (Path): path to the trendify output directory's `.db` file
+        output_dir (Path): directory tables/figures are written under
+        n_procs (int): number of worker processes rendering tags in parallel. `n_procs == 1` runs sequentially in this process (easier to debug with full tracebacks). `n_procs > 1` uses a `ProcessPoolExecutor`, with one read-only `RecordStore` connection opened per worker, which is safe and contention-free since rendering never writes.
+        on_progress (ProgressCallback | None): called once per tag finished, always from this
+            process -- never a worker, so it never needs to be picklable. Left to raise: a
+            failing callback aborts rendering rather than being silently swallowed.
+
+    """
+    db_path = Path(db_path)
+    output_dir = Path(output_dir)
+
+    with RecordStore.open(db_path, readonly=True) as store:
+        tags = store.tag_tree()
+    total_tags = len(tags)
+
+    def _report(tag: Tag, completed: int) -> None:
+        if on_progress is not None:
+            on_progress(
+                ProgressEvent(
+                    stage="render",
+                    completed=completed,
+                    total=total_tags,
+                    detail=str(tag),
+                )
+            )
+
+    if n_procs > 1:
+        root_logger = logging.getLogger()
+        log_queue, listener = create_queue_listener(*root_logger.handlers)
+        listener.start()
+        try:
+            with ProcessPoolExecutor(
+                max_workers=n_procs,
+                initializer=_init_worker_with_logging,
+                initargs=(str(db_path), log_queue, root_logger.level),
+            ) as executor:
+                futures = [
+                    executor.submit(
+                        _render_tag,
+                        tag,
+                        str(output_dir),
+                    )
+                    for tag in tags
+                ]
+                for completed, future in enumerate(as_completed(futures), start=1):
+                    finished_tag = future.result()
+                    _report(finished_tag, completed)
+        finally:
+            listener.stop()
+    else:
+        with RecordStore.open(db_path, readonly=True) as store:
+            for completed, tag in enumerate(tags, start=1):
+                _render_tag_assets(store, tag, output_dir)
+                _report(tag, completed)

trendify/generator/table_builder.py

@@ -0,0 +1,194 @@
+"""
+Builds melted/pivot/stats CSV tables for `TableEntry` records, from `RecordStore` query
+results, using Polars. `RecordStore.get_table_entries` already hands back the melted
+row/col/value/unit shape as a `pl.DataFrame` via one indexed SQL query, so this module only
+needs to pivot and compute statistics, not reparse or reshape raw records.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+
+import polars as pl
+from pydantic import BaseModel
+
+from trendify.base.helpers import Tag
+from trendify.store.tags import tag_to_path_parts
+
+__all__ = ["TableBuilder"]
+
+logger = logging.getLogger(__name__)
+
+_DIGIT_RUN = re.compile(r"\d+")
+
+
+def _natural_sort_key(value: str) -> str:
+    """
+    Zero-pads every run of digits in `value` so a plain lexicographic sort orders numeric
+    substrings numerically (e.g. "row_2" before "row_10") instead of by leading digit
+    ("row_10" before "row_2"). Row labels that aren't numeric at all still compare as
+    ordinary strings.
+    """
+    return _DIGIT_RUN.sub(lambda m: m.group().zfill(20), value)
+
+
+class TableBuilder(BaseModel):
+    """
+    Builds tables (melted, pivot, and stats) from `TableEntry` records for a given tag.
+    """
+
+    @classmethod
+    def process_table_entries(
+        cls, tag: Tag, melted: pl.DataFrame, out_dir: Path
+    ) -> None:
+        """
+        Saves CSV files for the melted data frame, pivot dataframe, and pivot dataframe stats.
+
+        File names all use the tag with different suffixes: `<tag>_melted.csv`,
+        `<tag>_pivot.csv`, `<tag>_stats.csv`.
+
+        Args:
+            tag (Tag): record tag these entries belong to
+            melted (pl.DataFrame): row/col/value/unit table, as returned by
+                `RecordStore.get_table_entries`
+            out_dir (Path): directory under which the CSVs are saved (nested per tag)
+
+        """
+        if melted.height == 0:
+            return
+
+        melted = (
+            melted.with_columns(
+                _sort_key=pl.Series([_natural_sort_key(r) for r in melted["row"]])
+            )
+            .sort("_sort_key")
+            .drop("_sort_key")
+        )
+
+        *parents, stem = tag_to_path_parts(tag)
+        save_dir = out_dir.joinpath(*parents)
+        save_dir.mkdir(parents=True, exist_ok=True)
+        logger.info(f"Saving tables for {tag = } to '{save_dir}/{stem}_*.csv'")
+
+        _write_csv(melted, save_dir / f"{stem}_melted.csv")
+
+        pivot = cls.pivot_table(melted)
+        if pivot is not None:
+            _write_csv(pivot, save_dir / f"{stem}_pivot.csv")
+
+            try:
+                stats = cls.get_stats_table(pivot)
+                if stats is not None:
+                    stats.write_csv(save_dir / f"{stem}_stats.csv")
+                else:
+                    logger.debug(
+                        f"No stats table for {tag = }: no numeric pivoted columns"
+                    )
+            except Exception as e:
+                logger.error(
+                    f"Could not generate stats table for {tag = }. Error: {e!s}"
+                )
+        else:
+            logger.debug(
+                f"No pivot table for {tag = }: a (row, col) pair repeats in the melted table"
+            )
+
+    @classmethod
+    def pivot_table(cls, melted: pl.DataFrame) -> pl.DataFrame | None:
+        """
+        Attempts to pivot a melted row/col/value DataFrame into wide form.
+
+        Args:
+            melted (pl.DataFrame): DataFrame with `row`/`col`/`value` columns
+
+        Returns:
+            (pl.DataFrame | None): pivoted DataFrame, or `None` if the pivot fails because a
+                `(row, col)` pair repeats.
+
+        """
+        try:
+            return melted.pivot(on="col", index="row", values="value")
+        except pl.exceptions.ComputeError:
+            return None
+
+    @classmethod
+    def get_stats_table(cls, pivot: pl.DataFrame) -> pl.DataFrame | None:
+        """
+        Computes min/mean/max/3-sigma statistics for each pivoted column.
+
+        Non-numeric values are coerced to null rather than causing the whole column to be
+        skipped.
+
+        Args:
+            pivot (pl.DataFrame): pivoted table (as returned by `pivot_table`), with a `"row"`
+                index column plus one column per pivoted `col` value.
+
+        Returns:
+            (pl.DataFrame | None): stats table with a `"Name"` column (one row per pivoted
+                column) plus `min`/`mean`/`max`/`sigma3` columns, or `None` if there are no
+                value columns or every computed statistic is null.
+
+        """
+        value_columns = [c for c in pivot.columns if c != "row"]
+        if not value_columns:
+            return None
+
+        rows = []
+        any_stat = False
+        for col in value_columns:
+            numeric = _coerce_numeric(pivot[col])
+            mn, mean, mx, std = (
+                numeric.min(),
+                numeric.mean(),
+                numeric.max(),
+                numeric.std(),
+            )
+            if any(v is not None for v in (mn, mean, mx, std)):
+                any_stat = True
+            rows.append(
+                {
+                    "Name": col,
+                    "min": mn,
+                    "mean": mean,
+                    "max": mx,
+                    "sigma3": None if std is None else std * 3,
+                }
+            )
+        if not any_stat:
+            return None
+        return pl.DataFrame(rows)
+
+
+def _coerce_numeric(series: pl.Series) -> pl.Series:
+    """Best-effort element-wise cast to float, `None` for values that don't convert."""
+    if series.dtype != pl.Object:
+        return series.cast(pl.Float64, strict=False)
+
+    def _to_float(v) -> float | None:
+        if v is None:
+            return None
+        try:
+            return float(v)
+        except (TypeError, ValueError):
+            return None
+
+    return pl.Series(
+        series.name, [_to_float(v) for v in series.to_list()], dtype=pl.Float64
+    )
+
+
+def _write_csv(df: pl.DataFrame, path: Path) -> None:
+    """
+    Writes `df` to CSV, stringifying any `Object`-dtype columns first. Polars' CSV writer
+    doesn't support `Object` directly, and `TableEntry.value`'s heterogeneous float/str/bool
+    union is stored as `Object` by `RecordStore.get_table_entries`.
+    """
+    object_cols = [c for c, dtype in df.schema.items() if dtype == pl.Object]
+    if object_cols:
+        df = df.with_columns(
+            pl.Series(c, [None if v is None else str(v) for v in df[c].to_list()])
+            for c in object_cols
+        )
+    df.write_csv(path)