trendify2 2.0.4__py3-none-any.whl

trendify/examples.py

@@ -0,0 +1,474 @@
+"""
+Sample data + a `RecordGenerator` used as a fixture for exercising the `trendify` pipeline end to end.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import cast
+
+import numpy as np
+import polars as pl
+
+import trendify
+
+__all__ = ["example_record_generator", "make_example_data"]
+
+
+# results.csv has a `time` column followed by `wave_1`, `wave_2`, ... counting upward. Only the
+# first N_PLOTTED_WAVES of those are plotted by `example_record_generator`.
+N_PLOTTED_WAVES = 3
+
+
+def make_example_data(workdir: Path, n_folders: int = 10, n_cols: int = 3) -> None:
+    """
+    Makes a highly diverse, varied sample dataset featuring distinct waveform signatures.
+
+    Args:
+        workdir (Path): Directory in which the sample data is to be generated
+        n_folders (int): Number of sample data files to generate (in separate subfolders).
+        n_cols (int): Number of data columns in each `results.csv` (not counting `time`), named
+            `wave_1`, `wave_2`, ... counting upward. The first three are the waveforms
+            `example_record_generator` plots. Any beyond that (`wave_4` onward) are unused
+            filler that exist only to make reading each file more expensive.
+
+    """
+    if n_cols < N_PLOTTED_WAVES:
+        raise ValueError(f"n_cols must be at least {N_PLOTTED_WAVES}, got {n_cols}")
+    models_dir = workdir.joinpath("models")
+    models_dir.mkdir(parents=True, exist_ok=True)
+
+    if not (models_dir / ".gitignore").exists():
+        (models_dir / ".gitignore").write_text("*")
+
+    for n in range(n_folders):
+        subdir = models_dir.joinpath(str(n))
+        subdir.mkdir(exist_ok=True, parents=True)
+
+        rng = np.random.default_rng(seed=n)
+
+        n_samples = rng.integers(
+            low=60, high=80
+        )  # Bumped slightly to resolve sharp shapes beautifully
+        t = np.linspace(0, 1, n_samples)
+        amplitudes = rng.uniform(low=0.8, high=1.5, size=3)
+        noise_level = 0.03
+
+        # --- 1. WAVE_1: Damped Harmonic Oscillation ---
+        decay_rate = rng.uniform(1.5, 3.5)
+        freq = rng.uniform(3.0, 5.0)
+        wave_1 = amplitudes[0] * np.sin(2 * np.pi * freq * t) * np.exp(
+            -decay_rate * t
+        ) + noise_level * rng.normal(size=len(t))
+
+        # --- 2. WAVE_2: Sharp Step / Discontinuity ---
+        step_time = rng.uniform(0.3, 0.6)
+        base_level = rng.uniform(-0.2, 0.2)
+        wave_2 = np.where(
+            t < step_time, base_level, base_level + amplitudes[1]
+        ) + noise_level * rng.normal(size=len(t))
+
+        # --- 3. WAVE_3: Logistic Saturation (S-Curve) ---
+        midpoint = rng.uniform(0.4, 0.6)
+        steepness = rng.uniform(10.0, 16.0)
+        wave_3 = amplitudes[2] / (
+            1.0 + np.exp(-steepness * (t - midpoint))
+        ) + noise_level * rng.normal(size=len(t))
+
+        # Collect metadata parameters uniquely for this run to write to stdin.csv
+        inputs: dict[str, int | float] = {
+            "n_samples": int(n_samples),
+            "w1_decay": float(decay_rate),
+            "w1_freq": float(freq),
+            "w2_step_time": float(step_time),
+            "w3_midpoint": float(midpoint),
+        }
+
+        pl.DataFrame(
+            {"key": list(inputs.keys()), "value": [str(v) for v in inputs.values()]}
+        ).write_csv(subdir.joinpath("stdin.csv"), include_header=False)
+
+        # Package channels cleanly matching your original structure
+        columns = {"time": t, "wave_1": wave_1, "wave_2": wave_2, "wave_3": wave_3}
+
+        # Filler columns are drawn last, so the waveforms above (and every file written with
+        # the default n_cols) are identical whether or not filler is requested.
+        n_extra = n_cols - N_PLOTTED_WAVES
+        if n_extra:
+            filler = rng.normal(size=(len(t), n_extra))
+            for j in range(n_extra):
+                columns[f"wave_{N_PLOTTED_WAVES + 1 + j}"] = filler[:, j]
+
+        pl.DataFrame(columns).write_csv(subdir.joinpath("results.csv"))
+
+
+def transform(data: np.ndarray, scale: trendify.AxisScale) -> np.ndarray:
+    if scale == trendify.AxisScale.LINEAR:
+        return data
+    elif scale == trendify.AxisScale.LOG:
+        return np.exp(data)  # ensures positivity and preserves shape
+    else:
+        raise ValueError(f"Unsupported scale: {scale}")
+
+
+def example_record_generator(workdir: Path) -> trendify.RecordList:
+    """
+    Processes the generated sample data in given workdir returning several types of records.
+
+    Args:
+        workdir (Path): Directory containing sample data.
+
+    """
+    records = []
+
+    df = pl.read_csv(workdir.joinpath("results.csv"))
+    time = df["time"].to_numpy()
+    # Only the first N_PLOTTED_WAVES columns, not "every column but time": results.csv may carry
+    # unused filler (see `make_example_data`'s `n_cols`) that must not be plotted or tabulated.
+    # The whole file is still read above, so filler still costs read time.
+    value_columns = [f"wave_{i}" for i in range(1, N_PLOTTED_WAVES + 1)]
+
+    colors = ["#FF0000", "#000B81", "#FFAA00"]
+    alphas = [1.0, 0.3, 1.0]
+    linestyles = ["-", ":", (0, (3, 1, 1, 1))]
+
+    run_num = workdir.name
+
+    trendify.Format2D(
+        tags=[("an_xy_plot", "trace_plot")],
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+        scale_x=trendify.AxisScale.LINEAR,
+        scale_y=trendify.AxisScale.LINEAR,
+    ).append_to_list(records)
+    traces = [
+        trendify.Trace2D(
+            x=time,
+            y=df[col].to_numpy(),
+            tags=[("an_xy_plot", "trace_plot")],
+            pen=trendify.Pen(
+                label=col,
+                color=colors[i],
+                linestyle=linestyles[i % len(linestyles)],
+                alpha=alphas[i],
+            ),
+        )
+        .append_to_list(records)
+        .set_metadata({"run_num": run_num})
+        for i, col in enumerate(value_columns)
+    ]
+
+    trendify.Format2D(
+        tags=[("an_xy_plot", "another_trace_plot")],
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+        scale_x=trendify.AxisScale.LINEAR,
+        scale_y=trendify.AxisScale.LINEAR,
+    ).append_to_list(records)
+    traces = [
+        trendify.Trace2D(
+            x=time,
+            y=df[col].to_numpy(),
+            tags=[("an_xy_plot", "another_trace_plot")],
+            pen=trendify.Pen(
+                label=col,
+                color=colors[i],
+                linestyle=linestyles[i % len(linestyles)],
+                alpha=alphas[i],
+            ),
+        )
+        .append_to_list(records)
+        .set_metadata({"run_num": run_num})
+        for i, col in enumerate(value_columns)
+    ]
+
+    trendify.Format2D(
+        tags=[("another_xy_plot", "trace_plot")],
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+        scale_x=trendify.AxisScale.LINEAR,
+        scale_y=trendify.AxisScale.LINEAR,
+        legend=trendify.Legend(loc=trendify.LegendLocation.LOWER_CENTER),
+        figure_width=8,
+        figure_height=4,
+    ).append_to_list(records)
+    traces = [
+        trendify.Trace2D(
+            x=time,
+            y=df[col].to_numpy(),
+            tags=[("another_xy_plot", "trace_plot")],
+            pen=trendify.Pen(
+                label=col,
+                color=colors[i],
+                linestyle=linestyles[i % len(linestyles)],
+                alpha=alphas[i],
+            ),
+        )
+        .append_to_list(records)
+        .set_metadata({"run_num": run_num})
+        for i, col in enumerate(value_columns)
+    ]
+    trendify.AxLine(
+        tags=[("another_xy_plot", "trace_plot")],
+        value=0.5,
+        orientation=trendify.LineOrientation.VERTICAL,
+        pen=trendify.Pen(zorder=11, color="k"),
+    ).append_to_list(records)
+    trendify.AxLine(
+        tags=[("another_xy_plot", "trace_plot")],
+        value=0.45,
+        orientation=trendify.LineOrientation.VERTICAL,
+        pen=trendify.Pen(zorder=9, color="r"),
+    ).append_to_list(records)
+
+    trendify.Format2D(
+        tags=["trace_plot_log_y"],
+        legend=trendify.Legend(
+            title="example",
+            loc=trendify.LegendLocation.CENTER_RIGHT,
+            framealpha=0,
+        ),
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+        scale_x=trendify.AxisScale.LINEAR,
+        scale_y=trendify.AxisScale.LOG,
+    ).append_to_list(records)
+    traces = [
+        trendify.Trace2D(
+            x=time,
+            y=transform(df[col].to_numpy(), trendify.AxisScale.LOG),
+            tags=["trace_plot_log_y"],
+            pen=trendify.Pen(
+                label=col,
+                color=colors[i],
+                linestyle=linestyles[i % len(linestyles)],
+                alpha=alphas[i],
+            ),
+        )
+        .append_to_list(records)
+        .set_metadata({"run_num": run_num})
+        for i, col in enumerate(value_columns)
+    ]
+
+    trendify.Format2D(
+        tags=["trace_plot_log_xy"],
+        legend=trendify.Legend(
+            fancybox=False,
+            loc=trendify.LegendLocation.CENTER,
+            edgecolor="red",
+            framealpha=1,
+        ),
+        lim_y=(0.1, 10),
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+        scale_x=trendify.AxisScale.LOG,
+        scale_y=trendify.AxisScale.LOG,
+    ).append_to_list(records)
+    traces = [
+        trendify.Trace2D(
+            x=transform(time, trendify.AxisScale.LOG),
+            y=transform(df[col].to_numpy(), trendify.AxisScale.LOG),
+            tags=["trace_plot_log_xy"],
+            pen=trendify.Pen(
+                label=col,
+                color=colors[i],
+                linestyle="-",
+                alpha=alphas[i],
+                zorder=[1, 1, 2][i],
+                size=5,
+            ),
+        )
+        .append_to_list(records)
+        .set_metadata({"run_num": run_num})
+        for i, col in enumerate(value_columns)
+    ]
+    trendify.AxLine(
+        tags=["trace_plot_log_xy"],
+        value=2.5,
+        orientation=trendify.LineOrientation.HORIZONTAL,
+        pen=trendify.Pen(
+            alpha=0.5, color="r", linestyle="-", label="test line", zorder=1
+        ),
+    ).append_to_list(records)
+
+    trendify.Format2D(
+        tags=["trace_plot_log_x"],
+        legend=trendify.Legend(visible=False),
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+        scale_x=trendify.AxisScale.LOG,
+        scale_y=trendify.AxisScale.LINEAR,
+    ).append_to_list(records)
+    traces = [
+        trendify.Trace2D(
+            x=transform(time, trendify.AxisScale.LOG),
+            y=df[col].to_numpy(),
+            tags=["trace_plot_log_x"],
+            pen=trendify.Pen(
+                label=col,
+                color=colors[i],
+                linestyle=linestyles[i % len(linestyles)],
+                alpha=alphas[i],
+            ),
+        )
+        .append_to_list(records)
+        .set_metadata({"run_num": run_num})
+        for i, col in enumerate(value_columns)
+    ]
+
+    trendify.Format2D(tags=["scatter_plot"], title_fig="N Points").append_to_list(
+        records
+    )
+    for i, trace in enumerate(traces):
+        trendify.Point2D(
+            x=workdir.name,
+            y=len(trace.x),
+            marker=trendify.Marker(
+                size=10,
+                label=trace.pen.label,
+                color=trace.pen.color,
+                alpha=alphas[i],
+            ),
+            tags=["scatter_plot"],
+        ).append_to_list(records).set_metadata({"run_num": run_num})
+
+    trendify.Format2D(
+        tags=[("nested_plots", "group_a", "deep_trace")],
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+    ).append_to_list(records)
+    trendify.Trace2D(
+        x=time,
+        y=df[value_columns[0]].to_numpy(),
+        tags=[("nested_plots", "group_a", "deep_trace")],
+        pen=trendify.Pen(label=value_columns[0], color=colors[0]),
+    ).append_to_list(records).set_metadata({"run_num": run_num})
+    trendify.Format2D(
+        tags=[("nested_plots", "group_b", "deep_trace")],
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+    ).append_to_list(records)
+    trendify.Trace2D(
+        x=time,
+        y=df[value_columns[-1]].to_numpy(),
+        tags=[("nested_plots", "group_b", "deep_trace")],
+        pen=trendify.Pen(label=value_columns[-1], color=colors[-1]),
+    ).append_to_list(records).set_metadata({"run_num": run_num})
+
+    trendify.Format2D(
+        tags=["histogram"],
+        title_ax="Idk lol",
+        title_fig="Idk lol2",
+        legend=trendify.Legend(
+            loc=trendify.LegendLocation.UPPER_LEFT,
+            bbox_to_anchor=(1.05, 1),
+        ),
+        label_x="Series value",
+        label_y="Counts",
+        grid=trendify.Grid.from_theme(trendify.GridTheme.MATLAB),
+    ).append_to_list(records)
+
+    for col in value_columns:
+        series = df[col]
+        trendify.TableEntry(
+            row=workdir.name,
+            col=col,
+            value=series.len(),
+            tags=[("tables", "lengths")],
+            unit=None,
+        ).append_to_list(records)
+        trendify.TableEntry(
+            row=workdir.name,
+            col=col,
+            value=cast(float, series.mean()),
+            tags=[("tables", "means")],
+            unit=None,
+        ).append_to_list(records)
+        trendify.TableEntry(
+            row=workdir.name,
+            col=col,
+            value=cast(float, series.std()),
+            tags=[("tables", "std_devs")],
+            unit=None,
+        ).append_to_list(records)
+        trendify.TableEntry(
+            row=workdir.name,
+            col=col,
+            value=cast(float, series.max()),
+            tags=[("extrema", "max")],
+            unit=None,
+        ).append_to_list(records)
+        trendify.TableEntry(
+            row=workdir.name,
+            col=col,
+            value=cast(float, series.min()),
+            tags=[("extrema", "min")],
+            unit=None,
+        ).append_to_list(records)
+
+        mean = cast(float, series.mean())
+        trendify.HistogramEntry(
+            tags=["histogram"],
+            value=mean,
+            style=trendify.HistogramStyle(
+                alpha_face=0.75,
+                alpha_edge=1,
+                bins=6,
+                label="A histogram entry",
+            ),
+        ).append_to_list(records)
+
+        trendify.AxLine(
+            tags=["histogram"],
+            value=mean,
+            orientation=trendify.LineOrientation.VERTICAL,
+            pen=trendify.Pen(color="r", label="mean", zorder=2),
+        ).append_to_list(records)
+
+    return records
+
+
+def make_sample_data() -> None:
+    """
+    Generates sample data to run the trendify code on
+    """
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        prog="make_sample_data_for_trendify",
+    )
+    parser.add_argument(
+        "-wd",
+        "--working-directory",
+        required=True,
+        help="Directory to be created and filled with sample data from a batch run",
+    )
+    parser.add_argument(
+        "-n",
+        "--number-of-data-sets",
+        type=int,
+        default=5,
+        help="Number of sample data sets to generate",
+    )
+    args = parser.parse_args()
+    make_example_data(
+        workdir=Path(args.working_directory),
+        n_folders=args.number_of_data_sets,
+    )
+
+
+def _main() -> None:
+    """
+    Makes sample data and runs the generate pipeline against it.
+    """
+    here = Path(__file__).parent
+    workdir = here.joinpath("workdir")
+
+    make_example_data(workdir=workdir, n_folders=100)
+
+    process_dirs = list(workdir.joinpath("models").glob("*/"))
+    db_path = workdir.joinpath("trendify.db")
+    n_procs = 30
+
+    trendify.generate_records(
+        record_generator=example_record_generator,
+        data_dirs=process_dirs,
+        db_path=db_path,
+        n_procs=n_procs,
+    )
+
+
+if __name__ == "__main__":
+    _main()

trendify/formats/__init__.py

@@ -0,0 +1,28 @@
+"""Plot formatting (`Format2D`) and table entry records."""
+
+from trendify.formats import format2d
+from trendify.formats import table
+
+from trendify.formats.format2d import (
+    AxisScale,
+    Format2D,
+    PlottableData2D,
+    Rastered,
+    Vector,
+    XYData,
+)
+from trendify.formats.table import (
+    TableEntry,
+)
+
+__all__ = [
+    "AxisScale",
+    "Format2D",
+    "PlottableData2D",
+    "Rastered",
+    "TableEntry",
+    "Vector",
+    "XYData",
+    "format2d",
+    "table",
+]

trendify/formats/format2d.py

@@ -0,0 +1,128 @@
+"""
+`Format2D` (axis/legend/grid/scale/limit settings for a plot) and the `PlottableData2D`/
+`XYData` base classes that know how to draw themselves onto a Plotly figure.
+
+`Format2D` is a `Record` in its own right, written once per tag (like any other
+record) rather than embedded in every plotted record; see `RecordStore.write_run`'s
+upsert-by-tag handling for it, and `render.py`'s single per-tag lookup.
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from typing import TYPE_CHECKING, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from trendify.base.record import Record
+from trendify.styling.grid import Grid
+from trendify.styling.legend import Legend
+
+if TYPE_CHECKING:
+    from trendify.plotting.figure import PlotlyFigure
+logger = logging.getLogger(__name__)
+
+__all__ = ["AxisScale", "Format2D", "PlottableData2D", "Rastered", "Vector", "XYData"]
+
+
+class AxisScale(StrEnum):
+    LINEAR = "linear"
+    """Format axis as linear"""
+    LOG = "log"
+    """Format axis with log base 10"""
+
+
+class Rastered(BaseModel):
+    """Renders to a raster image format (matplotlib's `dpi` setting applies)."""
+
+    type: Literal["rastered"] = "rastered"
+    """Discriminator identifying this renderer variant."""
+
+    filetype: str = ".jpg"
+    """File extension used when saving the rendered image."""
+
+    dpi: int = 500
+    """Resolution (dots per inch) used when saving the rendered image."""
+
+
+class Vector(BaseModel):
+    """Renders to a vector image format (matplotlib's `dpi` setting doesn't apply)."""
+
+    type: Literal["vector"] = "vector"
+    """Discriminator identifying this renderer variant."""
+
+    filetype: str = ".svg"
+    """File extension used when saving the rendered image."""
+
+
+class Format2D(Record):
+    """
+    Formatting data for matplotlib figure and axes. Written once per tag.
+    """
+
+    title_fig: str | None = None
+    """Sets the [figure title][matplotlib.figure.Figure.suptitle]."""
+
+    legend: Legend | None = Legend()
+    """Sets the [legend style][trendify.styling.legend.Legend]."""
+
+    title_ax: str | None = None
+    """Sets the [axis title][matplotlib.axes.Axes.set_title]."""
+
+    label_x: str | None = None
+    """Sets the [x-axis label][matplotlib.axes.Axes.set_xlabel]."""
+
+    label_y: str | None = None
+    """Sets the [y-axis label][matplotlib.axes.Axes.set_ylabel]."""
+
+    lim_x: tuple[float | None, float | None] = (None, None)
+    """x-axis (lower, upper) bound. Either side `None` means autofit that side to whatever's
+    plotted."""
+
+    lim_y: tuple[float | None, float | None] = (None, None)
+    """y-axis (lower, upper) bound, same semantics as `lim_x`."""
+
+    grid: Grid | None = None
+    """Sets the [grid][matplotlib.pyplot.grid]."""
+
+    scale_x: AxisScale = AxisScale.LINEAR
+    """Sets the x axis scale to an option from [AxisScale][trendify.formats.format2d.AxisScale]."""
+
+    scale_y: AxisScale = AxisScale.LINEAR
+    """Sets the y axis scale to an option from [AxisScale][trendify.formats.format2d.AxisScale]."""
+
+    figure_width: float = 6.4
+    """Sets the width of the rendered figure in inches."""
+
+    figure_height: float = 4.8
+    """Sets the height of the rendered figure in inches."""
+
+    renderer: Rastered | Vector = Field(default_factory=Rastered, discriminator="type")
+    """Chooses the saved file format: `Rastered` for a raster image at a configurable `dpi`,
+    or `Vector` for a vector image."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PlottableData2D(Record, ABC):
+    """
+    Base class for children of Record to be plotted ax xy data on a 2D plot
+    """
+
+    @abstractmethod
+    def add_to_plotly(self, plotly_figure: PlotlyFigure) -> PlotlyFigure:
+        """
+        Add this record to a plotly figure
+
+        Args:
+            plotly_figure (PlotlyFigure): Plotly figure to add data to
+
+        """
+
+
+class XYData(PlottableData2D):
+    """
+    Base class for children of Record to be plotted ax xy data on a 2D plot
+    """

trendify/formats/table.py

@@ -0,0 +1,55 @@
+"""
+`TableEntry`: the one `Record` subclass the store gives real SQL columns instead of an
+opaque JSON payload, since its whole purpose (pivoting into a wide table, then computing
+per-column stats) is naturally SQL-shaped. See `trendify.generator.table_builder`.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from pydantic import ConfigDict
+
+from trendify.base.record import Record
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["TableEntry"]
+
+
+class TableEntry(Record):
+    """
+    Defines an entry to be collected into a table.
+
+    Collected table entries will be printed in three forms when possible: melted, pivot (when
+    possible), and stats (on pivot columns, when possible).
+    """
+
+    row: float | str
+    """Row label"""
+
+    col: float | str
+    """Column label"""
+
+    value: float | str | bool
+    """Value"""
+
+    unit: str | None = None
+    """Units for value"""
+
+    model_config = ConfigDict(extra="forbid")
+
+    def get_entry_dict(self) -> dict[str, float | str | bool | None]:
+        """
+        Returns a dictionary of entries to be used in creating a table.
+
+        Returns:
+            (dict[str, str | float]): Dictionary of entries to be used in creating a melted table
+
+        """
+        return {
+            "row": self.row,
+            "col": self.col,
+            "value": self.value,
+            "unit": self.unit,
+        }