trendify2 2.0.4__py3-none-any.whl

trendify/generator/xy_data_plotter.py

@@ -0,0 +1,195 @@
+"""
+Draws `Point2D`/`Trace2D`/`AxLine` records onto a matplotlib figure. Fed directly by
+`RecordStore` query results, so there's no directory-loading state to manage.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+from matplotlib.axes import Axes
+from matplotlib.collections import LineCollection
+from matplotlib.colors import to_rgba
+from matplotlib.lines import Line2D
+from matplotlib.transforms import Transform
+from pydantic import BaseModel
+
+from trendify.base.helpers import Tag
+from trendify.base.pen import Pen
+from trendify.plotting.axline import AxLine, LineOrientation
+from trendify.plotting.figure import SingleAxisFigure
+from trendify.plotting.point import Point2D
+from trendify.plotting.scatter import Scatter2D
+from trendify.plotting.trace import Trace2D
+
+__all__ = ["XYDataPlotter"]
+
+logger = logging.getLogger(__name__)
+
+
+def _nan_separated(segments: list[np.ndarray]) -> np.ndarray:
+    """
+    Stacks `(N, 2)` vertex arrays into one array with a NaN row after each, so a single
+    `Line2D` can stand in for many separate polylines.
+    """
+    gap = np.full((1, 2), np.nan)
+    return np.concatenate([part for segment in segments for part in (segment, gap)])
+
+
+def _add_batched_lines(
+    ax: Axes,
+    segments: list[np.ndarray],
+    pens: list[Pen],
+    transform: Transform | None = None,
+) -> None:
+    """
+    Draws many lines as one `LineCollection` per z-order instead of one `Line2D` each. Every
+    artist costs matplotlib transform, clipping and draw work, which dominates when a tag
+    holds thousands of lines. Lines keep their given order within a z-order, so overlap is
+    unchanged. Each line keeps its own color, width, linestyle and alpha.
+
+    A collection is not a `Line2D`, so two things `Line2D`s used to provide are restored here:
+    - **Legend entries**: one empty, labeled `Line2D` per distinct label (the last pen with that
+      label wins, as when handles were de-duplicated by label).
+    - **Data limits and `loc="best"` legend placement**: both are computed from `Line2D`
+      geometry, and matplotlib ignores `LineCollection`s there. One invisible `Line2D` tracing
+      every segment carries the geometry, so the axes autoscale and the legend avoids the data
+      exactly as before. It is never drawn.
+
+    Args:
+        ax (Axes): axes to draw onto
+        segments (list[np.ndarray]): one `(N, 2)` array of vertices per line
+        pens (list[Pen]): style of each line, parallel to `segments`
+        transform (Transform | None): transform for the vertices; `None` means data coordinates
+
+    """
+    by_zorder: dict[float, list[int]] = {}
+    for i, pen in enumerate(pens):
+        by_zorder.setdefault(pen.zorder, []).append(i)
+    for zorder, indices in by_zorder.items():
+        collection = LineCollection(
+            [segments[i] for i in indices],
+            colors=[to_rgba(pens[i].color, pens[i].alpha) for i in indices],
+            linewidths=[pens[i].size for i in indices],
+            linestyles=[pens[i].linestyle for i in indices],
+            zorder=zorder,
+        )
+        if transform is not None:
+            collection.set_transform(transform)
+        ax.add_collection(collection, autolim=False)
+
+    for pen in {pen.label: pen for pen in pens if pen.label is not None}.values():
+        ax.plot([], [], **pen.as_scatter_plot_kwargs())
+
+    vertices = _nan_separated(segments)
+    stand_in = Line2D(vertices[:, 0], vertices[:, 1], visible=False, label="_nolegend_")
+    if transform is not None:
+        stand_in.set_transform(transform)
+    ax.add_line(stand_in)
+
+
+def _plot_traces(ax: Axes, traces: list[Trace2D]) -> None:
+    """
+    Plots traces, batching the plain ones (no marker, visible line) with `_add_batched_lines`.
+    Traces with markers or no line are drawn individually afterward, so at an equal z-order
+    they paint over the batched lines.
+    """
+    plain = [t for t in traces if t.marker is None and t.pen.linestyle is not None]
+    if plain:
+        _add_batched_lines(
+            ax, [np.column_stack([t.x, t.y]) for t in plain], [t.pen for t in plain]
+        )
+    for trace in traces:
+        if trace.marker is not None or trace.pen.linestyle is None:
+            trace.plot_to_ax(ax)
+
+
+def _plot_axlines(ax: Axes, axlines: list[AxLine]) -> None:
+    """
+    Plots axis lines, batching each orientation with `_add_batched_lines`. Vertices use the
+    same blended transform `Axes.axhline`/`axvline` use (axes fraction along the line, data
+    value across it), so the lines span the full axes and only the across-axis limit autoscales.
+    """
+    plain = [a for a in axlines if a.pen.linestyle is not None]
+    horizontal = [a for a in plain if a.orientation == LineOrientation.HORIZONTAL]
+    vertical = [a for a in plain if a.orientation == LineOrientation.VERTICAL]
+    if horizontal:
+        _add_batched_lines(
+            ax,
+            [np.array([[0.0, a.value], [1.0, a.value]]) for a in horizontal],
+            [a.pen for a in horizontal],
+            ax.get_yaxis_transform(which="grid"),
+        )
+    if vertical:
+        _add_batched_lines(
+            ax,
+            [np.array([[a.value, 0.0], [a.value, 1.0]]) for a in vertical],
+            [a.pen for a in vertical],
+            ax.get_xaxis_transform(which="grid"),
+        )
+    for axline in axlines:
+        if axline.pen.linestyle is None:
+            axline.plot_to_ax(ax)
+
+
+class XYDataPlotter(BaseModel):
+    """
+    Draws `Point2D`/`Scatter2D`/`Trace2D`/`AxLine` records sharing a tag onto a matplotlib
+    axes.
+    """
+
+    @classmethod
+    def handle_points_and_traces(
+        cls,
+        tag: Tag,
+        points: list[Point2D],
+        traces: list[Trace2D],
+        axlines: list[AxLine],
+        scatters: list[Scatter2D],
+        saf: SingleAxisFigure | None = None,
+    ) -> SingleAxisFigure:
+        """
+        Plots points (scattered, grouped by marker), scatters, traces, and axlines onto `saf`.
+
+        Args:
+            tag (Tag): tag these records belong to (used only if a new figure is created)
+            points (list[Point2D]): points to scatter, grouped by `Marker` (each distinct
+                marker becomes one `ax.scatter` call/series)
+            traces (list[Trace2D]): traces to plot
+            axlines (list[AxLine]): axis lines to plot
+            scatters (list[Scatter2D]): bulk scatter arrays to plot, each already carrying
+                its own single shared `Marker`
+            saf (SingleAxisFigure | None): figure to draw onto; a new one is created if `None`
+
+        Returns:
+            (SingleAxisFigure): the figure drawn onto
+
+        """
+        if saf is None:
+            saf = SingleAxisFigure.new(tag=tag)
+
+        logger.debug(
+            f"Plotting {len(points)} point(s), {len(traces)} trace(s), {len(axlines)} "
+            f"axline(s), {len(scatters)} scatter(s) for {tag = }"
+        )
+        if points:
+            markers = set(p.marker for p in points)
+            for marker in markers:
+                matching_points = [p for p in points if p.marker == marker]
+                x = [p.x for p in matching_points]
+                y = [p.y for p in matching_points]
+                if x and y:
+                    if marker is not None:
+                        saf.ax.scatter(x, y, **marker.as_scatter_plot_kwargs())
+                    else:
+                        saf.ax.scatter(x, y)
+
+        for scatter in scatters:
+            scatter.plot_to_ax(saf.ax)
+
+        _plot_traces(saf.ax, traces)
+        _plot_axlines(saf.ax, axlines)
+        saf.ax.autoscale_view()
+
+        return saf

trendify/log.py

@@ -0,0 +1,195 @@
+"""
+Trendify's logging setup: a Rich-formatted console handler plus a rotating log file
+(`setup_logger`), a multiprocessing-safe queue/listener pair for routing worker-process log
+records back through those same handlers (`create_queue_listener`/`worker_init`), and
+`set_log_level` for a Python caller who just wants to control verbosity.
+"""
+
+from __future__ import annotations
+
+import logging
+import multiprocessing
+from logging.handlers import QueueHandler, QueueListener, RotatingFileHandler
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.logging import RichHandler
+from rich.theme import Theme
+
+__all__ = [
+    "create_queue_listener",
+    "get_logger",
+    "set_log_level",
+    "setup_logger",
+    "worker_init",
+]
+
+# Library-standard courtesy: don't let logging print "No handlers could be found for logger
+# trendify.*" if the consuming application never configures logging at all. This only affects
+# the "nothing configured anywhere" case. An application that calls `logging.basicConfig()`
+# or `setup_logger()` still sees everything, since a `NullHandler` doesn't stop propagation.
+logging.getLogger("trendify").addHandler(logging.NullHandler())
+
+theme = Theme(
+    {
+        "logging.level.debug": "bold dodger_blue1",
+        "logging.level.info": "bold green",
+        "logging.level.warning": "bold yellow",
+        "logging.level.error": "bold red",
+        "path": "dim white",
+        "message": "white",
+    }
+)
+
+
+class TerminalFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Block if 'file_only' is True
+        return not getattr(record, "file_only", False)
+
+
+class FileFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Block if 'terminal_only' is True
+        return not getattr(record, "terminal_only", False)
+
+
+def get_logger(name: str) -> logging.Logger:
+    logger = logging.getLogger(name)
+
+    if not logger.hasHandlers():
+        logger.addHandler(logging.NullHandler())
+    return logger
+
+
+def setup_logger(
+    level=logging.INFO,
+    terminal: bool = True,
+    log_file: Path | str | None = Path("trendify.log"),
+    max_bytes: int = 10 * 1024 * 1024,  # 10MB
+    backup_count: int = 5,
+) -> logging.Logger:
+    """Shortcut to a sensible trendify logger using Rich for the terminal."""
+    logger = logging.getLogger()
+    logger.setLevel(level)
+
+    # avoid double logging by preventing logs from being sent to the root logger
+    logger.propagate = False
+
+    if not terminal and log_file is None:
+        return logger
+
+    # clear existing handlers
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    # --- RICH HANDLER FOR TERMINAL ---
+    if terminal:
+        console = Console(theme=theme)
+        rich_handler = RichHandler(
+            console=console,
+            show_time=True,
+            log_time_format="[%Y-%m-%d %H:%M:%S]",
+            show_path=True,
+            enable_link_path=False,
+            # highlighter=None,
+            markup=True,  # allows color in the log message itself currently disabled in case an array is logged. Square brackets will break logging.
+            rich_tracebacks=True,
+        )
+        rich_handler.addFilter(TerminalFilter())
+        logger.addHandler(rich_handler)
+
+    # --- PLAIN TEXT HANDLER ---
+    if log_file is not None:
+        log_file = Path(log_file)
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+
+        file_format = (
+            "%(asctime)s [%(levelname)s] %(pathname)s:%(lineno)d - %(message)s"
+        )
+        file_handler = RotatingFileHandler(
+            log_file, maxBytes=max_bytes, backupCount=backup_count
+        )
+        file_handler.setFormatter(logging.Formatter(file_format))
+        file_handler.addFilter(FileFilter())
+        logger.addHandler(file_handler)
+
+    return logger
+
+
+def create_queue_listener(
+    *handlers: logging.Handler,
+) -> tuple[multiprocessing.Queue[Any], QueueListener]:
+    """
+    Creates a multiprocessing-safe log queue plus a `QueueListener` that drains it in the
+    calling process and dispatches records to `handlers`.
+
+    This is what makes multiprocess logging safe, including on Windows: pair it with
+    `worker_init` in each worker process, so that no worker ever opens the real console/file
+    handlers directly. Only the listener, running in the parent process, does that. On Windows,
+    the default `spawn` start method means workers don't inherit the parent's configured
+    handlers at all; on POSIX with `fork`, workers *do* inherit them, which is its own hazard
+    (multiple processes writing to the same file concurrently). Routing everything through one
+    queue and one listener avoids both failure modes uniformly.
+
+    Caller owns the listener's lifecycle: call `.start()` before spinning up worker processes
+    and `.stop()` once they've finished (a `try`/`finally` around a `ProcessPoolExecutor` block
+    is the natural place, since `.stop()` blocks until every already-queued record has been
+    dispatched, so no log messages are lost when the pool shuts down).
+
+    Args:
+        *handlers (logging.Handler): the real handlers worker log records should ultimately
+            reach (e.g. the handlers already attached to the root logger by `setup_logger`)
+
+    Returns:
+        (tuple[multiprocessing.Queue, QueueListener]): the queue to hand to worker processes
+            (via `ProcessPoolExecutor`'s `initializer`/`initargs`, calling `worker_init` in
+            each) and the not-yet-started listener draining it
+
+    """
+    log_queue: multiprocessing.Queue[Any] = multiprocessing.Queue(-1)
+    listener = QueueListener(log_queue, *handlers, respect_handler_level=True)
+    return log_queue, listener
+
+
+def worker_init(queue: multiprocessing.Queue[Any], level: int) -> None:
+    """
+    Initializes a worker process's root logger to ship every record through `queue` instead of
+    touching a real handler directly. Pair with `create_queue_listener`, called once in the
+    parent process before workers start.
+
+    Args:
+        queue (multiprocessing.Queue): queue created by `create_queue_listener`
+        level (int): log level to apply to this worker's root logger
+
+    """
+    root = logging.getLogger()
+
+    # Clear any handlers inherited via fork (POSIX default) so this worker never writes
+    # directly to a file/console shared with the parent or sibling workers. Only the
+    # QueueHandler below is installed, so everything funnels through the parent's
+    # QueueListener.
+    root.handlers.clear()
+
+    handler = QueueHandler(queue)
+    root.addHandler(handler)
+
+    root.setLevel(level)
+
+
+def set_log_level(level: int | str, logger_name: str = "trendify") -> None:
+    """
+    Sets the log level for `logger_name` (defaults to the top-level `"trendify"` logger, i.e.
+    every `trendify.*` module logger, since none of them set an explicit level of their own
+    and so inherit whatever their nearest ancestor has set). This is the supported way for
+    a Python caller to control trendify's log verbosity without needing to call `setup_logger`
+    or otherwise touch logging internals directly.
+
+    Args:
+        level (int | str): a `logging` level (e.g. `logging.DEBUG`) or level name (e.g.
+            `"DEBUG"`)
+        logger_name (str): logger to set the level on; defaults to `"trendify"`
+
+    """
+    logging.getLogger(logger_name).setLevel(level)

trendify/pipeline.py

@@ -0,0 +1,136 @@
+"""
+Top-level pipeline API: generate -> render -> include files, backed by one `RecordStore`
+`.db` file per output directory. `TrendifyPipeline` is the single implementation both the
+`trendify` CLI (`trendify.cli`) and direct Python callers use. The CLI is a thin
+argument-parsing layer over this class, not a separate code path.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from trendify.base.record import RecordGenerator
+from trendify.generator.generate import generate_records
+from trendify.generator.render import render_assets
+from trendify.progress import ProgressCallback
+
+__all__ = ["TrendifyPipeline"]
+
+logger = logging.getLogger(__name__)
+
+
+class TrendifyPipeline(BaseModel):
+    """
+    Runs the trendify pipeline (generate -> render -> include files) against one output
+    directory, backed by a single `RecordStore` `.db` file.
+    """
+
+    output_dir: Path
+    """Directory the pipeline reads/writes under. Holds `trendify.db` (the `RecordStore`) and
+    an `assets/` subdirectory (rendered CSVs/figures)."""
+
+    n_procs: int = Field(default=1, ge=1)
+    """Number of worker processes used by `generate`."""
+
+    @property
+    def db_path(self) -> Path:
+        return self.output_dir / "trendify.db"
+
+    @property
+    def assets_dir(self) -> Path:
+        return self.output_dir / "assets"
+
+    def generate(
+        self,
+        record_generator: RecordGenerator,
+        data_dirs: list[Path],
+        on_progress: ProgressCallback | None = None,
+    ) -> int:
+        """
+        Maps `record_generator` over `data_dirs`, writing records directly to this
+        pipeline's `RecordStore`. See `trendify.generator.generate.generate_records`.
+
+        Args:
+            record_generator (RecordGenerator): callable mapped over `data_dirs`.
+            data_dirs (list[Path]): directories to map `record_generator` over.
+            on_progress (ProgressCallback | None): called once per directory finished, e.g.
+                to report status from a containerized deployment back to an end user.
+
+        Returns:
+            (int): total number of records written
+
+        """
+        logger.info(
+            f"Generating records for {len(data_dirs)} directory(ies) into {self.db_path} "
+            f"({self.n_procs} worker process{'es' if self.n_procs > 1 else ''})"
+        )
+        total = generate_records(
+            record_generator=record_generator,
+            data_dirs=data_dirs,
+            db_path=self.db_path,
+            n_procs=self.n_procs,
+            on_progress=on_progress,
+        )
+        logger.info(f"Generated {total} record(s)")
+        return total
+
+    def render(self, on_progress: ProgressCallback | None = None) -> None:
+        """
+        Renders CSV tables and matplotlib figures for every tag into `self.assets_dir`. See
+        `trendify.generator.render.render_assets`.
+
+        Args:
+            on_progress (ProgressCallback | None): called once per tag finished, e.g. to
+                report status from a containerized deployment back to an end user.
+
+        """
+        logger.info(
+            f"Rendering assets from '{self.db_path}' into '{self.assets_dir}' "
+            f"({self.n_procs} worker process{'es' if self.n_procs > 1 else ''})"
+        )
+        render_assets(
+            self.db_path,
+            self.assets_dir,
+            n_procs=self.n_procs,
+            on_progress=on_progress,
+        )
+        logger.info(f"Finished rendering assets into '{self.assets_dir}'")
+
+    def run(
+        self,
+        record_generator: RecordGenerator,
+        data_dirs: list[Path],
+        on_progress: ProgressCallback | None = None,
+    ) -> int:
+        """
+        Runs the full pipeline: generate, then render (unless all three asset kinds are
+        suppressed).
+
+        Args:
+            record_generator (RecordGenerator): callable mapped over `data_dirs`.
+            data_dirs (list[Path]): directories to map `record_generator` over.
+            on_progress (ProgressCallback | None): called once per unit of work finished in
+                *both* stages -- `ProgressEvent.stage` distinguishes "generate" from "render"
+                for a single handler covering the whole run.
+
+        Returns:
+            (int): total number of records written by `generate`
+
+        """
+        logger.info(
+            f"Running full pipeline (generate -> render) for '{self.output_dir}'"
+        )
+
+        total = self.generate(
+            record_generator=record_generator,
+            data_dirs=data_dirs,
+            on_progress=on_progress,
+        )
+
+        self.render(on_progress=on_progress)
+
+        logger.info(f"Finished full pipeline for '{self.output_dir}'")
+        return total

trendify/plotting/__init__.py

@@ -0,0 +1,48 @@
+"""Plottable record types (points, traces, scatters, axis lines, histogram entries) and figure wrappers."""
+
+from trendify.plotting import axline
+from trendify.plotting import figure
+from trendify.plotting import histogram
+from trendify.plotting import point
+from trendify.plotting import scatter
+from trendify.plotting import trace
+
+from trendify.plotting.axline import (
+    AxLine,
+    LineOrientation,
+)
+from trendify.plotting.figure import (
+    PlotlyFigure,
+    SingleAxisFigure,
+)
+from trendify.plotting.histogram import (
+    HistogramEntry,
+    HistogramStyle,
+)
+from trendify.plotting.point import (
+    Point2D,
+)
+from trendify.plotting.scatter import (
+    Scatter2D,
+)
+from trendify.plotting.trace import (
+    Trace2D,
+)
+
+__all__ = [
+    "AxLine",
+    "HistogramEntry",
+    "HistogramStyle",
+    "LineOrientation",
+    "PlotlyFigure",
+    "Point2D",
+    "Scatter2D",
+    "SingleAxisFigure",
+    "Trace2D",
+    "axline",
+    "figure",
+    "histogram",
+    "point",
+    "scatter",
+    "trace",
+]

trendify/plotting/axline.py

@@ -0,0 +1,122 @@
+"""
+`AxLine`: a horizontal or vertical reference line drawn on a plot, styled with a `Pen`.
+"""
+
+from __future__ import annotations
+
+import logging
+from enum import StrEnum
+
+from matplotlib.axes import Axes
+from pydantic import ConfigDict
+
+from trendify.base.pen import Pen
+from trendify.formats.format2d import PlottableData2D
+from trendify.plotting.figure import PlotlyFigure
+
+__all__ = ["AxLine", "LineOrientation"]
+
+logger = logging.getLogger(__name__)
+
+
+class LineOrientation(StrEnum):
+    """
+    Defines orientation for axis lines.
+    """
+
+    HORIZONTAL = "horizontal"
+    """Horizontal line"""
+
+    VERTICAL = "vertical"
+    """Vertical line"""
+
+
+class AxLine(PlottableData2D):
+    """
+    Defines a horizontal or vertical line to be drawn on a plot.
+    """
+
+    value: float
+    """Value at which to draw the line (x-value for vertical, y-value for horizontal)."""
+
+    orientation: LineOrientation
+    """Whether line should be horizontal or vertical."""
+
+    pen: Pen = Pen()
+    """Style and label information for drawing to matplotlib axes."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    def plot_to_ax(self, ax: Axes) -> None:
+        """
+        Plots line to matplotlib axes object.
+
+        Args:
+            ax (Axes): axes to which line should be plotted
+
+        """
+        match self.orientation:
+            case LineOrientation.HORIZONTAL:
+                ax.axhline(y=self.value, **self.pen.as_scatter_plot_kwargs())
+            case LineOrientation.VERTICAL:
+                ax.axvline(x=self.value, **self.pen.as_scatter_plot_kwargs())
+            case _:
+                logger.error(f"Unrecognized line orientation {self.orientation}")
+
+    def add_to_plotly(self, plotly_figure: PlotlyFigure) -> PlotlyFigure:
+        """Add axis line to plotly figure without showing it in the legend"""
+        match self.orientation:
+            case LineOrientation.HORIZONTAL:
+                plotly_figure.fig.add_hline(
+                    y=self.value,
+                    line=dict(
+                        color=self.pen.rgba if self.pen else None,
+                        width=self.pen.size if self.pen else None,
+                        dash=(
+                            self.pen._convert_linestyle_to_plotly()
+                            if self.pen
+                            else None
+                        ),
+                    ),
+                    showlegend=False,  # Do not show in the legend
+                    opacity=self.pen.alpha if self.pen else None,
+                )
+                plotly_figure.fig.add_annotation(
+                    x=1.0,
+                    y=self.value,
+                    xref="paper",  # Reference to the entire plotting area
+                    yref="y",
+                    text=self.pen.label,
+                    showarrow=False,  # Hide the arrow if not needed
+                    xanchor="left",
+                    yanchor="middle",  # Center the text horizontally with the vline
+                )
+            case LineOrientation.VERTICAL:
+                plotly_figure.fig.add_vline(
+                    x=self.value,
+                    line=dict(
+                        color=self.pen.rgba if self.pen else None,
+                        width=self.pen.size if self.pen else None,
+                        dash=(
+                            self.pen._convert_linestyle_to_plotly()
+                            if self.pen
+                            else None
+                        ),
+                    ),
+                    showlegend=False,  # Do not show in the legend
+                    opacity=self.pen.alpha if self.pen else None,
+                )
+                plotly_figure.fig.add_annotation(
+                    x=self.value,
+                    y=1.0,  # Position slightly above the plot area
+                    yref="paper",  # Reference to the entire plotting area
+                    text=self.pen.label,
+                    showarrow=False,  # Hide the arrow if not needed
+                    xanchor="center",  # Center the text horizontally with the vline
+                    yanchor="bottom",
+                )
+            case _:
+                logger.error(f"Unrecognized line orientation {self.orientation}")
+                return plotly_figure
+
+        return plotly_figure