scitex 2.8.1__py3-none-any.whl → 2.10.0__py3-none-any.whl

scitex/fts/_bundle/_extractors/_extract_line.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-12-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/fts/_bundle/_extractors/_extract_line.py
+
+"""Line plot data and encoding extraction."""
+
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from matplotlib.axes import Axes
+
+    from ..._fig._dataclasses import TraceEncoding
+
+
+def extract_line_data(ax: "Axes", ax_idx: int) -> Dict[str, np.ndarray]:
+    """Extract line plot data from axes.
+
+    Args:
+        ax: Matplotlib axes
+        ax_idx: Axes index for column naming
+
+    Returns:
+        Dict mapping column names to data arrays
+    """
+    data = {}
+    for line_idx, line in enumerate(ax.get_lines()):
+        label = line.get_label()
+        if label is None or label.startswith("_"):
+            label = f"series_{line_idx}"
+
+        xdata, ydata = line.get_data()
+        if len(xdata) > 0:
+            x_col = f"ax{ax_idx}_line{line_idx}_x"
+            y_col = f"ax{ax_idx}_line{line_idx}_y"
+            data[x_col] = np.array(xdata, dtype=float)
+            data[y_col] = np.array(ydata, dtype=float)
+
+    return data
+
+
+def build_line_traces(ax: "Axes", ax_idx: int) -> List["TraceEncoding"]:
+    """Build encoding traces for line plots.
+
+    Args:
+        ax: Matplotlib axes
+        ax_idx: Axes index for trace ID
+
+    Returns:
+        List of TraceEncoding objects
+    """
+    from ..._fig._dataclasses import ChannelEncoding, TraceEncoding
+
+    traces = []
+    for line_idx, line in enumerate(ax.get_lines()):
+        label = line.get_label()
+        if label and not label.startswith("_"):
+            trace = TraceEncoding(
+                trace_id=f"line_{ax_idx}_{line_idx}",
+                x=ChannelEncoding(column=f"ax{ax_idx}_line{line_idx}_x"),
+                y=ChannelEncoding(column=f"ax{ax_idx}_line{line_idx}_y"),
+            )
+            traces.append(trace)
+
+    return traces
+
+
+__all__ = ["extract_line_data", "build_line_traces"]
+
+# EOF

scitex/fts/_bundle/_extractors/_extract_scatter.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-12-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/fts/_bundle/_extractors/_extract_scatter.py
+
+"""Scatter plot data and encoding extraction."""
+
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from matplotlib.axes import Axes
+
+    from ..._fig._dataclasses import TraceEncoding
+
+
+def extract_scatter_data(ax: "Axes", ax_idx: int) -> Dict[str, np.ndarray]:
+    """Extract scatter plot data from axes (PathCollection).
+
+    Args:
+        ax: Matplotlib axes
+        ax_idx: Axes index for column naming
+
+    Returns:
+        Dict mapping column names to data arrays
+    """
+    from matplotlib.collections import PathCollection
+
+    data = {}
+    scatter_idx = 0
+
+    for child in ax.get_children():
+        if isinstance(child, PathCollection):
+            offsets = child.get_offsets()
+            if len(offsets) > 0:
+                x_col = f"ax{ax_idx}_scatter{scatter_idx}_x"
+                y_col = f"ax{ax_idx}_scatter{scatter_idx}_y"
+                data[x_col] = np.array(offsets[:, 0], dtype=float)
+                data[y_col] = np.array(offsets[:, 1], dtype=float)
+                scatter_idx += 1
+
+    return data
+
+
+def build_scatter_traces(ax: "Axes", ax_idx: int) -> List["TraceEncoding"]:
+    """Build encoding traces for scatter plots.
+
+    Args:
+        ax: Matplotlib axes
+        ax_idx: Axes index for trace ID
+
+    Returns:
+        List of TraceEncoding objects
+    """
+    from matplotlib.collections import PathCollection
+
+    from ..._fig._dataclasses import ChannelEncoding, TraceEncoding
+
+    traces = []
+    scatter_idx = 0
+
+    for child in ax.get_children():
+        if isinstance(child, PathCollection):
+            offsets = child.get_offsets()
+            if len(offsets) > 0:
+                trace = TraceEncoding(
+                    trace_id=f"scatter_{ax_idx}_{scatter_idx}",
+                    x=ChannelEncoding(column=f"ax{ax_idx}_scatter{scatter_idx}_x"),
+                    y=ChannelEncoding(column=f"ax{ax_idx}_scatter{scatter_idx}_y"),
+                )
+                traces.append(trace)
+                scatter_idx += 1
+
+    return traces
+
+
+__all__ = ["extract_scatter_data", "build_scatter_traces"]
+
+# EOF

scitex/fts/_bundle/_loader.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-12-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/fts/_bundle/_loader.py
+
+"""FTS Bundle loading utilities.
+
+Loads bundles using the new canonical/artifacts/payload/children structure.
+Supports backwards compatibility with old flat structure (node.json at root).
+
+New structure:
+    canonical/spec.json     (was node.json)
+    canonical/encoding.json (was encoding.json)
+    canonical/theme.json    (was theme.json)
+    canonical/data_info.json (was data/data_info.json)
+    payload/stats.json      (was stats/stats.json)
+"""
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Tuple
+
+from ._storage import get_storage
+
+if TYPE_CHECKING:
+    from ._dataclasses import DataInfo, Node
+    from .._fig import Encoding, Theme
+    from .._stats import Stats
+
+
+def load_bundle_components(
+    path: Path,
+) -> Tuple[
+    Optional["Node"],
+    Optional["Encoding"],
+    Optional["Theme"],
+    Optional["Stats"],
+    Optional["DataInfo"],
+]:
+    """Load all bundle components from storage.
+
+    Supports both new canonical/ structure and legacy flat structure.
+
+    Args:
+        path: Bundle path (directory or ZIP)
+
+    Returns:
+        Tuple of (node, encoding, theme, stats, data_info)
+    """
+    from ._dataclasses import DataInfo, Node
+    from .._fig import Encoding, Theme
+    from .._stats import Stats
+
+    storage = get_storage(path)
+
+    node = None
+    encoding = None
+    theme = None
+    stats = None
+    data_info = None
+
+    # Detect structure: new (canonical/) or legacy (flat)
+    # - New: canonical/spec.json
+    # - Legacy FTS: node.json at root
+    # - Legacy sio.save(): spec.json at root
+    if storage.exists("canonical/spec.json"):
+        structure = "v2"  # New canonical/ structure
+    elif storage.exists("spec.json"):
+        structure = "sio"  # sio.save() structure
+    else:
+        structure = "v1"  # Legacy node.json structure
+    is_new_structure = structure == "v2"
+
+    # Node / spec.json
+    if structure == "v2":
+        node_data = storage.read_json("canonical/spec.json")
+    elif structure == "sio":
+        node_data = storage.read_json("spec.json")
+    else:
+        node_data = storage.read_json("node.json")
+    if node_data:
+        node = Node.from_dict(node_data)
+
+    # Encoding
+    if is_new_structure:
+        encoding_data = storage.read_json("canonical/encoding.json")
+    else:
+        encoding_data = storage.read_json("encoding.json")
+    if encoding_data:
+        encoding = Encoding.from_dict(encoding_data)
+
+    # Theme
+    if is_new_structure:
+        theme_data = storage.read_json("canonical/theme.json")
+    else:
+        theme_data = storage.read_json("theme.json")
+    if theme_data:
+        theme = Theme.from_dict(theme_data)
+
+    # Stats (payload for kind=stats, or legacy stats/)
+    if is_new_structure:
+        stats_data = storage.read_json("payload/stats.json")
+    else:
+        stats_data = storage.read_json("stats/stats.json")
+    if stats_data:
+        stats = Stats.from_dict(stats_data)
+
+    # Data info
+    if is_new_structure:
+        data_info_data = storage.read_json("canonical/data_info.json")
+    else:
+        data_info_data = storage.read_json("data/data_info.json")
+    if data_info_data:
+        data_info = DataInfo.from_dict(data_info_data)
+
+    return node, encoding, theme, stats, data_info
+
+
+def get_bundle_structure_version(path: Path) -> str:
+    """Detect bundle structure version.
+
+    Args:
+        path: Bundle path
+
+    Returns:
+        "v2" for new canonical/ structure, "v1" for legacy flat structure
+    """
+    storage = get_storage(path)
+    if storage.exists("canonical/spec.json"):
+        return "v2"
+    return "v1"
+
+
+__all__ = ["load_bundle_components", "get_bundle_structure_version"]
+
+# EOF

scitex/fts/_bundle/_mpl_helpers.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-12-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/fts/_bundle/_mpl_helpers.py
+
+"""Matplotlib helper functions for FTS bundle creation."""
+
+import warnings
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from matplotlib.figure import Figure as MplFigure
+
+    from .._fig import Encoding, Theme
+
+
+def _get_scitex_axes(fig: "MplFigure") -> Optional[Any]:
+    """Find scitex.plt wrapped axes with tracking data.
+
+    Uses the same helper as sio.save to find objects with export_as_csv.
+    """
+    try:
+        from scitex.io._save_modules._figure_utils import get_figure_with_data
+
+        return get_figure_with_data(fig)
+    except ImportError:
+        pass
+
+    # Fallback: check figure axes directly
+    axes_list = list(fig.axes) if hasattr(fig.axes, "__iter__") else [fig.axes]
+    for ax in axes_list:
+        if hasattr(ax, "export_as_csv") and hasattr(ax, "history"):
+            return ax
+    return None
+
+
+def _build_encoding_from_csv_columns(csv_df: "Any") -> "Encoding":
+    """Build encoding from actual CSV column names.
+
+    Handles two formats:
+    1. Verbose: ax-row-{row}-col-{col}_trace-id-{id}_variable-{var}
+    2. Simple: column names like 'x', 'y' (user-provided DataFrames)
+
+    This ensures encoding references match actual data columns.
+    """
+    from .._fig._dataclasses import ChannelEncoding, Encoding, TraceEncoding
+
+    if csv_df is None or csv_df.empty:
+        return Encoding(traces=[])
+
+    from scitex.plt.utils._csv_column_naming import parse_csv_column_name
+
+    # Group columns by trace (for verbose format)
+    trace_columns = {}  # {(ax_row, ax_col, trace_id): {variable: column_name}}
+
+    for col in csv_df.columns:
+        parsed = parse_csv_column_name(col)
+        if parsed["valid"]:
+            key = (parsed["ax_row"], parsed["ax_col"], parsed["trace_id"])
+            if key not in trace_columns:
+                trace_columns[key] = {}
+            trace_columns[key][parsed["variable"]] = col
+
+    # Build traces from verbose column names
+    traces = []
+    for (ax_row, ax_col, trace_id), variables in trace_columns.items():
+        trace = TraceEncoding(
+            trace_id=f"ax-row-{ax_row}-col-{ax_col}_trace-id-{trace_id}",
+            x=ChannelEncoding(column=variables.get("x")) if "x" in variables else None,
+            y=ChannelEncoding(column=variables.get("y")) if "y" in variables else None,
+        )
+        traces.append(trace)
+
+    # If no verbose columns found, try simple column names
+    if not traces:
+        columns = list(csv_df.columns)
+        # Check for common x/y patterns
+        x_col = None
+        y_col = None
+        for col in columns:
+            col_lower = col.lower()
+            if col_lower in ("x", "time", "index"):
+                x_col = col
+            elif col_lower in ("y", "value", "values"):
+                y_col = col
+        # If no pattern match, use first two numeric columns
+        if x_col is None or y_col is None:
+            numeric_cols = csv_df.select_dtypes(include=["number"]).columns.tolist()
+            if len(numeric_cols) >= 2:
+                if x_col is None:
+                    x_col = numeric_cols[0]
+                if y_col is None:
+                    y_col = numeric_cols[1] if numeric_cols[1] != x_col else numeric_cols[0]
+            elif len(numeric_cols) == 1:
+                if x_col is None:
+                    x_col = numeric_cols[0]
+                if y_col is None:
+                    y_col = numeric_cols[0]
+
+        if x_col and y_col:
+            trace = TraceEncoding(
+                trace_id="main",
+                x=ChannelEncoding(column=x_col),
+                y=ChannelEncoding(column=y_col),
+            )
+            traces.append(trace)
+
+    return Encoding(traces=traces)
+
+
+def validate_encoding_csv_link(encoding: "Encoding", csv_df: "Any") -> list:
+    """Validate that encoding column references exist in CSV data.
+
+    Args:
+        encoding: Encoding object with trace definitions
+        csv_df: DataFrame with CSV data
+
+    Returns:
+        List of validation errors (empty if valid)
+    """
+    errors = []
+
+    if csv_df is None or csv_df.empty:
+        return errors
+
+    csv_columns = set(csv_df.columns)
+
+    for trace in encoding.traces:
+        if trace.x and trace.x.column:
+            if trace.x.column not in csv_columns:
+                errors.append(
+                    f"Encoding references missing column: {trace.x.column}"
+                )
+        if trace.y and trace.y.column:
+            if trace.y.column not in csv_columns:
+                errors.append(
+                    f"Encoding references missing column: {trace.y.column}"
+                )
+
+    return errors
+
+
+def extract_data_from_mpl_figure(fig: "MplFigure") -> Optional[Any]:
+    """Extract plotted data from matplotlib figure.
+
+    Uses scitex.plt tracking if available (supports all 60+ plot types),
+    otherwise falls back to extracting from rendered figure.
+    """
+    import numpy as np
+    import pandas as pd
+
+    # Try scitex.plt tracking first (supports all plot types)
+    scitex_ax = _get_scitex_axes(fig)
+    if scitex_ax is not None:
+        try:
+            csv_df = scitex_ax.export_as_csv()
+            if csv_df is not None and not csv_df.empty:
+                return csv_df
+        except Exception:
+            pass
+
+    # Fallback: extract from rendered figure (limited plot types)
+    from ._extractors import extract_bar_data, extract_line_data, extract_scatter_data
+
+    extracted_data = {}
+    axes_list = list(fig.axes) if hasattr(fig.axes, "__iter__") else [fig.axes]
+
+    for ax_idx, ax in enumerate(axes_list):
+        extracted_data.update(extract_line_data(ax, ax_idx))
+        extracted_data.update(extract_scatter_data(ax, ax_idx))
+        extracted_data.update(extract_bar_data(ax, ax_idx))
+
+    if not extracted_data:
+        return None
+
+    max_len = max(len(v) for v in extracted_data.values())
+    padded = {}
+    for k, v in extracted_data.items():
+        if len(v) < max_len:
+            padded[k] = np.pad(v, (0, max_len - len(v)), constant_values=np.nan)
+        else:
+            padded[k] = v
+
+    return pd.DataFrame(padded)
+
+
+def build_encoding_from_mpl_figure(fig: "MplFigure") -> "Encoding":
+    """Build encoding specification from matplotlib figure.
+
+    Uses scitex.plt tracking if available (captures actual plot method),
+    otherwise falls back to detecting from rendered figure.
+    """
+    from .._fig._dataclasses import Encoding
+
+    # Try scitex.plt tracking first (knows exact plot method)
+    scitex_ax = _get_scitex_axes(fig)
+    if scitex_ax is not None and hasattr(scitex_ax, "history") and scitex_ax.history:
+        return _build_encoding_from_history(scitex_ax.history)
+
+    # Fallback: detect from rendered figure
+    from ._extractors import build_bar_traces, build_line_traces, build_scatter_traces
+
+    traces = []
+    axes_list = list(fig.axes) if hasattr(fig.axes, "__iter__") else [fig.axes]
+
+    for ax_idx, ax in enumerate(axes_list):
+        traces.extend(build_line_traces(ax, ax_idx))
+        traces.extend(build_scatter_traces(ax, ax_idx))
+        traces.extend(build_bar_traces(ax, ax_idx))
+
+    return Encoding(traces=traces)
+
+
+def build_theme_from_mpl_figure(fig: "MplFigure") -> "Theme":
+    """Build theme specification from matplotlib figure."""
+    from .._fig import Theme
+
+    return Theme()
+
+
+def extract_geometry_from_mpl_figure(fig: "MplFigure") -> dict:
+    """Extract geometry data from matplotlib figure for hit testing."""
+    try:
+        from scitex.plt.utils._hitmap import extract_path_data, extract_selectable_regions
+
+        return {
+            "path_data": extract_path_data(fig),
+            "selectable_regions": extract_selectable_regions(fig),
+        }
+    except Exception:
+        return {"elements": []}
+
+
+def generate_hitmap_from_mpl_figure(fig: "MplFigure", dpi: int = 300) -> tuple:
+    """Generate hitmap images from matplotlib figure."""
+    try:
+        import io
+
+        from scitex.plt.utils._hitmap import (
+            HITMAP_AXES_COLOR,
+            HITMAP_BACKGROUND_COLOR,
+            apply_hitmap_colors,
+            restore_original_colors,
+        )
+
+        axes_list = list(fig.axes) if hasattr(fig.axes, "__iter__") else [fig.axes]
+        original_props, color_map, groups = apply_hitmap_colors(fig)
+
+        saved_fig_facecolor = fig.patch.get_facecolor()
+        saved_ax_facecolors = []
+        for ax in axes_list:
+            saved_ax_facecolors.append(ax.get_facecolor())
+            ax.set_facecolor(HITMAP_BACKGROUND_COLOR)
+            for spine in ax.spines.values():
+                spine.set_color(HITMAP_AXES_COLOR)
+        fig.patch.set_facecolor(HITMAP_BACKGROUND_COLOR)
+
+        png_buf = io.BytesIO()
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message=".*tight_layout.*")
+            fig.savefig(png_buf, format="png", dpi=dpi, facecolor=HITMAP_BACKGROUND_COLOR)
+        png_bytes = png_buf.getvalue()
+
+        svg_buf = io.BytesIO()
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message=".*tight_layout.*")
+            fig.savefig(svg_buf, format="svg", facecolor=HITMAP_BACKGROUND_COLOR)
+        svg_bytes = svg_buf.getvalue()
+
+        restore_original_colors(original_props)
+        fig.patch.set_facecolor(saved_fig_facecolor)
+        for i, ax in enumerate(axes_list):
+            ax.set_facecolor(saved_ax_facecolors[i])
+
+        return png_bytes, svg_bytes
+    except Exception:
+        return None, None
+
+
+def from_matplotlib(
+    fig: "MplFigure",
+    path,
+    name: Optional[str] = None,
+    csv_df: Optional[Any] = None,
+    dpi: int = 300,
+):
+    """Create FTS bundle from matplotlib figure.
+
+    Args:
+        fig: Matplotlib figure object
+        path: Output path for bundle (.zip or directory)
+        name: Bundle name (defaults to path stem)
+        csv_df: Pre-extracted CSV data (uses scitex.plt tracking if None)
+        dpi: Resolution for raster exports
+
+    Note:
+        Encoding is built from CSV column names (single source of truth).
+        This ensures encoding references always match actual data columns.
+    """
+    import io
+    import json
+    from pathlib import Path
+
+    from ._FTS import FTS
+    from ._saver import save_bundle_components
+
+    path = Path(path)
+
+    fig_width_inch, fig_height_inch = fig.get_size_inches()
+    size_mm = {
+        "width": round(fig_width_inch * 25.4, 2),
+        "height": round(fig_height_inch * 25.4, 2),
+    }
+
+    bundle = FTS(path, create=True, kind="plot", name=name, size_mm=size_mm)
+
+    if csv_df is None:
+        csv_df = extract_data_from_mpl_figure(fig)
+
+    if csv_df is not None and not csv_df.empty:
+        bundle._node.payload_schema = "scitex.fts.payload.plot@1"
+
+    # Build encoding from actual CSV columns (single source of truth)
+    # This ensures encoding references match real data columns
+    if csv_df is not None and not csv_df.empty:
+        bundle._encoding = _build_encoding_from_csv_columns(csv_df)
+        # Validate encoding-CSV link
+        errors = validate_encoding_csv_link(bundle._encoding, csv_df)
+        if errors:
+            warnings.warn(f"Encoding validation errors: {errors}")
+    else:
+        bundle._encoding = build_encoding_from_mpl_figure(fig)
+    bundle._theme = build_theme_from_mpl_figure(fig)
+
+    storage = bundle.storage
+
+    if csv_df is not None and not csv_df.empty:
+        csv_bytes = csv_df.to_csv(index=False).encode("utf-8")
+        storage.write("payload/data.csv", csv_bytes)
+
+        data_info = {
+            "columns": list(csv_df.columns),
+            "dtypes": {col: str(dtype) for col, dtype in csv_df.dtypes.items()},
+            "shape": list(csv_df.shape),
+        }
+        storage.write("canonical/data_info.json", json.dumps(data_info, indent=2).encode())
+
+    for fmt in ["png", "svg", "pdf"]:
+        buf = io.BytesIO()
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message=".*tight_layout.*")
+            fig.savefig(buf, format=fmt, dpi=dpi, bbox_inches="tight")
+        storage.write(f"artifacts/exports/figure.{fmt}", buf.getvalue())
+
+    geometry = extract_geometry_from_mpl_figure(fig)
+    geometry["space"] = "figure_px"
+    storage.write("artifacts/cache/geometry_px.json", json.dumps(geometry, indent=2).encode())
+
+    hitmap_png, hitmap_svg = generate_hitmap_from_mpl_figure(fig, dpi)
+    if hitmap_png:
+        storage.write("artifacts/cache/hitmap.png", hitmap_png)
+    if hitmap_svg:
+        storage.write("artifacts/cache/hitmap.svg", hitmap_svg)
+
+    manifest = {"dpi": dpi, "formats": ["png", "svg", "pdf"], "size_mm": size_mm}
+    storage.write("artifacts/cache/render_manifest.json", json.dumps(manifest, indent=2).encode())
+
+    save_bundle_components(
+        path,
+        node=bundle._node,
+        encoding=bundle._encoding,
+        theme=bundle._theme,
+        render=False,
+    )
+
+    bundle._dirty = False
+    return bundle
+
+
+__all__ = [
+    "extract_data_from_mpl_figure",
+    "build_encoding_from_mpl_figure",
+    "build_theme_from_mpl_figure",
+    "extract_geometry_from_mpl_figure",
+    "generate_hitmap_from_mpl_figure",
+    "validate_encoding_csv_link",
+    "from_matplotlib",
+]
+
+# EOF