lispyn 0.17.0a1__py3-none-any.whl

lispyn/__init__.py

@@ -0,0 +1,7 @@
+"""Lispyn: Lisp-powered experiments and notebooks.
+
+Copyright 2026 Eduardo J. Barrios. SPDX-License-Identifier: Apache-2.0.
+"""
+
+__version__ = "0.17.0a1"
+__author__ = "Eduardo J. Barrios"

lispyn/classification.py

@@ -0,0 +1,170 @@
+"""Classification evaluation for PyTorch/Jupyter without a sklearn dependency.
+
+Copyright 2026 Eduardo J. Barrios. SPDX-License-Identifier: Apache-2.0.
+
+Confusion matrices use existing labels and predictions; this module does
+not run inference, train a model, or download anything.
+"""
+
+from __future__ import annotations
+
+from functools import partial
+from pathlib import Path
+
+from .plots import _design, _figure, _positive
+
+
+def _scalar(value):
+    """Convert scalar tensor/NumPy values to ordinary Python label values."""
+    if hasattr(value, "item"):
+        try:
+            return value.item()
+        except (TypeError, ValueError):
+            pass
+    return value
+
+
+def confusion_counts(actual, predicted, *, classes=None):
+    """Return (counts, classes) with rows=actual and columns=predicted.
+
+    The original order of class labels is preserved unless classes explicitly
+    specifies the desired order. Mis-typed or unseen classes are rejected
+    instead of silently dropping observations.
+    """
+    if isinstance(actual, (str, bytes)) or isinstance(predicted, (str, bytes)):
+        raise ValueError("actual and predicted must be sequences of class labels")
+    try:
+        actual = [_scalar(x) for x in actual]
+        predicted = [_scalar(x) for x in predicted]
+    except (TypeError, ValueError) as exc:
+        raise ValueError("actual and predicted must be one-dimensional sequences") from exc
+    if not actual or len(actual) != len(predicted):
+        raise ValueError("actual and predicted must have equal, nonzero lengths")
+
+    observed = []
+    try:
+        for label in actual + predicted:
+            hash(label)
+            if label not in observed:
+                observed.append(label)
+    except TypeError as exc:
+        raise ValueError("class labels must be hashable scalar values") from exc
+
+    if classes is None:
+        classes = observed
+    else:
+        if isinstance(classes, (str, bytes)):
+            raise ValueError("classes must be an ordered sequence, not a string")
+        try:
+            classes = [_scalar(label) for label in classes]
+            unique = set(classes)
+        except (TypeError, ValueError) as exc:
+            raise ValueError("classes must contain hashable scalar labels") from exc
+        if not classes or len(unique) != len(classes):
+            raise ValueError("classes must contain unique, nonempty class labels")
+        if set(observed) - set(classes):
+            raise ValueError("classes must include every observed class")
+
+    positions = {label: idx for idx, label in enumerate(classes)}
+    counts = [[0 for _ in classes] for _ in classes]
+    for truth, guess in zip(actual, predicted):
+        counts[positions[truth]][positions[guess]] += 1
+    return counts, classes
+
+
+def normalized_counts(counts, mode=None):
+    """Normalize counts by actual row, predicted column, or all samples."""
+    if mode not in (None, "true", "pred", "all"):
+        raise ValueError("normalize must be None, 'true', 'pred', or 'all'")
+    if mode is None:
+        return [[int(value) for value in row] for row in counts]
+    total = sum(sum(row) for row in counts)
+    row_sums = [sum(row) for row in counts]
+    col_sums = [sum(row[idx] for row in counts) for idx in range(len(counts))]
+    result = []
+    for row_idx, row in enumerate(counts):
+        out = []
+        for col_idx, value in enumerate(row):
+            denom = (
+                row_sums[row_idx] if mode == "true"
+                else col_sums[col_idx] if mode == "pred"
+                else total
+            )
+            out.append(value / denom if denom else 0.0)
+        result.append(out)
+    return result
+
+
+def plot_confusion_matrix(actual, predicted, *, classes=None, normalize=None,
+                          theme="docusaurus-dark", palette="turquoise-spectrum",
+                          title="Confusion matrix", subtitle=None,
+                          xlabel="Predicted", ylabel="Actual",
+                          figsize=(7, 5.6), title_size=16, label_size=11,
+                          tick_size=10, annotate=True, colorbar=True,
+                          save=None, dpi=150):
+    """Render a themed confusion matrix from actual/predicted classification labels.
+
+    Returns Matplotlib Figure for customization or inline Jupyter display.
+    'true' normalization divides by row support, 'pred' by column support,
+    'all' by total samples. Empty denominators are displayed as zero.
+    """
+    counts, names = confusion_counts(actual, predicted, classes=classes)
+    matrix = normalized_counts(counts, normalize)
+    if not isinstance(annotate, bool) or not isinstance(colorbar, bool):
+        raise ValueError("annotate and colorbar must be booleans")
+    _positive("dpi", dpi, 600)
+    design = _design(theme=theme, palette=palette)
+    try:
+        import matplotlib.pyplot as plt
+        from matplotlib.colors import LinearSegmentedColormap
+    except ImportError as exc:
+        raise RuntimeError('Confusion matrix plotting requires: pip install -e ".[plots]"') from exc
+
+    # All shades derive from the requested palette; the lowest intensities
+    # blend into the dark axes while the highest become bright turquoise.
+    from .plots import PALETTES
+    swatches = PALETTES[palette] if palette is not None else (
+        design["axes_bg"], design["train_color"], design["val_color"]
+    )
+    cmap = LinearSegmentedColormap.from_list(
+        "lispyn-confusion", [design["axes_bg"], *swatches]
+    )
+    fig, ax = _figure(
+        figsize=figsize, design=design, title=title, subtitle=subtitle,
+        xlabel=xlabel, ylabel=ylabel, grid=False, title_size=title_size,
+        label_size=label_size, tick_size=tick_size,
+    )
+    maximum = max(max(row) for row in matrix)
+    maximum = max(maximum, 1 if normalize is not None else 1)
+    image = ax.imshow(matrix, interpolation="nearest", cmap=cmap,
+                      vmin=0, vmax=maximum, aspect="equal")
+    ax.set_xticks(list(range(len(names))))
+    ax.set_yticks(list(range(len(names))))
+    ax.set_xticklabels([str(x) for x in names], rotation=30, ha="right")
+    ax.set_yticklabels([str(x) for x in names])
+    ax.tick_params(length=0, colors=design["text_color"])
+    if annotate:
+        for row_idx, row in enumerate(matrix):
+            for col_idx, value in enumerate(row):
+                text_color = (
+                    "#152129" if value / maximum > 0.58
+                    else design["text_color"]
+                )
+                text = f"{value:.2f}" if normalize is not None else str(value)
+                ax.text(col_idx, row_idx, text, color=text_color,
+                        ha="center", va="center", fontsize=tick_size)
+    if colorbar:
+        colorbar_obj = fig.colorbar(image, ax=ax, shrink=0.78, pad=0.035)
+        colorbar_obj.ax.tick_params(colors=design["text_color"])
+        colorbar_obj.outline.set_edgecolor(design["spine_color"])
+    fig.tight_layout(rect=(0, 0, 1, 0.93 if fig._suptitle else 1))
+    if save is not None:
+        output = Path(save).expanduser()
+        output.parent.mkdir(parents=True, exist_ok=True)
+        fig.savefig(output, facecolor=fig.get_facecolor(), dpi=dpi)
+    return fig
+
+
+def make_confusion_plotter(**defaults):
+    """Return a reusable confusion-matrix plotter, customizable per call."""
+    return partial(plot_confusion_matrix, **defaults)

lispyn/cli.py

@@ -0,0 +1,126 @@
+"""Command-line interface for Lispyn.
+
+Copyright 2026 Eduardo J. Barrios. SPDX-License-Identifier: Apache-2.0.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+
+from .compiler import LispynError, check_source, run_hy
+from .notebooks import build_notebook, execute_notebook, write_notebook
+from .reproducibility import summarize_run
+from .scaffold import create_experiment
+from .gallery import generate_gallery
+from .doctor import inspect_environment, format_report
+
+
+def parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="lispyn", description="Hy experiments, Python notebooks, PyTorch freedom."
+    )
+    p.add_argument("--version", action="version", version="lispyn 0.17.0a1")
+    sub = p.add_subparsers(dest="command", required=True)
+    health = sub.add_parser("doctor", help="Inspect installation and optional tools without modifying anything")
+    health.add_argument("--json", action="store_true", help="Print machine-readable environment summary")
+    for name, help_text in [
+        ("build", "Generate a Python Jupyter notebook without executing it"),
+        ("execute", "Generate, execute, and save the Jupyter notebook"),
+        ("run", "Run the Hy file directly without creating a notebook"),
+    ]:
+        action = sub.add_parser(name, help=help_text)
+        action.add_argument("source", type=Path, help="Trusted .hy source file")
+        if name != "run":
+            action.add_argument("-o", "--output", type=Path, default=None)
+        if name == "execute":
+            action.add_argument("--timeout", type=int, default=900, help="Per-cell timeout in seconds")
+            action.add_argument("--kernel", default="python3", help="Installed Jupyter kernel name")
+    gallery = sub.add_parser(
+        "gallery", help="Compile examples into example_generated_notebooks/ without execution"
+    )
+    gallery.add_argument("--source-dir", type=Path, default=Path("examples"))
+    gallery.add_argument("--output-dir", type=Path, default=Path("example_generated_notebooks"))
+    starter = sub.add_parser("new", help="Create a ready-to-edit Hy/Jupyter experiment")
+    starter.add_argument("source", type=Path, help="Path for a new .hy file")
+    report = sub.add_parser("report", help="Summarize a saved Lispyn experiment")
+    report.add_argument("directory", type=Path, help="Directory with manifest.json")
+    report.add_argument("--plot", type=Path, help="Save an optional PNG/SVG metric plot")
+    report.add_argument("--metric", default="loss", help="Metric to plot (default: loss)")
+    report.add_argument("--theme", default="clean", choices=["clean", "dark", "paper", "docusaurus-dark"])
+    report.add_argument("--title", help="Custom plot title")
+    report.add_argument("--subtitle", help="Optional secondary title")
+    report.add_argument("--palette", choices=["turquoise-spectrum"],
+                        help="Named color palette for the curves")
+    report.add_argument("--linewidth", type=float, default=2.2)
+    report.add_argument("--fill-alpha", type=float, default=0.0)
+    report.add_argument("--xlabel", default="Epoch", help="Horizontal axis label")
+    report.add_argument("--ylabel", help="Vertical axis label")
+    report.add_argument("--train-color", help="Matplotlib training-curve color")
+    report.add_argument("--val-color", help="Matplotlib validation-curve color")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parser().parse_args(argv)
+    try:
+        if args.command == "doctor":
+            report = inspect_environment()
+            print(json.dumps(report, indent=2) if args.json else format_report(report))
+            return 0
+        if args.command == "gallery":
+            paths = generate_gallery(args.source_dir, args.output_dir)
+            for path in paths:
+                print(f"Generated: {path}")
+            return 0
+        if args.command == "new":
+            print(f"Created: {create_experiment(args.source)}")
+            return 0
+        if args.command == "report":
+            print(summarize_run(args.directory))
+            if args.plot is not None:
+                from .plots import plot_history
+                fig = plot_history(args.directory / "history.json",
+                                   metric=args.metric, save=args.plot,
+                                    theme=args.theme, title=args.title, subtitle=args.subtitle,
+                                   palette=args.palette, linewidth=args.linewidth,
+                                   fill_alpha=args.fill_alpha,
+                                   xlabel=args.xlabel, ylabel=args.ylabel,
+                                   train_color=args.train_color,
+                                   val_color=args.val_color)
+                import matplotlib.pyplot as plt
+                plt.close(fig)
+                print(f"Plot saved: {args.plot.expanduser().resolve()}")
+            return 0
+        source = check_source(args.source)
+        if args.command == "run":
+            return run_hy(source)
+        output = args.output or source.with_suffix(".ipynb")
+        notebook = build_notebook(source)
+        if args.command == "build":
+            print(f"Generated: {write_notebook(notebook, output)}")
+            return 0
+        if args.timeout < -1:
+            raise LispynError("--timeout must be -1 (unlimited) or a nonnegative number")
+        # Save the notebook before running so the generated artifact survives
+        # kernel startup errors, and save available outputs after cell errors.
+        write_notebook(notebook, output)
+        try:
+            execute_notebook(notebook, cwd=source.parent, timeout=args.timeout, kernel=args.kernel)
+        finally:
+            write_notebook(notebook, output)
+        print(f"Executed and saved: {Path(output).resolve()}")
+        return 0
+    except (LispynError, OSError) as exc:
+        print(f"lispyn: {exc}", file=sys.stderr)
+        return 1
+    except Exception as exc:
+        # A failed notebook should not be reported as a successful run.
+        print(f"lispyn: execution failed: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

lispyn/compiler.py

@@ -0,0 +1,68 @@
+"""Compile a trusted Hy file into Python using Hy's official CLI.
+
+Copyright 2026 Eduardo J. Barrios. SPDX-License-Identifier: Apache-2.0.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+
+def find_hy_executable(name: str) -> str | None:
+    """Find the Hy script in the *current Python environment*, even if its
+    bin/Scripts directory has not been activated in PATH.
+    """
+    scripts = Path(sys.executable).parent
+    for candidate in (scripts / name, scripts / (name + ".exe")):
+        if candidate.is_file():
+            return str(candidate)
+    return shutil.which(name)
+
+
+class LispynError(Exception):
+    """An expected error while building or running a Lispyn experiment."""
+
+
+def check_source(source: Path) -> Path:
+    source = source.expanduser().resolve()
+    if not source.is_file():
+        raise LispynError(f"File not found: {source}")
+    if source.suffix != ".hy":
+        raise LispynError("Input must be a .hy file")
+    return source
+
+
+def compile_hy(source: Path, *, cwd: Path | None = None) -> str:
+    """Translate a Hy source file to Python; never execute its output here.
+
+    IMPORTANT: Hy macros may run arbitrary Python at *compile time*. Only build
+    code you trust. This is not a sandbox.
+    """
+    source = check_source(source)
+    hy2py = find_hy_executable("hy2py")
+    if hy2py is None:
+        raise LispynError("hy2py not found in this Python environment. Install Hy and Lispyn.")
+    result = subprocess.run(
+        [hy2py, str(source)],
+        cwd=cwd or source.parent,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    if result.returncode:
+        raise LispynError(
+            "Hy compilation failed:\n" + (result.stderr.strip() or result.stdout.strip())
+        )
+    return result.stdout
+
+
+def run_hy(source: Path) -> int:
+    """Execute trusted Hy source without creating a notebook."""
+    source = check_source(source)
+    hy = find_hy_executable("hy")
+    if hy is None:
+        raise LispynError("hy not found in this Python environment. Install Hy and Lispyn.")
+    return subprocess.run([hy, str(source)], cwd=source.parent, check=False).returncode

lispyn/data.py

@@ -0,0 +1,185 @@
+"""Opt-in dataset utilities for small, reproducible PyTorch experiments.
+
+Copyright 2026 Eduardo J. Barrios. SPDX-License-Identifier: Apache-2.0.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Sequence
+import random
+
+
+def read_dataset(path: str | Path, **kwargs):
+    """Load CSV, JSON, JSON Lines or Parquet as a pandas DataFrame.
+
+    pandas is optional: ``pip install -e '.[data]'``. Parquet also requires
+    an installed backend (typically pyarrow). File data is not executed.
+    """
+    try:
+        import pandas as pd
+    except ImportError as exc:
+        raise RuntimeError("Install pandas first: pip install -e '.[data]'") from exc
+
+    path = Path(path).expanduser()
+    readers = {
+        ".csv": pd.read_csv,
+        ".json": pd.read_json,
+        ".jsonl": pd.read_json,
+        ".parquet": pd.read_parquet,
+    }
+    reader = readers.get(path.suffix.lower())
+    if reader is None:
+        raise ValueError(f"Unsupported dataset extension: {path.suffix}")
+    if path.suffix.lower() == ".jsonl":
+        kwargs.setdefault("lines", True)
+    return reader(path, **kwargs)
+
+
+def drop_missing(frame, *, columns: Sequence[str] | None = None):
+    """Return a new DataFrame with incomplete rows removed."""
+    return frame.dropna(subset=list(columns) if columns is not None else None).reset_index(drop=True)
+
+
+def train_validation_split(frame, *, validation_fraction: float = 0.2,
+                           seed: int = 42, shuffle: bool = True):
+    """Reproducible row-level train/validation split; no sklearn needed.
+
+    Splits at least one row into each partition. This is not stratified or
+    group-aware: users must choose a different split for time-series or
+    datasets that could leak information between groups.
+    """
+    if not 0 < validation_fraction < 1:
+        raise ValueError("validation_fraction must be strictly between 0 and 1")
+    if len(frame) < 2:
+        raise ValueError("At least two rows are needed for a train/validation split")
+    shuffled = frame.sample(frac=1, random_state=seed) if shuffle else frame
+    validation_rows = max(1, min(len(frame) - 1, int(round(len(frame) * validation_fraction))))
+    return (
+        shuffled.iloc[:-validation_rows].reset_index(drop=True),
+        shuffled.iloc[-validation_rows:].reset_index(drop=True),
+    )
+
+
+
+def split_dataset(dataset, *, validation_fraction: float = 0.2,
+                  seed: int = 42, stratify: bool = False, labels=None):
+    """Split any indexable PyTorch dataset into two non-overlapping Subsets.
+
+    Set stratify=True to read dataset.targets (ImageFolder, AudioFolder,
+    labeled Lispyn text datasets). Or pass labels explicitly, which also
+    enables stratification; e.g. labels=[0, 1, ...] for TensorDataset.
+
+    A class must contain >=2 samples so both partitions contain that class.
+    Splitting is sample-level, not group-, speaker- or subject-aware: choose
+    a grouped split when images/audio/text from one source can leak across
+    partitions. Per-class rounding can change the overall validation fraction.
+    """
+    from torch.utils.data import Subset
+
+    if isinstance(validation_fraction, bool) or not 0 < validation_fraction < 1:
+        raise ValueError("validation_fraction must be strictly between 0 and 1")
+    if isinstance(seed, bool) or not isinstance(seed, int):
+        raise ValueError("seed must be an integer")
+    count = len(dataset)
+    if count < 2:
+        raise ValueError("At least two samples are needed for a dataset split")
+    rng = random.Random(seed)
+
+    if stratify or labels is not None:
+        if labels is None:
+            labels = getattr(dataset, "targets", None)
+        if labels is None:
+            raise ValueError("Stratified split requires labels or dataset.targets")
+        labels = list(labels)
+        if len(labels) != count:
+            raise ValueError("Number of labels must match dataset length")
+        groups = {}
+        try:
+            for index, label in enumerate(labels):
+                if hasattr(label, "item"):
+                    label = label.item()
+                groups.setdefault(label, []).append(index)
+        except TypeError as exc:
+            raise ValueError("Class labels must be hashable scalars") from exc
+
+        train_indices, valid_indices = [], []
+        for group in groups.values():
+            if len(group) < 2:
+                raise ValueError(
+                    "Stratified split needs at least two samples per class"
+                )
+            rng.shuffle(group)
+            n_valid = max(1, min(len(group) - 1,
+                                 round(len(group) * validation_fraction)))
+            valid_indices.extend(group[:n_valid])
+            train_indices.extend(group[n_valid:])
+        rng.shuffle(train_indices)
+        rng.shuffle(valid_indices)
+    else:
+        indices = list(range(count))
+        rng.shuffle(indices)
+        n_valid = max(1, min(count - 1, round(count * validation_fraction)))
+        valid_indices = indices[:n_valid]
+        train_indices = indices[n_valid:]
+    return Subset(dataset, train_indices), Subset(dataset, valid_indices)
+
+
+def to_tensor(values, *, dtype=None, device=None):
+    """Convert values into a tensor using the real, unrestricted torch API."""
+    try:
+        import torch
+    except ImportError as exc:
+        raise RuntimeError("Install PyTorch separately for your platform") from exc
+    return torch.as_tensor(values, dtype=dtype, device=device)
+
+
+def dataframe_to_dataset(frame, *, features: Sequence[str], target: str,
+                         feature_dtype=None, target_dtype=None):
+    """Turn numeric DataFrame columns into a PyTorch TensorDataset.
+
+    Features default to torch.float32; target defaults to torch.float32 for
+    regression. For classification, pass target_dtype=torch.long. Target is
+    a 1-D tensor, and the original DataFrame is not modified.
+    """
+    try:
+        import torch
+    except ImportError as exc:
+        raise RuntimeError("Install PyTorch separately for your platform") from exc
+    features = list(features)
+    if not features:
+        raise ValueError("features must include at least one column")
+    if target in features:
+        raise ValueError("target must not also be a feature (target leakage)")
+    missing = [name for name in [*features, target] if name not in frame.columns]
+    if missing:
+        raise ValueError(f"Missing dataset columns: {missing}")
+    x = torch.as_tensor(
+        frame[features].to_numpy(dtype="float32", copy=True),
+        dtype=feature_dtype or torch.float32,
+    )
+    y = torch.as_tensor(
+        frame[target].to_numpy(copy=True), dtype=target_dtype or torch.float32
+    )
+    return torch.utils.data.TensorDataset(x, y)
+
+
+def make_dataloader(dataset, *, batch_size: int = 32, shuffle: bool = False,
+                    seed: int | None = None, **kwargs):
+    """Build a native PyTorch DataLoader, optionally with seeded shuffling.
+
+    Arbitrary PyTorch DataLoader keyword options can pass through ``kwargs``.
+    """
+    try:
+        import torch
+    except ImportError as exc:
+        raise RuntimeError("Install PyTorch separately for your platform") from exc
+    if batch_size < 1:
+        raise ValueError("batch_size must be positive")
+    if seed is not None:
+        if "generator" in kwargs:
+            raise ValueError("Pass either seed or generator, not both")
+        kwargs["generator"] = torch.Generator().manual_seed(seed)
+    return torch.utils.data.DataLoader(
+        dataset, batch_size=batch_size, shuffle=shuffle, **kwargs
+    )