polyptych 0.1.0__py3-none-any.whl

common/compat.py

@@ -0,0 +1,91 @@
+"""Backward-compatible resolution of renamed env vars and runtime directories.
+
+The Polyptych rebrand renamed user-facing env vars from the ``SLIDE_GEN_*``
+namespace to ``POLYPTYCH_*`` and runtime directories from ``slide-analysis`` to
+``polyptych``. To avoid breaking existing setups, the old names are honored as
+deprecated fallbacks: the new name/location takes precedence, the old one is
+used only when the new is absent, and a one-time warning is logged on first use.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import overload
+
+logger = logging.getLogger("polyptych")
+
+APP_NAME = "polyptych"
+LEGACY_APP_NAME = "slide-analysis"
+
+# Module-global so the deprecation warning fires at most once per process per name.
+_warned_env: set[str] = set()
+
+
+@overload
+def getenv_compat(new_name: str, old_name: str, default: str) -> str: ...
+@overload
+def getenv_compat(new_name: str, old_name: str, default: None = None) -> str | None: ...
+
+
+def getenv_compat(
+    new_name: str, old_name: str, default: str | None = None
+) -> str | None:
+    """Read ``new_name`` from the environment, falling back to ``old_name``.
+
+    Precedence: ``$new_name`` > ``$old_name`` (deprecated) > ``default``. The
+    first time the deprecated ``old_name`` supplies the value, a one-time
+    warning is emitted on the ``polyptych`` logger. Matches ``os.environ.get``
+    semantics: an env var set to the empty string counts as set.
+    """
+    value = os.environ.get(new_name)
+    if value is not None:
+        return value
+    legacy = os.environ.get(old_name)
+    if legacy is not None:
+        if old_name not in _warned_env:
+            _warned_env.add(old_name)
+            logger.warning(
+                "Environment variable %s is deprecated and will be removed in a "
+                "future release; use %s instead.",
+                old_name,
+                new_name,
+            )
+        return legacy
+    return default
+
+
+def _runtime_base(kind: str) -> Path:
+    """Base dir for a runtime ``kind`` — ``config`` → ~/.config, ``cache`` → ~/.cache."""
+    return Path.home() / f".{kind}"
+
+
+def runtime_dirs(kind: str) -> tuple[Path, Path]:
+    """Return ``(new, legacy)`` runtime dirs for ``kind`` (``config`` or ``cache``)."""
+    base = _runtime_base(kind)
+    return base / APP_NAME, base / LEGACY_APP_NAME
+
+
+def resolve_runtime_file(kind: str, filename: str) -> Path | None:
+    """First existing ``filename`` across the new then legacy runtime dir, or ``None``."""
+    new, legacy = runtime_dirs(kind)
+    for directory in (new, legacy):
+        candidate = directory / filename
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def runtime_write_dir(kind: str) -> Path:
+    """Dir that new runtime files should be written to.
+
+    Prefers the new ``polyptych`` namespace, but keeps writing into an existing
+    legacy ``slide-analysis`` dir when that is the only one present, so a user's
+    credentials/state stay consolidated in one location rather than split across
+    two.
+    """
+    new, legacy = runtime_dirs(kind)
+    if not new.exists() and legacy.exists():
+        return legacy
+    return new

common/logging_setup.py

@@ -0,0 +1,96 @@
+"""Shared logging configuration helper for the CLI entry point.
+
+The ``polyptych`` CLI installs handlers on a named logger at startup so
+library modules can use ``logger = logging.getLogger(__name__)`` and emit
+diagnostics at the usual ``debug/info/warning/error`` levels without each
+module wiring up its own output. ``polyptych.logging_setup`` is a thin
+wrapper over this for the ``polyptych`` logger (kept separate because its
+public API is pinned by tests).
+
+This lives in ``common`` — the lowest shared layer — so the configurator can
+be reused without coupling modules together.
+
+User-facing CLI output (transcripts, timing tables, file paths, summaries)
+stays on ``print`` to stdout; only diagnostics flow through these loggers.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+
+from common.compat import getenv_compat
+
+LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+ENV_VAR = "POLYPTYCH_LOG_LEVEL"
+LEGACY_ENV_VAR = "SLIDE_GEN_LOG_LEVEL"
+
+_configured: set[str] = set()
+
+
+def resolve_level(level: str | int | None) -> int:
+    """Turn a user-supplied level (name, int, or ``None``) into a ``logging`` int."""
+    if level is None:
+        env = getenv_compat(ENV_VAR, LEGACY_ENV_VAR)
+        level = env if env else "INFO"
+    if isinstance(level, int):
+        return level
+    try:
+        return getattr(logging, level.upper())
+    except AttributeError as err:
+        raise ValueError(
+            f"Unknown log level: {level!r}. "
+            "Use DEBUG, INFO, WARNING, ERROR, or CRITICAL."
+        ) from err
+
+
+def configure_logging(
+    logger_name: str,
+    level: str | int | None = None,
+    log_file: str | Path | None = None,
+    *,
+    force: bool = False,
+) -> logging.Logger:
+    """Install stderr (and optional file) handlers on the named logger.
+
+    Level resolution order: explicit ``level`` argument, then the
+    ``POLYPTYCH_LOG_LEVEL`` env var (deprecated alias ``SLIDE_GEN_LOG_LEVEL``),
+    then ``INFO``. Safe to call more than once for a given ``logger_name`` —
+    repeat calls are ignored unless ``force=True``, which swaps the handlers so
+    a CLI flag can override the env var.
+    """
+    logger = logging.getLogger(logger_name)
+    if logger_name in _configured and not force:
+        return logger
+
+    if force:
+        for handler in list(logger.handlers):
+            logger.removeHandler(handler)
+            handler.close()
+
+    logger.setLevel(resolve_level(level))
+    formatter = logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)
+
+    stderr_handler = logging.StreamHandler(stream=sys.stderr)
+    stderr_handler.setFormatter(formatter)
+    logger.addHandler(stderr_handler)
+
+    if log_file is not None:
+        file_handler = logging.FileHandler(Path(log_file), encoding="utf-8")
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    logger.propagate = False  # stop log records from double-printing via root
+    _configured.add(logger_name)
+    return logger
+
+
+def reset_logging(logger_name: str) -> None:
+    """Drop all handlers and mark the named logger unconfigured. Used by tests."""
+    logger = logging.getLogger(logger_name)
+    for handler in list(logger.handlers):
+        logger.removeHandler(handler)
+        handler.close()
+    _configured.discard(logger_name)

common/usage_log.py

@@ -0,0 +1,32 @@
+"""JSONL logging for API usage (LLM, image generation)."""
+
+import json
+import threading
+from pathlib import Path
+
+from .compat import getenv_compat, runtime_write_dir
+
+_log_lock = threading.Lock()
+
+
+def default_usage_log() -> Path:
+    """Return the default usage-log path, resolved at call time.
+
+    Resolution order:
+    1. ``$POLYPTYCH_USAGE_LOG`` env var (new name).
+    2. ``$SLIDE_GEN_USAGE_LOG`` env var (deprecated alias; emits a one-time warning).
+    3. ``~/.cache/polyptych/usage.jsonl`` (stable per-user default via
+       :func:`common.compat.runtime_write_dir`).
+    """
+    env_val = getenv_compat("POLYPTYCH_USAGE_LOG", "SLIDE_GEN_USAGE_LOG")
+    if env_val is not None:
+        return Path(env_val)
+    return runtime_write_dir("cache") / "usage.jsonl"
+
+
+def log_usage(path: Path, entry: dict) -> None:
+    """Append a single usage entry as one JSON line (thread-safe)."""
+    line = json.dumps(entry, default=str) + "\n"
+    with _log_lock:
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(line)

polyptych/__init__.py

@@ -0,0 +1,26 @@
+"""Slide generation pipeline using Gemini API."""
+
+from .models import (
+    Task1Output,
+    Task2Output,
+    Task3Output,
+    Task4Output,
+    Task5Output,
+    Task6Output,
+    Task7Output,
+)
+from .client import TextClient, GeminiTextClient
+from .pipeline import SlidePipeline
+
+__all__ = [
+    "Task1Output",
+    "Task2Output",
+    "Task3Output",
+    "Task4Output",
+    "Task5Output",
+    "Task6Output",
+    "Task7Output",
+    "TextClient",
+    "GeminiTextClient",
+    "SlidePipeline",
+]

polyptych/batch_utils.py

@@ -0,0 +1,85 @@
+"""Generic batch orchestration: overlap splitting + parallel execution.
+
+Shared by task_a1_sentence_beats (paragraphs) and task_a2_shot_planning (beats).
+Each domain keeps its own spec dataclass (with domain-specific derived fields)
+and wraps `split_with_overlap()` to enrich the generic `OverlapBatch` records.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import Callable, Generic, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+@dataclass
+class OverlapBatch(Generic[T]):
+    """Generic overlap-split batch carrying core items + before/after context.
+
+    core_start is the 0-based index of the first core item in the original list.
+    Callers can derive 1-based IDs or other projections from core_start and
+    the lengths of core / context_before / context_after.
+    """
+
+    batch_index: int
+    core: list[T]
+    context_before: list[T]
+    context_after: list[T]
+    core_start: int
+
+
+def split_with_overlap(
+    items: list[T],
+    batch_size: int,
+    overlap: int = 0,
+) -> list[OverlapBatch[T]]:
+    """Split `items` into batches of `batch_size` with `overlap` before/after context.
+
+    The last batch may be shorter than batch_size. Context lists are clamped
+    to the available items at either end. Returns [] for empty input.
+    """
+    batches: list[OverlapBatch[T]] = []
+    for idx, start in enumerate(range(0, len(items), batch_size)):
+        end = min(start + batch_size, len(items))
+        before_start = max(0, start - overlap)
+        after_end = min(len(items), end + overlap)
+        batches.append(
+            OverlapBatch(
+                batch_index=idx,
+                core=items[start:end],
+                context_before=items[before_start:start],
+                context_after=items[end:after_end],
+                core_start=start,
+            )
+        )
+    return batches
+
+
+def run_batches(
+    specs: list[T],
+    process: Callable[[T], R],
+    max_workers: int,
+) -> list[R]:
+    """Run `process(spec)` for each spec, return results in submission order.
+
+    Single-batch fast path: if there is exactly one spec, it runs directly
+    without spawning a ThreadPoolExecutor. Otherwise results are collected
+    via `as_completed` and re-ordered by the spec's original index so that
+    ordering is deterministic regardless of completion order.
+    """
+    if len(specs) == 1:
+        return [process(specs[0])]
+
+    effective_workers = min(len(specs), max_workers)
+    results: dict[int, R] = {}
+    with ThreadPoolExecutor(max_workers=effective_workers) as executor:
+        futures = {
+            executor.submit(process, spec): idx for idx, spec in enumerate(specs)
+        }
+        for future in as_completed(futures):
+            idx = futures[future]
+            results[idx] = future.result()
+    return [results[i] for i in range(len(specs))]

polyptych/clean_source.py

@@ -0,0 +1,316 @@
+"""Clean PDF-to-markdown conversion artifacts from source files.
+
+Removes page numbers, footnote blocks, footnote superscripts, TOC with dot leaders,
+broken image references, and unnecessary backslash escaping.
+"""
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class CleaningConfig:
+    """Configuration for which cleaning passes to enable."""
+
+    remove_toc: bool = True
+    remove_image_refs: bool = True
+    remove_footnotes: bool = True  # strips both blocks and superscripts
+    remove_page_numbers: bool = True
+    unescape_chars: bool = True
+
+
+@dataclass
+class CleaningResult:
+    """Statistics from a cleaning run."""
+
+    original_lines: int = 0
+    cleaned_lines: int = 0
+    toc_lines_removed: int = 0
+    image_refs_removed: int = 0
+    footnote_refs_removed: int = 0  # superscripts stripped
+    footnote_blocks_removed: int = 0  # block lines stripped
+    page_numbers_removed: int = 0
+    escapes_fixed: int = 0
+
+
+def remove_toc(text: str) -> tuple[str, int]:
+    """Remove table of contents with dot-leader lines.
+
+    Finds a CONTENTS heading, then removes all lines until the first chapter
+    heading that is NOT followed by dot leaders.
+    """
+    lines = text.split("\n")
+    # Find the CONTENTS heading
+    contents_idx = None
+    for i, line in enumerate(lines):
+        if re.match(r"^\*{0,2}CONTENTS\*{0,2}\s*$", line.strip()):
+            contents_idx = i
+            break
+
+    if contents_idx is None:
+        return text, 0
+
+    # Find the end of TOC: first chapter heading pattern not part of TOC
+    # Chapter headings look like **1. or **1\. at the start
+    # TOC lines contain 5+ consecutive dots
+    toc_end = None
+    for i in range(contents_idx + 1, len(lines)):
+        line = lines[i].strip()
+        # A chapter heading that starts the actual content (not a TOC entry)
+        if re.match(r"\*{2}\d+\\?\.", line) and "....." not in line:
+            toc_end = i
+            break
+
+    if toc_end is None:
+        # No chapter heading found after TOC; remove to end of consecutive TOC-like lines
+        toc_end = contents_idx + 1
+        for i in range(contents_idx + 1, len(lines)):
+            line = lines[i].strip()
+            if (
+                line == ""
+                or "....." in line
+                or re.match(r"^\*", line)
+                or re.match(r"^[A-Z]", line)
+            ):
+                toc_end = i + 1
+            else:
+                break
+
+    removed = toc_end - contents_idx
+    result_lines = lines[:contents_idx] + lines[toc_end:]
+    return "\n".join(result_lines), removed
+
+
+def remove_image_refs(text: str) -> tuple[str, int]:
+    """Remove broken image references and their captions.
+
+    Matches lines like *![][imageN]* and the immediately following caption line.
+    """
+    lines = text.split("\n")
+    result = []
+    count = 0
+    skip_next = False
+
+    for i, line in enumerate(lines):
+        if skip_next:
+            skip_next = False
+            continue
+        if re.match(r"^\s*\*?!\[\]\[image\d+\]\*?\s*$", line.strip()):
+            count += 1
+            # Skip the following caption line if it's a short non-heading, non-blank line
+            if i + 1 < len(lines):
+                next_line = lines[i + 1].strip()
+                if (
+                    next_line
+                    and not next_line.startswith("**")
+                    and len(next_line) < 200
+                ):
+                    skip_next = True
+            continue
+        result.append(line)
+
+    return "\n".join(result), count
+
+
+def remove_footnote_blocks(text: str) -> tuple[str, int]:
+    """Remove footnote reference blocks.
+
+    Footnote blocks start with a line of whitespace (typically `    ` or blank),
+    followed by lines starting with a number + space (e.g., `1 General C.C. Krulak...`).
+    """
+    lines = text.split("\n")
+    result = []
+    count = 0
+    i = 0
+
+    while i < len(lines):
+        # Check for separator line (whitespace-only, often `    `) followed by footnote
+        if re.match(r"^[\s]*$", lines[i]):
+            # Look ahead for footnote lines
+            j = i + 1
+            # Skip additional blank lines
+            while j < len(lines) and re.match(r"^\s*$", lines[j]):
+                j += 1
+            # Check if next non-blank line starts a footnote (number + space at start)
+            if j < len(lines) and re.match(r"^\d{1,3}\s+\S", lines[j]):
+                # This is a footnote block - consume all footnote lines
+                block_start = i
+                # Consume the separator lines
+                while j < len(lines):
+                    line = lines[j].strip()
+                    if line == "":
+                        # Blank line - check if followed by another footnote
+                        k = j + 1
+                        while k < len(lines) and re.match(r"^\s*$", lines[k]):
+                            k += 1
+                        if k < len(lines) and re.match(r"^\d{1,3}\s+\S", lines[k]):
+                            j = k + 1
+                            continue
+                        else:
+                            break
+                    elif re.match(r"^\d{1,3}\s+\S", line):
+                        j += 1
+                        continue
+                    else:
+                        # Continuation line of a footnote (wrapped text)
+                        # Check if it looks like content (starts with capital, no footnote number)
+                        # Heuristic: if the previous line was a footnote, this could be continuation
+                        if j > 0 and (
+                            re.match(r"^\d{1,3}\s+\S", lines[j - 1].strip())
+                            or lines[j - 1].strip() == ""
+                        ):
+                            j += 1
+                            continue
+                        else:
+                            break
+
+                removed_lines = j - block_start
+                count += removed_lines
+                i = j
+                continue
+
+        result.append(lines[i])
+        i += 1
+
+    return "\n".join(result), count
+
+
+def remove_footnote_superscripts(text: str) -> tuple[str, int]:
+    """Strip footnote superscript digits glued to word endings.
+
+    Handles: `intellect1.` → `intellect.`, `war2.` → `war.`
+    Does NOT strip: dates like 1997, model numbers like F-16, standalone numbers.
+    """
+    # Match 1-3 digits after a letter, before punctuation or whitespace/end
+    # Negative lookbehind: don't match after digits (avoids stripping from years/numbers)
+    # Negative lookbehind: don't match after hyphen (avoids F-16 → F-)
+    pattern = r"(?<=[a-zA-Z])(\d{1,3})(?=[.,;:!?\s\)\]\"\'»]|$)"
+
+    count = 0
+
+    def _replace(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return ""
+
+    result = re.sub(pattern, _replace, text)
+    return result, count
+
+
+def remove_page_numbers(text: str) -> tuple[str, int]:
+    r"""Remove standalone page number lines.
+
+    Removes lines matching `^\d{1,3}\s*$` (Arabic) or `^[ivxlc]+\s*$` (Roman)
+    that are surrounded by blank lines.
+    """
+    lines = text.split("\n")
+    result = []
+    count = 0
+
+    # Find first chapter heading to determine front matter boundary for Roman numerals
+    first_chapter = len(lines)
+    for i, line in enumerate(lines):
+        if re.match(r"\*{2}\d+\\?\.", line.strip()):
+            first_chapter = i
+            break
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+
+        # Check for Arabic page numbers
+        is_arabic = bool(re.match(r"^\d{1,3}$", stripped))
+
+        # Check for Roman numeral page numbers (only in front matter)
+        is_roman = bool(re.match(r"^[ivxlc]+$", stripped)) and i < first_chapter
+
+        if (is_arabic or is_roman) and _is_surrounded_by_blanks(lines, i):
+            count += 1
+            continue
+
+        result.append(line)
+
+    return "\n".join(result), count
+
+
+def _is_surrounded_by_blanks(lines: list[str], idx: int) -> bool:
+    """Check if a line is surrounded by blank (or near-blank) lines."""
+    prev_blank = idx == 0 or lines[idx - 1].strip() == ""
+    next_blank = idx == len(lines) - 1 or lines[idx + 1].strip() == ""
+    return prev_blank and next_blank
+
+
+def unescape_chars(text: str) -> tuple[str, int]:
+    r"""Remove unnecessary backslash escaping from PDF conversion.
+
+    Cleans: `\.` → `.`, `\-` → `-`, `\+` → `+`, `\[` → `[`, `\]` → `]`,
+    `\(` → `(`, `\)` → `)`.
+    """
+    chars = r".\-+[]()"
+    count = 0
+    for ch in chars:
+        escaped = "\\" + ch
+        occurrences = text.count(escaped)
+        if occurrences > 0:
+            count += occurrences
+            text = text.replace(escaped, ch)
+    return text, count
+
+
+def normalize_whitespace(text: str) -> str:
+    """Collapse 3+ consecutive blank lines to 2. Strip trailing whitespace per line."""
+    # Strip trailing whitespace per line
+    lines = [line.rstrip() for line in text.split("\n")]
+    text = "\n".join(lines)
+    # Collapse 3+ consecutive blank lines to 2
+    text = re.sub(r"\n{4,}", "\n\n\n", text)
+    return text
+
+
+def clean_source(
+    text: str, config: CleaningConfig | None = None
+) -> tuple[str, CleaningResult]:
+    """Run all enabled cleaning passes in order. Idempotent."""
+    if config is None:
+        config = CleaningConfig()
+
+    result = CleaningResult()
+    result.original_lines = len(text.split("\n"))
+
+    if config.remove_toc:
+        text, result.toc_lines_removed = remove_toc(text)
+
+    if config.remove_image_refs:
+        text, result.image_refs_removed = remove_image_refs(text)
+
+    if config.remove_footnotes:
+        text, result.footnote_blocks_removed = remove_footnote_blocks(text)
+        text, result.footnote_refs_removed = remove_footnote_superscripts(text)
+
+    if config.remove_page_numbers:
+        text, result.page_numbers_removed = remove_page_numbers(text)
+
+    if config.unescape_chars:
+        text, result.escapes_fixed = unescape_chars(text)
+
+    text = normalize_whitespace(text)
+    result.cleaned_lines = len(text.split("\n"))
+
+    return text, result
+
+
+def clean_file(
+    input_path: Path,
+    output_path: Path | None = None,
+    config: CleaningConfig | None = None,
+) -> tuple[Path, CleaningResult]:
+    """Read file, clean, write output. Default output: <stem>-clean.md in same directory."""
+    text = input_path.read_text(encoding="utf-8")
+
+    if output_path is None:
+        output_path = input_path.parent / f"{input_path.stem}-clean{input_path.suffix}"
+
+    cleaned_text, result = clean_source(text, config)
+    output_path.write_text(cleaned_text, encoding="utf-8")
+
+    return output_path, result