learnx-cli 0.3.0__py3-none-any.whl

tutor/generation/segment_planner.py

@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from collections.abc import Callable
+from dataclasses import asdict
+from pathlib import Path
+
+from tutor.constants import SUMMARY_CACHE_DIR
+from tutor.generation.segment_parser import fallback_segments, parse_segments_response
+from tutor.infra.llm import load_prompt
+from tutor.models import DialogueLine, SlideSegment
+
+log = logging.getLogger(__name__)
+
+
+def plan_segments(
+    units_json_path: Path,
+    video_dir: Path,
+    llm_fn: Callable,
+    no_cache: bool = False,
+) -> dict[int, list[SlideSegment]]:
+    """For each teaching unit call LLM with its dialogue lines.
+
+    Returns dict keyed by unit_index (int) → list[SlideSegment] in line order.
+    Writes tutorial.segments.json to video_dir.
+    Skips units with no dialogue lines — logs a warning, does not crash.
+    Never raises; returns fallback segments on any LLM or parse error.
+    """
+    unit_lines = _load_unit_lines(units_json_path)
+    all_segments: dict[int, list[SlideSegment]] = {}
+
+    for unit_index, (concept, lines) in sorted(unit_lines.items()):
+        if not lines:
+            log.warning("Unit %d has no dialogue lines — skipping", unit_index)
+            continue
+
+        cache_file = _cache_path(unit_index, lines)
+        if no_cache and cache_file.exists():
+            cache_file.unlink()
+
+        segs = _plan_unit_segments(unit_index, concept, lines, llm_fn, cache_file)
+        all_segments[unit_index] = segs
+
+    video_dir.mkdir(parents=True, exist_ok=True)
+    segments_path = video_dir / "tutorial.segments.json"
+    segments_path.write_text(
+        json.dumps(
+            {
+                "version": 1,
+                "units": {str(k): [asdict(s) for s in v] for k, v in sorted(all_segments.items())},
+            },
+            indent=2,
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    log.info("Segments written: %s (%d units)", segments_path, len(all_segments))
+    return all_segments
+
+
+def _plan_unit_segments(
+    unit_index: int,
+    unit_concept: str,
+    lines: list[DialogueLine],
+    llm_fn: Callable,
+    cache_file: Path,
+) -> list[SlideSegment]:
+    """Call LLM for one unit. Use file cache when available.
+
+    Returns fallback_segments() on any LLM or parse error.
+    """
+    if cache_file.exists():
+        log.debug("Cache hit for segments unit %d (%s)", unit_index, unit_concept)
+        try:
+            data = json.loads(cache_file.read_text(encoding="utf-8"))
+            return [SlideSegment(**s) for s in data]
+        except Exception:
+            pass
+
+    prompt = load_prompt("visual_v3.txt")
+    dialogue_text = "\n".join(f"{i}: [{ln.speaker}] {ln.text}" for i, ln in enumerate(lines))
+    unit_context = json.dumps(
+        {
+            "unit_index": unit_index,
+            "concept": unit_concept,
+            "total_lines": len(lines),
+            "dialogue": dialogue_text,
+        },
+        ensure_ascii=False,
+    )
+    messages = [
+        {"role": "system", "content": prompt},
+        {"role": "user", "content": unit_context},
+    ]
+
+    try:
+        raw = llm_fn(messages, call_type="segments")
+        segs = parse_segments_response(raw, unit_index, lines)
+    except Exception as exc:
+        log.warning(
+            "Segment planning failed for unit %d (%s): %s — using fallback",
+            unit_index,
+            unit_concept,
+            exc,
+        )
+        return fallback_segments(unit_index, lines)
+
+    cache_file.parent.mkdir(parents=True, exist_ok=True)
+    cache_file.write_text(
+        json.dumps([asdict(s) for s in segs], ensure_ascii=False),
+        encoding="utf-8",
+    )
+    return segs
+
+
+def _cache_path(unit_index: int, lines: list[DialogueLine]) -> Path:
+    """MD5 of all dialogue texts + 'segments_v3' → .tutor_cache/<hash>.segments.json"""
+    content = "segments_v3" + "".join(ln.text for ln in lines)
+    digest = hashlib.md5(content.encode()).hexdigest()
+    return Path(SUMMARY_CACHE_DIR) / f"{digest}.segments.json"
+
+
+def _load_unit_lines(units_json_path: Path) -> dict[int, tuple[str, list[DialogueLine]]]:
+    """Parse tutorial.units.json.
+
+    Returns dict: unit_number → (concept, list[DialogueLine]).
+    Only includes teaching units (unit_number >= 1).
+    Falls back to script.txt + timing.json when units.json has no lines.
+    """
+    import re
+
+    raw_units = json.loads(units_json_path.read_text(encoding="utf-8"))
+    result: dict[int, tuple[str, list[DialogueLine]]] = {}
+    for u in raw_units:
+        unit_num = int(u.get("unit", 0))
+        if unit_num < 1:
+            continue
+        concept = str(u.get("concept", ""))
+        raw_lines = u.get("lines", [])
+        lines = [DialogueLine(**ln) for ln in raw_lines]
+        result[unit_num] = (concept, lines)
+
+    if any(len(lines) > 0 for _, lines in result.values()):
+        return result
+
+    script_path = units_json_path.parent / "tutorial.script.txt"
+    if not script_path.exists():
+        return result
+
+    speaker_re = re.compile(r"^(ALEX|MAYA|SAM):\s*(.+)$")
+    all_pairs = [
+        (m.group(1), m.group(2))
+        for ln in script_path.read_text(encoding="utf-8").splitlines()
+        if (m := speaker_re.match(ln.strip()))
+    ]
+    if not all_pairs:
+        return result
+
+    timing_path = units_json_path.parent / "tutorial.timing.json"
+    lines_per_unit: dict[int, int] = {}
+    if timing_path.exists():
+        try:
+            timing = json.loads(timing_path.read_text(encoding="utf-8"))
+            if timing.get("version") == 1:
+                for uk, entries in timing.get("units", {}).items():
+                    lines_per_unit[int(uk)] = len(entries)
+        except Exception:
+            pass
+
+    if lines_per_unit and all(u in lines_per_unit for u in result):
+        n_teaching = sum(lines_per_unit.values())
+        n_non = max(0, len(all_pairs) - n_teaching)
+        n_intro = (n_non + 1) // 2
+        cursor = n_intro
+        for unit_num in sorted(result.keys()):
+            concept = result[unit_num][0]
+            count = lines_per_unit[unit_num]
+            pairs = all_pairs[cursor : cursor + count]
+            cursor += count
+            result[unit_num] = (
+                concept,
+                [DialogueLine(speaker=s, text=t, unit_number=unit_num) for s, t in pairs],
+            )
+    else:
+        n_units = len(result)
+        n_lines = len(all_pairs)
+        per_unit = max(1, n_lines // max(n_units, 1))
+        for i, unit_num in enumerate(sorted(result.keys())):
+            concept = result[unit_num][0]
+            start = i * per_unit
+            end = n_lines if i == n_units - 1 else min(start + per_unit, n_lines)
+            pairs = all_pairs[start:end]
+            result[unit_num] = (
+                concept,
+                [DialogueLine(speaker=s, text=t, unit_number=unit_num) for s, t in pairs],
+            )
+
+    return result

tutor/generation/visual_planner.py

@@ -0,0 +1,205 @@
+import hashlib
+import json
+import logging
+from dataclasses import asdict
+from pathlib import Path
+
+from tutor.constants import SUMMARY_CACHE_DIR
+from tutor.exceptions import LLMError
+from tutor.infra.llm import LLMFn, load_prompt, parse_json_response
+from tutor.models import TeachingUnit, VisualSpec
+
+log = logging.getLogger(__name__)
+
+VISUAL_PROMPT_VERSION = "visual_v1"
+VALID_DIAGRAM_TYPES = frozenset(
+    {"class_diagram", "flowchart", "code_comparison", "concept_map", "none"}
+)
+
+
+def plan_visuals(
+    units_json_path: Path,
+    doc_title: str,
+    session: str,
+    llm_fn: LLMFn,
+    difficulty: str,
+    video_dir: Path,
+    no_cache: bool = False,
+) -> list[VisualSpec]:
+    """
+    Read units from units_json_path, generate one VisualSpec per unit via LLM.
+    Returns [title_card, unit_1, ..., unit_N, outro].
+    Writes tutorial.visuals.json to video_dir.
+    """
+    raw_units = json.loads(units_json_path.read_text(encoding="utf-8"))
+    for u in raw_units:
+        u.setdefault("prerequisite_concepts", [])
+    units = [TeachingUnit(**u) for u in raw_units]
+
+    video_dir.mkdir(parents=True, exist_ok=True)
+
+    specs: list[VisualSpec] = [_build_title_card(doc_title, units, session)]
+    for unit in units:
+        cache_file = _cache_path(unit, difficulty)
+        if no_cache and cache_file.exists():
+            cache_file.unlink()
+        specs.append(_plan_unit(unit, llm_fn, difficulty, cache_file))
+    specs.append(_build_outro(units))
+
+    visuals_path = video_dir / "tutorial.visuals.json"
+    visuals_path.write_text(
+        json.dumps([asdict(s) for s in specs], indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    log.info("Visual specs written to %s (%d entries)", visuals_path, len(specs))
+    return specs
+
+
+def _plan_unit(
+    unit: TeachingUnit,
+    llm_fn: LLMFn,
+    difficulty: str,
+    cache_file: Path,
+) -> VisualSpec:
+    if cache_file.exists():
+        log.debug("Cache hit for visual spec unit %d (%s)", unit.unit, unit.concept)
+        data = json.loads(cache_file.read_text(encoding="utf-8"))
+        return VisualSpec(**data)
+
+    prompt = load_prompt("visual.txt")
+    unit_context = json.dumps(
+        {
+            "concept": unit.concept,
+            "key_facts": unit.key_facts,
+            "common_misconception": unit.common_misconception,
+            "good_analogy": unit.good_analogy,
+            "memory_hook": unit.memory_hook,
+            "word_budget": unit.word_budget,
+            "difficulty": difficulty,
+        },
+        indent=2,
+    )
+    messages = [
+        {"role": "system", "content": prompt},
+        {"role": "user", "content": unit_context},
+    ]
+
+    log.info("Generating visual spec for unit %d: %s", unit.unit, unit.concept)
+    try:
+        raw = llm_fn(messages, call_type="visual")
+        spec = _parse_visual_response(raw, unit)
+    except Exception as exc:
+        log.warning(
+            "Visual spec failed for unit %d (%s): %s — using fallback",
+            unit.unit,
+            unit.concept,
+            exc,
+        )
+        spec = _fallback_spec(unit)
+
+    cache_file.parent.mkdir(parents=True, exist_ok=True)
+    cache_file.write_text(
+        json.dumps(asdict(spec), ensure_ascii=False),
+        encoding="utf-8",
+    )
+    return spec
+
+
+def _parse_visual_response(raw: str, unit: TeachingUnit) -> VisualSpec:
+    try:
+        data = parse_json_response(raw)
+    except LLMError:
+        log.warning("Cannot parse visual spec JSON for unit %d", unit.unit)
+        return _fallback_spec(unit)
+
+    if not isinstance(data, dict):
+        log.warning("Visual spec is not a JSON object for unit %d", unit.unit)
+        return _fallback_spec(unit)
+
+    diagram_type = data.get("diagram_type", "none")
+    if diagram_type not in VALID_DIAGRAM_TYPES:
+        log.warning(
+            "Unknown diagram_type %r for unit %d — falling back to 'none'", diagram_type, unit.unit
+        )
+        diagram_type = "none"
+
+    diagram_spec = data.get("diagram_spec")
+    diagram_type, diagram_spec = _validate_diagram(diagram_type, diagram_spec, unit.unit)
+
+    return VisualSpec(
+        unit_index=unit.unit,
+        slide_type="unit",
+        concept=unit.concept,
+        hook_question=str(data.get("hook_question", "")),
+        key_points=list(data.get("key_points", unit.key_facts[:5])),
+        code_snippet=data.get("code_snippet") or None,
+        diagram_type=diagram_type,
+        diagram_spec=diagram_spec,  # type: ignore[arg-type]
+        memory_hook=str(data.get("memory_hook", unit.memory_hook)),
+        analogy=unit.good_analogy,
+    )
+
+
+def _validate_diagram(diagram_type: str, diagram_spec: object, unit_idx: int) -> tuple[str, object]:
+    if diagram_type in ("class_diagram", "flowchart", "concept_map"):
+        if not isinstance(diagram_spec, str) or not _looks_like_dot(diagram_spec):
+            log.warning("diagram_spec for unit %d is not valid DOT — setting to 'none'", unit_idx)
+            return "none", None
+    elif diagram_type == "code_comparison":
+        if not isinstance(diagram_spec, dict) or not all(
+            k in diagram_spec for k in ("wrong", "right")
+        ):
+            log.warning(
+                "code_comparison spec for unit %d is not a valid dict — setting to 'none'", unit_idx
+            )
+            return "none", None
+    else:
+        diagram_spec = None
+    return diagram_type, diagram_spec
+
+
+def _fallback_spec(unit: TeachingUnit) -> VisualSpec:
+    return VisualSpec(
+        unit_index=unit.unit,
+        slide_type="unit",
+        concept=unit.concept,
+        hook_question=f"What do you know about {unit.concept}?",
+        key_points=unit.key_facts[:5],
+        code_snippet=None,
+        diagram_type="none",
+        diagram_spec=None,
+        memory_hook=unit.memory_hook,
+        analogy=unit.good_analogy,
+    )
+
+
+def _build_title_card(doc_title: str, units: list[TeachingUnit], doc_source: str) -> VisualSpec:
+    n = len(units)
+    return VisualSpec(
+        unit_index=0,
+        slide_type="title_card",
+        title=doc_title,
+        subtitle=f"{n} unit{'s' if n != 1 else ''} - beginner",
+        doc_source=doc_source,
+    )
+
+
+def _build_outro(units: list[TeachingUnit]) -> VisualSpec:
+    return VisualSpec(
+        unit_index=len(units) + 1,
+        slide_type="outro",
+        memory_hooks=[u.memory_hook for u in units if u.memory_hook],
+        session_stats=f"{len(units)} unit{'s' if len(units) != 1 else ''}",
+    )
+
+
+def _cache_path(unit: TeachingUnit, difficulty: str) -> Path:
+    key = hashlib.md5(
+        (unit.concept + str(unit.key_facts) + difficulty + VISUAL_PROMPT_VERSION).encode()
+    ).hexdigest()
+    return Path(SUMMARY_CACHE_DIR) / f"{key}.visual.json"
+
+
+def _looks_like_dot(text: str) -> bool:
+    stripped = text.strip()
+    return stripped.startswith(("digraph", "graph", "strict"))

tutor/infra/llm.py

@@ -0,0 +1,152 @@
+import json
+import logging
+import re
+import time
+import tomllib
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, TypeAlias
+
+from openai import OpenAI
+
+from tutor.config import Config
+from tutor.exceptions import ConfigError, LLMError
+
+log = logging.getLogger(__name__)
+
+LLMFn: TypeAlias = Callable[..., str]
+
+# ---------------------------------------------------------------------------
+# Config loading — reads tutor/llm_config.toml at import time
+# ---------------------------------------------------------------------------
+
+_CONFIG_PATH = Path(__file__).parent.parent / "llm_config.toml"
+
+
+def _load() -> dict[str, Any]:
+    with open(_CONFIG_PATH, "rb") as fh:
+        return tomllib.load(fh)
+
+
+_cfg = _load()
+
+# Public dicts built from the TOML — used by chat() below and exported for
+# other modules that need to read limits (dialogue.py, summarizer.py).
+MODEL_MAP: dict[tuple[str, str], str] = {
+    (provider, call_type): model
+    for provider, calls in _cfg["providers"].items()
+    for call_type, model in calls.items()
+}
+
+MAX_TOKENS_MAP: dict[str, int] = _cfg["max_tokens"]
+LIMITS: dict[str, int] = _cfg["limits"]
+
+_temperature: float = _cfg["llm"]["temperature"]
+_retry_count: int = _cfg["llm"]["retry_count"]
+_retry_delay_s: float = _cfg["llm"]["retry_delay_s"]
+
+
+# ---------------------------------------------------------------------------
+# Client factory
+# ---------------------------------------------------------------------------
+
+
+def _build_client(provider: str, config: Config) -> OpenAI:
+    if provider == "groq":
+        if not config.groq_api_key:
+            raise ConfigError("GROQ_API_KEY not set. Add it to tutor/.env")
+        return OpenAI(
+            api_key=config.groq_api_key,
+            base_url="https://api.groq.com/openai/v1",
+        )
+    if provider == "openrouter":
+        if not config.openrouter_api_key:
+            raise ConfigError(
+                "OPENROUTER_API_KEY not set.\n"
+                "  Get a free key at openrouter.ai and add OPENROUTER_API_KEY to tutor/.env"
+            )
+        return OpenAI(
+            api_key=config.openrouter_api_key,
+            base_url="https://openrouter.ai/api/v1",
+            default_headers={"HTTP-Referer": "http://localhost"},
+            timeout=120.0,
+        )
+    raise ConfigError(f"Unknown provider: {provider!r}. Use 'groq' or 'openrouter'.")
+
+
+# ---------------------------------------------------------------------------
+# Chat
+# ---------------------------------------------------------------------------
+
+
+def chat(
+    messages: list[dict[str, str]],
+    config: Config,
+    provider: str = "groq",
+    call_type: str = "dialogue",
+) -> str:
+    model = MODEL_MAP.get((provider, call_type))
+    if model is None:
+        raise LLMError(f"No model configured for ({provider!r}, {call_type!r}) in llm_config.toml")
+
+    client = _build_client(provider, config)
+    log.debug("LLM call provider=%s call_type=%s model=%s", provider, call_type, model)
+
+    max_tokens = MAX_TOKENS_MAP.get(call_type, 1_000)
+
+    for attempt in range(_retry_count):
+        try:
+            response = client.chat.completions.create(
+                model=model,
+                messages=messages,  # type: ignore[arg-type]
+                temperature=_temperature,
+                max_tokens=max_tokens,
+            )
+            content = response.choices[0].message.content
+            assert content is not None, "LLM returned empty content"
+            log.debug("LLM response (first 200 chars): %s", content[:200])
+            return content
+        except Exception as e:
+            status = getattr(e, "status_code", None)
+            if status in (400, 401, 403):
+                raise LLMError(f"Auth/request error ({status}): {e}") from e
+            if status == 413:
+                raise LLMError(
+                    f"Request too large for {model}.\n"
+                    f"  Lower max_source_tokens or max_tokens.{call_type} in llm_config.toml."
+                ) from e
+            if attempt < _retry_count - 1:
+                log.warning("LLM call failed (%s), retrying in %.1fs...", e, _retry_delay_s)
+                time.sleep(_retry_delay_s)
+                continue
+            raise LLMError(f"LLM call failed after {_retry_count} attempts: {e}") from e
+
+    raise LLMError("Unreachable")
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def parse_json_response(raw: str) -> object:
+    text = re.sub(r"```(?:json)?\s*", "", raw).strip().rstrip("`").strip()
+
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    match = re.search(r"(\[.*\]|\{.*\})", text, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except json.JSONDecodeError:
+            pass
+
+    raise LLMError(f"Could not parse JSON from response: {raw[:200]}")
+
+
+def load_prompt(name: str) -> str:
+    prompts_dir = Path(__file__).parent.parent / "prompts"
+    return (prompts_dir / name).read_text(encoding="utf-8")