learnx-cli 0.3.0__py3-none-any.whl

tutor/generation/dialogue.py

@@ -0,0 +1,172 @@
+import hashlib
+import json
+import logging
+import re
+from pathlib import Path
+
+from tutor.constants import PROMPT_VERSION, SUMMARY_CACHE_DIR
+from tutor.exceptions import LLMError
+from tutor.infra.llm import LIMITS, LLMFn, load_prompt
+from tutor.models import Chunk, DialogueLine, TeachingUnit
+
+log = logging.getLogger(__name__)
+
+
+def generate(
+    unit: TeachingUnit,
+    source_chunks: list[Chunk],
+    fmt: str,
+    llm_fn: LLMFn,
+    difficulty: str = "beginner",
+    cache_dir: str = SUMMARY_CACHE_DIR,
+) -> list[DialogueLine]:
+    cache_key = hashlib.md5(
+        (unit.concept + str(unit.word_budget) + fmt + difficulty + PROMPT_VERSION).encode()
+    ).hexdigest()
+    cache_file = Path(cache_dir) / f"{cache_key}.dialogue.json"
+
+    if cache_file.exists():
+        log.debug("Cache hit for dialogue unit %d (%s)", unit.unit, unit.concept)
+        raw_lines = json.loads(cache_file.read_text(encoding="utf-8"))
+        return [DialogueLine(**d) for d in raw_lines]
+
+    relevant = [c for c in source_chunks if c.chunk_id in unit.source_sections]
+    if not relevant:
+        relevant = source_chunks[:2]
+    source_text = "\n\n".join(f"## {c.heading}\n{c.text}" for c in relevant)
+    source_text = _truncate_source(source_text, LIMITS["max_source_tokens"])
+
+    unit_json = json.dumps(
+        {
+            "concept": unit.concept,
+            "complexity": unit.complexity,
+            "word_budget": unit.word_budget,
+            "key_facts": unit.key_facts,
+            "common_misconception": unit.common_misconception,
+            "good_analogy": unit.good_analogy,
+            "js_contrast": unit.js_contrast,
+            "question_style": unit.question_style,
+            "memory_hook": unit.memory_hook,
+            "prerequisite_concepts": unit.prerequisite_concepts,
+            "production_relevance": unit.production_relevance,
+        },
+        indent=2,
+    )
+
+    speaker_constraint = (
+        "IMPORTANT: Only use ALEX and SAM speakers. Do NOT use MAYA."
+        if fmt == "dual-tutor"
+        else "IMPORTANT: Only use ALEX and MAYA speakers. Do NOT use SAM."
+    )
+    system_prompt = (
+        load_prompt("dialogue.txt").format(
+            format=fmt,
+            word_budget=unit.word_budget,
+        )
+        + f"\n\n{speaker_constraint}"
+    )
+
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": f"Unit:\n{unit_json}\n\nSource:\n{source_text}"},
+    ]
+
+    log.info("Generating dialogue for unit %d: %s", unit.unit, unit.concept)
+    raw = llm_fn(messages, call_type="dialogue")
+    lines = _parse_dialogue(raw, unit.unit)
+
+    if len(lines) < 4:
+        log.warning("Only %d lines parsed, retrying dialogue generation", len(lines))
+        raw = llm_fn(messages, call_type="dialogue")
+        lines = _parse_dialogue(raw, unit.unit)
+        if len(lines) < 4:
+            raise LLMError(
+                f"Dialogue generation returned fewer than 4 lines for unit {unit.unit}: {unit.concept}"
+            )
+
+    lines = _normalize_speakers(lines, fmt)
+    _validate_speakers(lines, fmt)
+
+    Path(cache_dir).mkdir(parents=True, exist_ok=True)
+    cache_file.write_text(
+        json.dumps(
+            [
+                {"speaker": ln.speaker, "text": ln.text, "unit_number": ln.unit_number}
+                for ln in lines
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    return lines
+
+
+def _truncate_source(text: str, max_tokens: int) -> str:
+    words = text.split()
+    max_words = int(max_tokens / 1.3)
+    if len(words) <= max_words:
+        return text
+    log.warning(
+        "Source text truncated from %d to %d words for context limit", len(words), max_words
+    )
+    return " ".join(words[:max_words])
+
+
+def _parse_dialogue_line(raw_line: str, unit_number: int) -> DialogueLine | None:
+    match = re.match(r"^(ALEX|MAYA|SAM)\s*[:\-]\s*(.+)", raw_line.strip(), re.IGNORECASE)
+    if not match:
+        return None
+    return DialogueLine(
+        speaker=match.group(1).upper(),
+        text=match.group(2).strip(),
+        unit_number=unit_number,
+    )
+
+
+def _normalize_speakers(lines: list[DialogueLine], fmt: str) -> list[DialogueLine]:
+    """Remap speakers so the output matches the requested format."""
+    if fmt == "dual-tutor":
+        return [
+            DialogueLine(
+                speaker="SAM" if ln.speaker == "MAYA" else ln.speaker,
+                text=ln.text,
+                unit_number=ln.unit_number,
+            )
+            for ln in lines
+        ]
+    return [
+        DialogueLine(
+            speaker="MAYA" if ln.speaker == "SAM" else ln.speaker,
+            text=ln.text,
+            unit_number=ln.unit_number,
+        )
+        for ln in lines
+    ]
+
+
+def _validate_speakers(lines: list[DialogueLine], fmt: str) -> None:
+    speakers = {line.speaker for line in lines}
+    if fmt == "tutor-student":
+        if "ALEX" not in speakers:
+            raise LLMError("tutor-student dialogue missing ALEX lines")
+        if "SAM" in speakers:
+            raise LLMError("tutor-student dialogue contains SAM — wrong format")
+    elif fmt == "dual-tutor":
+        if "MAYA" in speakers:
+            raise LLMError("dual-tutor dialogue contains MAYA — wrong format")
+        expected = {"ALEX", "SAM"}
+        if not expected.issubset(speakers):
+            raise LLMError(f"dual-tutor dialogue missing speakers: {expected - speakers}")
+
+
+def _parse_dialogue(raw: str, unit_number: int) -> list[DialogueLine]:
+    lines: list[DialogueLine] = []
+    for raw_line in raw.split("\n"):
+        if not raw_line.strip():
+            continue
+        parsed = _parse_dialogue_line(raw_line, unit_number)
+        if parsed:
+            lines.append(parsed)
+        else:
+            log.debug("Skipping unparseable line: %s", raw_line[:80])
+    return lines

tutor/generation/narrator.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+from pathlib import Path
+
+from tutor.audio import sanitizer
+from tutor.constants import SUMMARY_CACHE_DIR
+from tutor.infra.llm import LLMFn, load_prompt
+from tutor.models import Chunk, DialogueLine, TeachingUnit
+
+log = logging.getLogger(__name__)
+
+NARRATE_VERSION = "narrate_v1"
+_WORDS_PER_SOURCE_WORD = 1.25
+
+
+def narrate_all(
+    chunks: list[Chunk],
+    doc_title: str,
+    llm_fn: LLMFn,
+    cache_dir: str = SUMMARY_CACHE_DIR,
+) -> tuple[list[TeachingUnit], list[list[DialogueLine]]]:
+    """Narrate every chunk in document order. Returns (units, all_lines)."""
+    units: list[TeachingUnit] = []
+    all_lines: list[list[DialogueLine]] = []
+    total = len(chunks)
+
+    for i, chunk in enumerate(chunks):
+        lines = _narrate_chunk(chunk, i + 1, total, doc_title, llm_fn, cache_dir)
+        units.append(_chunk_to_unit(chunk, i + 1))
+        all_lines.append(lines)
+
+    return units, all_lines
+
+
+def _chunk_to_unit(chunk: Chunk, unit_index: int) -> TeachingUnit:
+    word_budget = max(100, int(len(chunk.text.split()) * _WORDS_PER_SOURCE_WORD))
+    return TeachingUnit(
+        unit=unit_index,
+        concept=chunk.heading or f"Section {unit_index}",
+        source_sections=[chunk.chunk_id],
+        complexity=1,
+        word_budget=word_budget,
+        key_facts=[],
+        common_misconception="",
+        good_analogy="",
+        question_style="recall",
+        memory_hook="",
+    )
+
+
+def _narrate_chunk(
+    chunk: Chunk,
+    section_index: int,
+    total_sections: int,
+    doc_title: str,
+    llm_fn: LLMFn,
+    cache_dir: str,
+) -> list[DialogueLine]:
+    cache_key = hashlib.md5((chunk.chunk_id + chunk.text + NARRATE_VERSION).encode()).hexdigest()
+    cache_file = Path(cache_dir) / f"{cache_key}.narrate.json"
+
+    if cache_file.exists():
+        log.debug(
+            "Cache hit for narration %d/%d (%s)", section_index, total_sections, chunk.heading
+        )
+        raw = json.loads(cache_file.read_text(encoding="utf-8"))
+        return [DialogueLine(**d) for d in raw]
+
+    word_budget = max(100, int(len(chunk.text.split()) * _WORDS_PER_SOURCE_WORD))
+    prompt = load_prompt("narrate.txt").format(
+        doc_title=doc_title,
+        section_index=section_index,
+        total_sections=total_sections,
+        heading=chunk.heading,
+        word_budget=word_budget,
+        section_text=chunk.text,
+    )
+
+    log.info("Narrating section %d/%d: %s", section_index, total_sections, chunk.heading)
+    raw_text = llm_fn([{"role": "user", "content": prompt}], call_type="dialogue")
+    lines = _parse_narration(raw_text, section_index)
+
+    if not lines:
+        log.warning("No lines parsed for section %d, retrying", section_index)
+        raw_text = llm_fn([{"role": "user", "content": prompt}], call_type="dialogue")
+        lines = _parse_narration(raw_text, section_index)
+
+    for line in lines:
+        line.text = sanitizer.apply(line.text)
+
+    Path(cache_dir).mkdir(parents=True, exist_ok=True)
+    cache_file.write_text(
+        json.dumps(
+            [
+                {"speaker": ln.speaker, "text": ln.text, "unit_number": ln.unit_number}
+                for ln in lines
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    return lines
+
+
+def _parse_narration(raw: str, unit_number: int) -> list[DialogueLine]:
+    lines: list[DialogueLine] = []
+    for raw_line in raw.split("\n"):
+        stripped = raw_line.strip()
+        if not stripped:
+            continue
+        match = re.match(r"^ALEX\s*[:\-]\s*(.+)", stripped, re.IGNORECASE)
+        if match:
+            lines.append(
+                DialogueLine(speaker="ALEX", text=match.group(1).strip(), unit_number=unit_number)
+            )
+        else:
+            log.debug("Skipping unparseable narration line: %s", stripped[:80])
+    return lines

tutor/generation/segment_parser.py

@@ -0,0 +1,223 @@
+"""Parse and normalise raw LLM segment responses into SlideSegment lists."""
+
+from __future__ import annotations
+
+import logging
+
+from tutor.infra.llm import parse_json_response
+from tutor.models import VALID_VISUAL_TYPES, DialogueLine, SlideSegment
+
+log = logging.getLogger(__name__)
+
+
+def parse_segments_response(
+    raw: str,
+    unit_index: int,
+    lines: list[DialogueLine],
+) -> list[SlideSegment]:
+    """Parse LLM JSON array into SlideSegment objects.
+
+    Validates visual_type, clamps indices, fills gaps.
+    Falls back to fallback_segments() on any parse failure.
+    """
+    try:
+        data = parse_json_response(raw)
+    except Exception:
+        return fallback_segments(unit_index, lines)
+
+    if not isinstance(data, list):
+        return fallback_segments(unit_index, lines)
+
+    n = len(lines)
+    result: list[SlideSegment] = []
+
+    for item in data:
+        if not isinstance(item, dict):
+            continue
+
+        vtype = item.get("visual_type", "key_insight")
+        if vtype not in VALID_VISUAL_TYPES:
+            vtype = "key_insight"
+
+        ls = int(item.get("lines_start", 0))
+        le = int(item.get("lines_end", 0))
+
+        if ls > le:
+            ls, le = le, ls
+
+        ls = max(0, ls)
+        le = min(n - 1, le) if n > 0 else 0
+
+        title = item.get("title") or vtype.replace("_", " ").title()
+        body = item.get("body") or None
+        code = item.get("code") or None
+        language = item.get("language") or None
+        mermaid = item.get("mermaid") if vtype == "diagram" else None
+        left = item.get("left") or None
+        right = item.get("right") or None
+        rows = item.get("rows") or None
+
+        if rows is not None:
+            if not (isinstance(rows, list) and all(isinstance(r, list) for r in rows)):
+                rows = None
+
+        seg = SlideSegment(
+            unit_index=unit_index,
+            segment_index=0,
+            lines_start=ls,
+            lines_end=le,
+            visual_type=vtype,
+            title=title,
+            body=body,
+            code=code,
+            language=language,
+            mermaid=mermaid,
+            left=left,
+            right=right,
+            rows=rows,
+        )
+        result.append(_validate_segment(seg))
+
+    if not result:
+        return fallback_segments(unit_index, lines)
+
+    return fill_gaps(result, unit_index, n)
+
+
+def fill_gaps(
+    raw_segments: list[SlideSegment],
+    unit_index: int,
+    total_lines: int,
+) -> list[SlideSegment]:
+    """Ensure every line 0..total_lines-1 is covered by exactly one segment.
+
+    Inserts key_insight segments for uncovered ranges and renumbers segment_index.
+    """
+    if total_lines == 0:
+        return raw_segments
+
+    segs = sorted(raw_segments, key=lambda s: s.lines_start)
+    result: list[SlideSegment] = []
+    cursor = 0
+
+    for seg in segs:
+        if seg.lines_end < cursor:
+            continue
+
+        start = max(seg.lines_start, cursor)
+
+        if start > cursor:
+            result.append(_make_gap_segment(unit_index, cursor, start - 1))
+
+        adjusted = (
+            seg
+            if seg.lines_start == start
+            else SlideSegment(
+                unit_index=seg.unit_index,
+                segment_index=seg.segment_index,
+                lines_start=start,
+                lines_end=seg.lines_end,
+                visual_type=seg.visual_type,
+                title=seg.title,
+                body=seg.body,
+                code=seg.code,
+                language=seg.language,
+                mermaid=seg.mermaid,
+                left=seg.left,
+                right=seg.right,
+                rows=seg.rows,
+            )
+        )
+        result.append(adjusted)
+        cursor = adjusted.lines_end + 1
+
+    if cursor < total_lines:
+        result.append(_make_gap_segment(unit_index, cursor, total_lines - 1))
+
+    for i, seg in enumerate(result):
+        seg.segment_index = i
+
+    return result
+
+
+def fallback_segments(
+    unit_index: int,
+    lines: list[DialogueLine],
+) -> list[SlideSegment]:
+    """Produce minimal valid segments without LLM. Never returns an empty list."""
+    n = len(lines)
+    if n == 0:
+        return [_make_segment(unit_index, 0, 0, 0, "hook_question", "Introduction")]
+
+    segs: list[SlideSegment] = []
+    idx = 0
+
+    hook_end = 0 if n <= 2 else min(1, n - 2)
+    segs.append(_make_segment(unit_index, idx, 0, hook_end, "hook_question", "Opening Question"))
+    idx += 1
+    cursor = hook_end + 1
+
+    while cursor < n - 1:
+        end = min(cursor + 2, n - 2)
+        segs.append(_make_segment(unit_index, idx, cursor, end, "key_insight", "Key Insight"))
+        idx += 1
+        cursor = end + 1
+
+    if cursor <= n - 1:
+        segs.append(_make_segment(unit_index, idx, cursor, n - 1, "memory_hook", "Remember This"))
+    elif len(segs) == 1:
+        segs.append(_make_segment(unit_index, idx, 0, 0, "memory_hook", "Remember This"))
+
+    return segs
+
+
+# ── private helpers ───────────────────────────────────────────────────────────
+
+
+def _validate_segment(seg: SlideSegment) -> SlideSegment:
+    """Post-process a segment: reclassify types that would produce blank slides."""
+    if seg.visual_type == "step_sequence" and not seg.body:
+        log.warning(
+            "segment %d-%d is step_sequence but body is empty — falling back to definition",
+            seg.lines_start,
+            seg.lines_end,
+        )
+        seg.visual_type = "definition"
+        seg.body = seg.title
+    if seg.visual_type == "callout" and not seg.body:
+        log.warning(
+            "segment %d-%d is callout but body is empty — falling back to key_insight",
+            seg.lines_start,
+            seg.lines_end,
+        )
+        seg.visual_type = "key_insight"
+    return seg
+
+
+def _make_segment(
+    unit_index: int,
+    segment_index: int,
+    lines_start: int,
+    lines_end: int,
+    visual_type: str,
+    title: str,
+) -> SlideSegment:
+    return SlideSegment(
+        unit_index=unit_index,
+        segment_index=segment_index,
+        lines_start=lines_start,
+        lines_end=lines_end,
+        visual_type=visual_type,
+        title=title,
+        body=None,
+        code=None,
+        language=None,
+        mermaid=None,
+        left=None,
+        right=None,
+        rows=None,
+    )
+
+
+def _make_gap_segment(unit_index: int, start: int, end: int) -> SlideSegment:
+    return _make_segment(unit_index, -1, start, end, "key_insight", "Key Insight")