learnx-cli 0.3.0__py3-none-any.whl

tutor/ingestion/chunker.py

@@ -0,0 +1,171 @@
+import logging
+import re
+
+from tutor.constants import (
+    MAX_CHUNK_TOKENS,
+    MIN_CHUNK_TOKENS,
+    STRATEGY_C_OVERLAP_TOKENS,
+    STRATEGY_C_WINDOW_TOKENS,
+)
+from tutor.ingestion import parse_content
+from tutor.models import Chunk, DocProfile
+
+log = logging.getLogger(__name__)
+
+
+def chunk(text: str, profile: DocProfile) -> list[Chunk]:
+    if profile.strategy == "A":
+        chunks = _strategy_a(text)
+    elif profile.strategy == "B":
+        chunks = _strategy_b(text)
+    else:
+        chunks = _strategy_c(text)
+    return _apply_quality_rules(chunks)
+
+
+def _slugify(heading: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", heading.lower()).strip("_")
+
+
+def _estimate_tokens(text: str) -> int:
+    return int(len(text.split()) * 1.3)
+
+
+def _strategy_a(text: str) -> list[Chunk]:
+    return [
+        Chunk(
+            chunk_id="full_doc",
+            breadcrumb="Full Document",
+            heading="Full Document",
+            level=0,
+            token_count=_estimate_tokens(text),
+            text=text,
+        )
+    ]
+
+
+def _strategy_b(text: str) -> list[Chunk]:
+    sections = re.split(r"\n(?=## )", text)
+    sections = [s for s in sections if s.strip()]
+
+    if len(sections) < 2:
+        log.warning(
+            "Document has no headings — falling back to Strategy C (sliding window). "
+            "Consider adding ## headings to improve chunk quality."
+        )
+        return _strategy_c(text)
+
+    chunks: list[Chunk] = []
+    for section in sections:
+        lines = section.strip().split("\n")
+        heading_line = lines[0].lstrip("#").strip()
+        chunks.extend(_split_section(section, heading_line, parent_heading=None))
+
+    return chunks
+
+
+def _split_section(section: str, heading: str, parent_heading: str | None) -> list[Chunk]:
+    token_count = _estimate_tokens(section)
+
+    if token_count <= MAX_CHUNK_TOKENS:
+        prefix = f"## {parent_heading}\n\n" if parent_heading else ""
+        return [
+            Chunk(
+                chunk_id=_slugify(heading),
+                breadcrumb=f"{parent_heading} > {heading}" if parent_heading else heading,
+                heading=heading,
+                level=2,
+                token_count=token_count,
+                text=prefix + section,
+            )
+        ]
+
+    subsections = re.split(r"\n(?=### )", section)
+    if len(subsections) < 2:
+        prefix = f"## {parent_heading}\n\n" if parent_heading else ""
+        return [
+            Chunk(
+                chunk_id=_slugify(heading),
+                breadcrumb=heading,
+                heading=heading,
+                level=2,
+                token_count=token_count,
+                text=prefix + section,
+            )
+        ]
+
+    result: list[Chunk] = []
+    for sub in subsections:
+        sub_lines = sub.strip().split("\n")
+        sub_heading = sub_lines[0].lstrip("#").strip()
+        prefix = f"## {heading}\n\n"
+        result.append(
+            Chunk(
+                chunk_id=_slugify(f"{heading}_{sub_heading}"),
+                breadcrumb=f"{heading} > {sub_heading}",
+                heading=sub_heading,
+                level=3,
+                token_count=_estimate_tokens(sub),
+                text=prefix + sub,
+            )
+        )
+    return result
+
+
+def _strategy_c(text: str) -> list[Chunk]:
+    word_window = int(STRATEGY_C_WINDOW_TOKENS / 1.3)
+    word_overlap = int(STRATEGY_C_OVERLAP_TOKENS / 1.3)
+
+    words = text.split()
+    chunks: list[Chunk] = []
+    start = 0
+    idx = 0
+
+    while start < len(words):
+        end = min(start + word_window, len(words))
+        window_words = words[start:end]
+
+        if end < len(words):
+            window_text = " ".join(window_words)
+            last_period = window_text.rfind(". ")
+            if last_period > len(window_text) // 2:
+                window_text = window_text[: last_period + 1]
+                window_words = window_text.split()
+
+        chunk_text = " ".join(window_words)
+        token_count = int(len(window_words) * 1.3)
+
+        chunks.append(
+            Chunk(
+                chunk_id=f"window_{idx:03d}",
+                breadcrumb=f"Window {idx + 1}",
+                heading=f"Window {idx + 1}",
+                level=0,
+                token_count=token_count,
+                text=chunk_text,
+                has_code=False,
+                overlapping=(idx > 0),
+            )
+        )
+
+        idx += 1
+        step = word_window - word_overlap
+        start += max(step, 1)
+
+    return chunks
+
+
+def _apply_quality_rules(chunks: list[Chunk]) -> list[Chunk]:
+    merged: list[Chunk] = []
+    for c in chunks:
+        if c.token_count < MIN_CHUNK_TOKENS and merged:
+            prev = merged[-1]
+            prev.text += "\n\n" + c.text
+            prev.token_count = _estimate_tokens(prev.text)
+        else:
+            merged.append(c)
+
+    for c in merged:
+        parse_content.enrich(c)
+
+    return merged

tutor/ingestion/doc_analyzer.py

@@ -0,0 +1,41 @@
+import re
+from pathlib import Path
+from typing import Literal
+
+from tutor.constants import STRATEGY_A_TOKEN_LIMIT, STRATEGY_B_TOKEN_LIMIT
+from tutor.exceptions import IngestionError
+from tutor.models import DocProfile
+
+
+def analyze(filepath: str) -> DocProfile:
+    path = Path(filepath)
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as e:
+        raise IngestionError(f"Cannot read file: {filepath}") from e
+
+    raw_bytes = path.stat().st_size
+    word_count = len(text.split())
+    estimated_tokens = int(word_count * 1.3)
+
+    strategy: Literal["A", "B", "C"]
+    if estimated_tokens <= STRATEGY_A_TOKEN_LIMIT:
+        strategy = "A"
+    elif estimated_tokens <= STRATEGY_B_TOKEN_LIMIT:
+        strategy = "B"
+    else:
+        strategy = "C"
+
+    section_count = len(re.findall(r"^#{1,3}\s", text, re.MULTILINE))
+    has_code_blocks = "```" in text
+    language_hint = "java" if "```java" in text.lower() else "general"
+
+    return DocProfile(
+        filepath=filepath,
+        raw_bytes=raw_bytes,
+        estimated_tokens=estimated_tokens,
+        strategy=strategy,
+        section_count=section_count,
+        has_code_blocks=has_code_blocks,
+        language_hint=language_hint,
+    )

tutor/ingestion/parse_content.py

@@ -0,0 +1,19 @@
+import re
+
+from tutor.models import Chunk
+
+
+def enrich(chunk: Chunk) -> Chunk:
+    chunk.has_code = "```" in chunk.text
+
+    raw_terms = re.findall(r"\*\*(.+?)\*\*|`(.+?)`", chunk.text)
+    seen: set[str] = set()
+    key_terms: list[str] = []
+    for bold, code in raw_terms:
+        term = bold or code
+        if term and term not in seen:
+            seen.add(term)
+            key_terms.append(term)
+
+    chunk.key_terms = key_terms
+    return chunk

tutor/ingestion/summarizer.py

@@ -0,0 +1,51 @@
+import hashlib
+import logging
+from pathlib import Path
+
+from tutor.constants import PROMPT_VERSION, SUMMARY_CACHE_DIR
+from tutor.infra.llm import LIMITS, LLMFn, load_prompt
+from tutor.models import Chunk
+
+log = logging.getLogger(__name__)
+
+
+def summarize_all(
+    chunks: list[Chunk],
+    llm_fn: LLMFn,
+    cache_dir: str = SUMMARY_CACHE_DIR,
+) -> list[Chunk]:
+    cache_path = Path(cache_dir)
+    cache_path.mkdir(parents=True, exist_ok=True)
+
+    prompt_text = load_prompt("summarize.txt")
+
+    for c in chunks:
+        if c.chunk_id == "full_doc":
+            c.summary = c.text[:500]
+            continue
+
+        cache_key = hashlib.md5((c.text + PROMPT_VERSION).encode()).hexdigest()
+        cache_file = cache_path / f"{cache_key}.summary.txt"
+
+        if cache_file.exists():
+            c.summary = cache_file.read_text(encoding="utf-8")
+            log.debug("Cache hit for chunk %s", c.chunk_id)
+            continue
+
+        log.info("Summarizing chunk %s (%d tokens)", c.chunk_id, c.token_count)
+        chunk_text = _truncate_to_tokens(c.text, LIMITS["max_summarize_input_tokens"])
+        messages = [
+            {"role": "system", "content": prompt_text},
+            {"role": "user", "content": chunk_text},
+        ]
+        summary = llm_fn(messages, call_type="summarize")
+        cache_file.write_text(summary, encoding="utf-8")
+        c.summary = summary
+
+    return chunks
+
+
+def _truncate_to_tokens(text: str, max_tokens: int) -> str:
+    max_words = int(max_tokens / 1.3)
+    words = text.split()
+    return " ".join(words[:max_words]) if len(words) > max_words else text

tutor/inspector.py

@@ -0,0 +1,117 @@
+from tutor.constants import WPM
+from tutor.models import Chunk, DocProfile, TeachingUnit
+
+
+def report_ingestion(profile: DocProfile, chunks: list[Chunk]) -> None:
+    print("\n=== Ingestion Report ===")
+    print(f"File:              {profile.filepath}")
+    print(f"Raw size:          {profile.raw_bytes:,} bytes")
+    print(f"Estimated tokens:  {profile.estimated_tokens:,}")
+    print(f"Strategy:          {profile.strategy}")
+    print(f"Sections found:    {profile.section_count}")
+    print(f"Chunks created:    {len(chunks)}")
+
+    if chunks:
+        avg = sum(c.token_count for c in chunks) // len(chunks)
+        largest = max(chunks, key=lambda c: c.token_count)
+        code_count = sum(1 for c in chunks if c.has_code)
+        print(f"  Avg chunk size:  {avg} tokens")
+        print(f"  Largest chunk:   {largest.token_count} tokens  ({largest.chunk_id})")
+        print(f"  Chunks with code: {code_count}/{len(chunks)}")
+
+    print("\n=== Chunk Map ===")
+    print(f"{'ID':<25} {'Heading':<35} {'Tokens':>7}  {'Code'}")
+    print("-" * 75)
+    for c in chunks:
+        code_flag = "yes" if c.has_code else "no"
+        print(f"{c.chunk_id:<25} {c.heading:<35} {c.token_count:>7}  {code_flag}")
+
+    _report_warnings(chunks)
+    _report_orphans(chunks)
+
+
+def _report_warnings(chunks: list[Chunk]) -> None:
+    from tutor.constants import MAX_CHUNK_TOKENS
+
+    warnings = []
+    for c in chunks:
+        if c.token_count > MAX_CHUNK_TOKENS and c.has_code:
+            warnings.append(
+                f"! {c.chunk_id} — code block preserved intact at {c.token_count} tokens (correct behavior)."
+            )
+        elif c.token_count > MAX_CHUNK_TOKENS:
+            warnings.append(
+                f"! {c.chunk_id} — {c.token_count} tokens, may produce shallow dialogue."
+            )
+
+    if warnings:
+        print("\n=== Chunk Quality Warnings ===")
+        for w in warnings:
+            print(w)
+
+
+def _report_orphans(chunks: list[Chunk]) -> None:
+    orphans = [c for c in chunks if c.token_count < 200]
+    if orphans:
+        print("\n=== Orphan Risk ===")
+        for c in orphans:
+            print(
+                f"  {c.chunk_id} ({c.token_count} tokens) — small section, may be skipped by planner"
+            )
+
+
+def report_curriculum(
+    units: list[TeachingUnit],
+    chunks: list[Chunk],
+    duration_min: int,
+) -> None:
+    print("\n=== Duration Plan ===")
+    print(f"Target duration:   {duration_min} min")
+    print(f"Word budget:       {duration_min * WPM} words (@ {WPM} WPM)")
+    print("Silence overhead:  ~1m 20s")
+
+    print("\n=== Teaching Units ===")
+    header = f"{'':45} {'Complexity':>10}  {'Words':>7}  {'Est. time'}"
+    print(header)
+    print("-" * 80)
+
+    intro_words = 100
+    intro_secs = intro_words * 60 // WPM
+    print(f"{'Intro':<45} {'—':>10}  {intro_words:>7}  {_fmt_time(intro_secs)}")
+
+    total_words = intro_words
+    total_secs = intro_secs
+
+    for u in units:
+        secs = u.word_budget * 60 // WPM
+        label = f'Unit {u.unit}  "{u.concept}"'
+        print(f"{label:<45} {u.complexity:>10}  {u.word_budget:>7}  {_fmt_time(secs)}")
+        total_words += u.word_budget
+        total_secs += secs
+
+    outro_words = 80
+    outro_secs = outro_words * 60 // WPM
+    print(f"{'Outro (memory hook recap)':<45} {'—':>10}  {outro_words:>7}  {_fmt_time(outro_secs)}")
+    total_words += outro_words
+    total_secs += outro_secs + 80  # silence overhead
+
+    print("-" * 80)
+    print(f"{'Total':<45} {'':>10}  {total_words:>7}  {_fmt_time(total_secs)}")
+
+    used_ids = {sid for u in units for sid in u.source_sections}
+    used = sum(1 for c in chunks if c.chunk_id in used_ids)
+    pct = used / len(chunks) * 100 if chunks else 0
+    skipped = [c.chunk_id for c in chunks if c.chunk_id not in used_ids]
+
+    print("\n=== Coverage ===")
+    print(f"Sections used:     {used}/{len(chunks)} ({pct:.1f}%)")
+    if skipped:
+        print(f"Sections skipped:  {', '.join(skipped[:8])}")
+        if len(skipped) > 8:
+            print(f"                   ... and {len(skipped) - 8} more")
+
+
+def _fmt_time(seconds: int) -> str:
+    m = seconds // 60
+    s = seconds % 60
+    return f"{m}m {s:02d}s"

tutor/llm_config.toml

@@ -0,0 +1,58 @@
+# ============================================================
+#  LearnX LLM configuration
+#  Edit this file to change models or token budgets.
+#  No Python code changes required.
+# ============================================================
+
+# ── Model selection ─────────────────────────────────────────
+# One model name per call_type per provider.
+# call_types: curriculum | dialogue | summarize | qa
+
+[providers.groq]
+curriculum = "llama-3.3-70b-versatile"
+dialogue   = "llama-3.3-70b-versatile"
+summarize  = "llama-3.1-8b-instant"
+qa         = "llama-3.1-8b-instant"
+visual     = "llama-3.3-70b-versatile"    # 70b needed for reliable DOT output
+segments   = "llama-3.3-70b-versatile"
+
+[providers.openrouter]
+curriculum = "poolside/laguna-xs.2:free"
+dialogue   = "poolside/laguna-xs.2:free"
+summarize  = "poolside/laguna-xs.2:free"
+qa         = "poolside/laguna-xs.2:free"
+visual     = "poolside/laguna-xs.2:free"
+segments   = "poolside/laguna-xs.2:free"
+
+# ── Response token caps ──────────────────────────────────────
+# Max tokens the model may generate per call type.
+# Together with the input prompt they must stay under the
+# provider's per-request token limit (Groq free tier: 6 000).
+
+[max_tokens]
+curriculum = 2000
+dialogue   = 2000
+summarize  = 400
+qa         = 600
+visual     = 1200
+segments   = 2000
+
+# ── Input size limits ────────────────────────────────────────
+# max_source_tokens:          source text sent with each dialogue prompt
+# max_summarize_input_tokens: chunk text sent to the summariser
+#
+# dialogue uses llama-3.3-70b-versatile (128k context on Groq) or owl-alpha (OpenRouter).
+# 8000 source + ~1200 prompt overhead + 2000 response = ~11200 tokens — fine for both.
+# max_summarize_input_tokens kept at 3000; those calls send much smaller chunks.
+
+[limits]
+max_source_tokens          = 8000
+max_summarize_input_tokens = 3000
+max_visual_source_tokens   = 800
+
+# ── Call behaviour ───────────────────────────────────────────
+
+[llm]
+temperature   = 0.7
+retry_count   = 3       # attempts before giving up
+retry_delay_s = 2.0     # seconds between retries

tutor/models.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+
+@dataclass
+class DocProfile:
+    filepath: str
+    raw_bytes: int
+    estimated_tokens: int
+    strategy: Literal["A", "B", "C"]
+    section_count: int
+    has_code_blocks: bool
+    language_hint: str
+
+
+@dataclass
+class Chunk:
+    chunk_id: str
+    breadcrumb: str
+    heading: str
+    level: int
+    token_count: int
+    text: str
+    has_code: bool = False
+    summary: str = ""
+    overlapping: bool = False
+    key_terms: list[str] = field(default_factory=list)
+
+
+@dataclass
+class TeachingUnit:
+    unit: int
+    concept: str
+    source_sections: list[str]
+    complexity: int  # 1 | 2 | 3
+    word_budget: int
+    key_facts: list[str]
+    common_misconception: str
+    good_analogy: str
+    question_style: str
+    memory_hook: str
+    prerequisite_concepts: list[str] = field(default_factory=list)
+    js_contrast: str = ""
+    production_relevance: str = ""
+
+
+@dataclass
+class DialogueLine:
+    speaker: str  # "ALEX" | "MAYA" | "SAM"
+    text: str
+    unit_number: int  # 0 = intro, 1+ = unit, -1 = outro
+
+
+@dataclass
+class RenderedSegment:
+    line: DialogueLine
+    audio_path: str
+    duration_ms: int
+
+
+@dataclass
+class QAExchange:
+    id: int
+    unit_number: int
+    unit_concept: str
+    position_seconds: int
+    question: str
+    answer: str
+    source_sections: list[str]
+    timestamp: str
+
+
+@dataclass
+class SessionLog:
+    source_file: str
+    session_start: str
+    format: str
+    duration_minutes: int
+    exchanges: list[QAExchange] = field(default_factory=list)
+
+
+@dataclass
+class TimingEntry:
+    line_index: int  # 0-based within the unit
+    speaker: str  # "ALEX" | "MAYA" | "SAM"
+    text: str  # dialogue line text — for cross-referencing only
+    start_ms: int  # offset from unit MP3 start, in milliseconds
+    end_ms: int  # exclusive end; end_ms - start_ms == len(audio) in ms
+
+
+VALID_VISUAL_TYPES: frozenset[str] = frozenset(
+    {
+        "hook_question",
+        "definition",
+        "analogy",
+        "comparison",
+        "code_example",
+        "diagram",
+        "question_prompt",
+        "decision_guide",
+        "key_insight",
+        "memory_hook",
+        "step_sequence",
+        "callout",
+    }
+)
+
+
+@dataclass
+class SlideSegment:
+    unit_index: int
+    segment_index: int
+    lines_start: int
+    lines_end: int
+    visual_type: str
+    title: str
+    body: str | None
+    code: str | None
+    language: str | None
+    mermaid: str | None
+    left: str | None
+    right: str | None
+    rows: list | None
+    png_path: str = ""
+
+
+@dataclass
+class VisualSpec:
+    unit_index: int
+    slide_type: str  # "title_card" | "unit" | "outro"
+    concept: str = ""
+    hook_question: str = ""
+    key_points: list[str] = field(default_factory=list)
+    code_snippet: str | None = None
+    diagram_type: str = "none"
+    diagram_spec: str | dict[str, object] | None = None
+    memory_hook: str = ""
+    analogy: str = ""
+    # title_card fields
+    title: str = ""
+    subtitle: str = ""
+    doc_source: str = ""
+    # outro fields
+    memory_hooks: list[str] = field(default_factory=list)
+    session_stats: str = ""

tutor/player/input_handler.py

@@ -0,0 +1,45 @@
+import logging
+import sys
+
+log = logging.getLogger(__name__)
+
+
+def get_key() -> str | None:
+    """Return the pressed key as a string, or None if no key is available."""
+    if sys.platform == "win32":
+        return _get_key_windows()
+    return _get_key_unix()
+
+
+def _get_key_windows() -> str | None:
+    import msvcrt
+
+    if msvcrt.kbhit():
+        raw = msvcrt.getch()
+        try:
+            return raw.decode("utf-8")
+        except UnicodeDecodeError:
+            return None
+    return None
+
+
+def _get_key_unix() -> str | None:
+    try:
+        import threading
+
+        import readchar
+
+        result: list[str | None] = [None]
+
+        def _read() -> None:
+            result[0] = readchar.readchar()
+
+        t = threading.Thread(target=_read, daemon=True)
+        t.start()
+        t.join(timeout=0.05)
+        return result[0]
+    except ImportError:
+        log.warning(
+            "readchar not installed — keyboard input unavailable on non-Windows. pip install readchar"
+        )
+        return None