booksmart-core 0.1.0__py3-none-any.whl

booksmart_core/__init__.py

@@ -0,0 +1,13 @@
+"""booksmart-core: the book-ingestion pipeline as a library.
+
+The alembic history and its config ship inside the package (not just the sdist)
+so any consumer installing core as a wheel — the CLI, booksmart-api — can locate
+and run the single migration history from the installed location.
+"""
+
+from pathlib import Path
+
+MIGRATIONS_PATH = Path(__file__).resolve().parent / "migrations"
+ALEMBIC_INI_PATH = Path(__file__).resolve().parent / "alembic.ini"
+
+__all__ = ["ALEMBIC_INI_PATH", "MIGRATIONS_PATH"]

booksmart_core/alembic.ini

@@ -0,0 +1,38 @@
+[alembic]
+script_location = migrations
+prepend_sys_path = .
+path_separator = os
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

booksmart_core/config.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="BOOKSMART_")
+
+    database_url: str = "postgresql+psycopg://booksmart:booksmart@localhost:5432/booksmart"
+    storage_root: Path = Path("storage")
+
+    # Vector store location. A server URL by default (booksmart-api's shape); set
+    # qdrant_path to run Qdrant embedded on-disk instead (the CLI's default, no
+    # service). When qdrant_path is set it wins and qdrant_url is ignored.
+    qdrant_url: str = "http://localhost:6333"
+    qdrant_path: Path | None = None
+
+    llm_provider: str = "anthropic"
+    llm_model: str | None = None  # None -> the selected provider's default model
+    # Reasoning/thinking control for OpenAI-compatible providers ("none", "low",
+    # "medium", "high"). "none" disables Gemini 2.5 Flash thinking so structured
+    # stages don't spend the completion budget deliberating; gemini-2.5-pro
+    # rejects "none". Ignored by the anthropic and fake providers.
+    llm_reasoning_effort: str | None = None
+    embedding_provider: str = "openai"  # Anthropic has no embeddings API
+    embedding_model: str | None = None
+    # API keys; when unset the providers fall back to the conventional
+    # ANTHROPIC_API_KEY / OPENAI_API_KEY / GEMINI_API_KEY environment variables.
+    anthropic_api_key: str | None = None
+    openai_api_key: str | None = None
+    gemini_api_key: str | None = None

booksmart_core/database.py

@@ -0,0 +1,46 @@
+"""Engine construction and migration helpers a consumer uses to stand up a
+database before driving the pipeline.
+
+A Runner owns engine/session lifecycle (core exposes only Stage functions over a
+``Session``). These helpers give a consumer — the CLI, or any embedded user — a
+correct SQLite engine (foreign keys enforced, so the stages' ON DELETE CASCADE
+works) and a one-call migration to head against the single packaged history.
+booksmart-api runs Postgres and manages its own engine, so it needs neither.
+"""
+
+from sqlalchemy import Engine, event
+from sqlalchemy import create_engine as _sa_create_engine
+
+from booksmart_core import MIGRATIONS_PATH
+
+
+def create_engine(url: str) -> Engine:
+    """A SQLAlchemy engine that enforces SQLite foreign keys.
+
+    SQLite honours foreign keys (and thus ON DELETE CASCADE) only when asked,
+    per connection; without this the structure/extraction stages' bulk deletes
+    leave orphaned rows. A no-op for other dialects."""
+    engine = _sa_create_engine(url)
+    if engine.dialect.name == "sqlite":
+
+        @event.listens_for(engine, "connect")
+        def _enable_sqlite_foreign_keys(dbapi_conn: object, _: object) -> None:
+            cursor = dbapi_conn.cursor()  # type: ignore[attr-defined]
+            cursor.execute("PRAGMA foreign_keys=ON")
+            cursor.close()
+
+    return engine
+
+
+def upgrade_to_head(url: str) -> None:
+    """Migrate the database at ``url`` to head using core's single packaged
+    alembic history. Idempotent — a database already at head is a no-op — so a
+    consumer can call it unconditionally on startup."""
+    # Imported lazily: alembic pulls in a fair bit, and only migration paths need it.
+    from alembic import command
+    from alembic.config import Config as AlembicConfig
+
+    cfg = AlembicConfig()
+    cfg.set_main_option("script_location", str(MIGRATIONS_PATH))
+    cfg.set_main_option("sqlalchemy.url", url)
+    command.upgrade(cfg, "head")

booksmart_core/errors.py

@@ -0,0 +1,61 @@
+"""Booksmart's error taxonomy (API-first).
+
+Every error the pipeline raises on purpose carries a class-level ``retriable``
+flag so a Runner can map it to its own retry semantics without parsing message
+strings: booksmart-api turns ``retriable=False`` into Inngest's
+``NonRetriableError``; the CLI renders one clean line. Vendor SDK exceptions
+(anthropic/openai/qdrant network errors and the like) are *not* wrapped — they
+pass through untouched, and a Runner treats an unrecognised exception as it
+sees fit.
+
+Non-retriable:
+- ``ProviderConfigError`` — a Preference conflicts with a Limit, or the
+  configuration is otherwise deterministically wrong.
+- ``StagePreconditionError`` — the data a stage needs is absent (missing book
+  or parsed artifact, unknown scope). Retrying the same call cannot fix it.
+
+Retriable:
+- ``ProviderResponseError`` — the model returned nothing usable (a refusal, an
+  empty completion, or a response still unparseable after the stage's own
+  single retry). A fresh attempt may well succeed.
+"""
+
+
+class BooksmartError(Exception):
+    """Base for every error booksmart raises deliberately.
+
+    ``retriable`` is a class-level fact about the failure mode, not an instance
+    detail: a whole subclass is either retriable or it is not.
+    """
+
+    retriable: bool = False
+
+
+class ProviderConfigError(BooksmartError, ValueError):
+    """A Preference conflicts with a Limit (or the configuration is otherwise
+    deterministically wrong) — at provider construction or at the first write
+    the model-locked vector collection rejects. Retrying cannot fix it.
+
+    Also a ``ValueError`` so callers that already catch configuration mistakes
+    that way keep working.
+    """
+
+    retriable = False
+
+
+class StagePreconditionError(BooksmartError):
+    """A stage was asked to run before the data it depends on exists: the book
+    row is gone, the parsed markdown artifact is missing, or the scope is
+    unknown. A Runner cannot fix this by retrying the same stage."""
+
+    retriable = False
+
+
+class ProviderResponseError(BooksmartError):
+    """A model provider returned no usable result — a refusal, an empty
+    completion, or a response still unparseable after the stage's own in-line
+    retry. Subsumes the narrower LLMError / ExtractionError / SummaryError,
+    which remain as descriptive subclasses. Retriable: a fresh call may
+    succeed."""
+
+    retriable = True

booksmart_core/extraction.py

@@ -0,0 +1,187 @@
+"""Knowledge object extraction from parsed chapter text.
+
+The LLM must return a strict JSON array; an unusable response raises
+ExtractionError rather than persisting objects with broken provenance, while
+a single invalid element is dropped with a reason so one mislabeled object
+does not cost a whole run. Bump EXTRACTION_PROMPT_VERSION whenever the prompt
+wording changes so stored objects record exactly what produced them.
+"""
+
+import json
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from typing import Literal, get_args
+
+from booksmart_core.errors import ProviderResponseError
+from booksmart_core.llm import strip_fences
+from booksmart_core.models import Book, Chapter, Section
+
+EXTRACTION_PROMPT_VERSION = "2"
+
+KnowledgeType = Literal[
+    "Practice",
+    "Principle",
+    "Tradeoff",
+    "Anti-pattern",
+    "Smell",
+    "Decision Rule",
+    "Definition",
+    "Glossary",
+    "Checklist",
+]
+
+KNOWLEDGE_OBJECT_TYPES: frozenset[str] = frozenset(get_args(KnowledgeType))
+
+EXTRACTION_SYSTEM_PROMPT = (
+    "You extract candidate knowledge objects from technical book chapters for "
+    "a knowledge repository. Respond with a JSON array only - no prose, no "
+    "markdown fences. Each element must be an object with fields: "
+    '"type" (one of: ' + ", ".join(sorted(KNOWLEDGE_OBJECT_TYPES)) + "), "
+    '"title" (a short name), "content" (the full idea in the book\'s own terms), '
+    '"summary" (one sentence), "confidence" (number from 0.0 to 1.0), '
+    '"section_index" (0-based index into the numbered section list, or null when '
+    "the idea is not tied to one section), "
+    '"page" (integer, only when the text carries an explicit page marker, else null), '
+    '"paragraph" (integer, only when the paragraph is unambiguous, else null). '
+    "Never guess page or paragraph numbers. Extract only ideas the chapter "
+    "actually asserts; return [] for a chapter with none. Use only the listed "
+    "types, exactly as written - never invent new type names, even when the "
+    "book uses its own vocabulary. Classify warning signs (what some books "
+    'call "red flags") as Smell.'
+)
+
+REQUIRED_FIELDS = ("type", "title", "content", "summary", "confidence")
+
+
+class ExtractionError(ProviderResponseError):
+    """The LLM response could not be turned into valid knowledge objects. A
+    retriable ProviderResponseError: when it escapes a stage (unparseable even
+    after the stage's own retry), a fresh attempt may still succeed."""
+
+
+@dataclass(frozen=True)
+class ExtractedObject:
+    type: str
+    title: str
+    content: str
+    summary: str
+    confidence: float
+    section_index: int | None
+    page: int | None
+    paragraph: int | None
+
+
+def _optional_int(item: dict[str, object], field: str, position: int) -> int | None:
+    value = item.get(field)
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ExtractionError(f"element {position}: {field!r} must be an integer or null")
+    return value
+
+
+def _parse_element(item: object, position: int) -> ExtractedObject:
+    if not isinstance(item, dict):
+        raise ExtractionError(f"element {position} is not a JSON object")
+    for field in REQUIRED_FIELDS:
+        if field not in item:
+            raise ExtractionError(f"element {position} is missing {field!r}")
+    if item["type"] not in KNOWLEDGE_OBJECT_TYPES:
+        raise ExtractionError(f"element {position} has unsupported type {item['type']!r}")
+    confidence = item["confidence"]
+    if isinstance(confidence, bool) or not isinstance(confidence, (int, float)):
+        raise ExtractionError(f"element {position}: 'confidence' must be a number")
+    return ExtractedObject(
+        type=str(item["type"]),
+        title=str(item["title"]),
+        content=str(item["content"]),
+        summary=str(item["summary"]),
+        confidence=float(confidence),
+        section_index=_optional_int(item, "section_index", position),
+        page=_optional_int(item, "page", position),
+        paragraph=_optional_int(item, "paragraph", position),
+    )
+
+
+def parse_extraction_response(text: str) -> tuple[list[ExtractedObject], list[str]]:
+    """The response's valid knowledge objects, plus a drop reason per invalid
+    element. An unusable response as a whole (not JSON, not an array) still
+    raises ExtractionError; a single bad element only costs that element."""
+    payload = strip_fences(text.strip())
+    try:
+        data = json.loads(payload)
+    except json.JSONDecodeError as exc:
+        raise ExtractionError(f"response is not valid JSON: {exc}") from exc
+    if not isinstance(data, list):
+        raise ExtractionError("response must be a JSON array of knowledge objects")
+
+    objects: list[ExtractedObject] = []
+    dropped: list[str] = []
+    for position, item in enumerate(data):
+        try:
+            objects.append(_parse_element(item, position))
+        except ExtractionError as exc:
+            dropped.append(str(exc))
+    return objects, dropped
+
+
+def chapter_body(markdown: str, start_line: int | None, next_start_line: int | None) -> str:
+    """The chapter's slice of the parsed markdown, heading line included.
+
+    Line numbers are 1-based, as recorded by structure detection. A chapter
+    without a recorded start falls back to the whole document.
+    """
+    if start_line is None:
+        return markdown
+    lines = markdown.splitlines()
+    end = next_start_line - 1 if next_start_line is not None else len(lines)
+    return "\n".join(lines[start_line - 1 : end])
+
+
+def iter_chapter_bodies(
+    chapters: Sequence[Chapter], markdown: str
+) -> Iterator[tuple[Chapter, str]]:
+    """Each chapter paired with its slice of the parsed markdown."""
+    for index, chapter in enumerate(chapters):
+        next_chapter = chapters[index + 1] if index + 1 < len(chapters) else None
+        yield (
+            chapter,
+            chapter_body(
+                markdown, chapter.source_line, next_chapter.source_line if next_chapter else None
+            ),
+        )
+
+
+def resolve_source(chapter: Chapter, section_index: int | None) -> tuple[Section | None, str]:
+    """The section the LLM pointed at (None when absent or out of range) and the
+    human-readable source location string."""
+    section = None
+    if section_index is not None and 0 <= section_index < len(chapter.sections):
+        section = chapter.sections[section_index]
+    source_location = f"chapter {chapter.position + 1}: {chapter.title}"
+    if section is not None:
+        source_location += f" > {section.title}"
+    return section, source_location
+
+
+def build_chapter_prompt(book: Book, chapter: Chapter, body: str, instruction: str) -> str:
+    """Shared prompt scaffold for per-chapter stages: book/chapter header, the
+    numbered section list (indexes matter - both extraction's section_index and
+    the summary stage's section alignment refer to it), the chapter text, and
+    the stage's closing instruction."""
+    lines = [
+        f"Book: {book.title} by {book.author}",
+        f"Chapter {chapter.position + 1}: {chapter.title}",
+        "",
+        "Numbered sections in this chapter:",
+    ]
+    if chapter.sections:
+        lines.extend(f"{index}. {section.title}" for index, section in enumerate(chapter.sections))
+    else:
+        lines.append("(none detected)")
+    lines.extend(["", "Chapter text:", body, "", instruction])
+    return "\n".join(lines)
+
+
+def build_extraction_prompt(book: Book, chapter: Chapter, body: str) -> str:
+    return build_chapter_prompt(book, chapter, body, "Extract the knowledge objects as JSON.")

booksmart_core/fakes.py

@@ -0,0 +1,89 @@
+"""Deterministic fake providers for CI and local development.
+
+Selected like any real provider (BOOKSMART_LLM_PROVIDER=fake), so the compose
+smoke test can drive the whole pipeline with no API keys, no network, and no
+cost. Responses are keyed by the stage's system prompt and shaped exactly as
+the stage's parser expects.
+"""
+
+import json
+
+from booksmart_core.extraction import EXTRACTION_SYSTEM_PROMPT
+from booksmart_core.llm import EmbeddingLimits, LLMLimits, LLMResponse, resolve_limits
+from booksmart_core.profile import PROFILE_SYSTEM_PROMPT
+from booksmart_core.summaries import SUMMARY_SYSTEM_PROMPT
+
+FAKE_LLM_MODEL = "fake-llm-1"
+FAKE_EMBEDDING_MODEL = "fake-embed-1"
+FAKE_EMBEDDING_SIZE = 8
+
+# Fakes carry Limits like any real provider so pipeline code exercised against
+# them (batching, budgets) behaves exactly as it will in production.
+_FAKE_LLM_LIMITS = {
+    FAKE_LLM_MODEL: LLMLimits(max_output_tokens=32000),
+}
+_FAKE_LLM_DEFAULT = LLMLimits(max_output_tokens=32000)
+
+_FAKE_EMBEDDING_LIMITS = {
+    FAKE_EMBEDDING_MODEL: EmbeddingLimits(
+        max_batch=100, embedding_dimensions=FAKE_EMBEDDING_SIZE
+    ),
+}
+_FAKE_EMBEDDING_DEFAULT = EmbeddingLimits(max_batch=100)
+
+# One well-formed knowledge object per chapter, so the extraction stage's
+# parsing and persistence run for real.
+FAKE_KNOWLEDGE_OBJECTS = [
+    {
+        "type": "Principle",
+        "title": "Fake determinism",
+        "content": "Fake providers return the same output for every call.",
+        "summary": "Deterministic canned responses.",
+        "confidence": 1.0,
+        "section_index": None,
+        "page": None,
+        "paragraph": None,
+    }
+]
+
+STAGE_RESPONSES: dict[str, str] = {
+    PROFILE_SYSTEM_PROMPT: (
+        "A deterministic fake book profile: this book covers the smoke-test "
+        "topic end to end."
+    ),
+    EXTRACTION_SYSTEM_PROMPT: json.dumps(FAKE_KNOWLEDGE_OBJECTS),
+    # Missing section summaries are padded with None by the summary parser,
+    # so the empty list stays valid for any section count.
+    SUMMARY_SYSTEM_PROMPT: json.dumps(
+        {"chapter_summary": "A deterministic fake chapter summary.", "section_summaries": []}
+    ),
+}
+
+DEFAULT_RESPONSE = "A deterministic fake response."
+
+
+class FakeLLMProvider:
+    def __init__(self, model: str = FAKE_LLM_MODEL) -> None:
+        self.model = model
+        limits = resolve_limits("fake", model, _FAKE_LLM_LIMITS, _FAKE_LLM_DEFAULT)
+        self.max_output_tokens = limits.max_output_tokens
+
+    def complete(self, prompt: str, *, system: str | None = None) -> LLMResponse:
+        text = STAGE_RESPONSES.get(system or "", DEFAULT_RESPONSE)
+        return LLMResponse(text=text, model=self.model, input_tokens=0, output_tokens=0)
+
+
+class FakeEmbeddingProvider:
+    def __init__(self, model: str = FAKE_EMBEDDING_MODEL) -> None:
+        self.model = model
+        limits = resolve_limits("fake", model, _FAKE_EMBEDDING_LIMITS, _FAKE_EMBEDDING_DEFAULT)
+        self.max_batch = limits.max_batch
+        self.embedding_dimensions = limits.embedding_dimensions
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Fixed-size vectors derived from text length: deterministic, and
+        distinct texts usually get distinct vectors."""
+        return [
+            [float((len(text) + position) % 7 + 1) for position in range(FAKE_EMBEDDING_SIZE)]
+            for text in texts
+        ]