lorebase-scribe 0.2.3__py3-none-any.whl

lorebase_scribe/extraction/settings.py

@@ -0,0 +1,293 @@
+"""The `scribe.extraction` settings: tiers, chunking, entity resolution, confidence, budgets.
+
+Models themselves (chat model, span and relation extractors) are configured in the `inference`
+section; this section decides how Scribe uses them.
+"""
+
+from __future__ import annotations
+
+from typing import Literal, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_platform.settings import SectionModel
+
+Tier = Literal[0, 1, 2]
+
+
+class ChunkSettings(SectionModel):
+    """Structure-aware chunks: paragraphs and headings are kept whole when they fit."""
+
+    max_chars: int = Field(
+        default=2000, ge=200, description="Largest chunk, in characters of parsed text."
+    )
+    overlap_chars: int = Field(
+        default=200,
+        ge=0,
+        description="Characters a chunk repeats from the end of the previous one, so a fact "
+        "cut by a boundary is still seen whole.",
+    )
+
+    @model_validator(mode="after")
+    def _overlap_below_size(self) -> Self:
+        if self.overlap_chars * 2 >= self.max_chars:
+            raise ValueError("overlap_chars must be less than half of max_chars")
+        return self
+
+
+class MatchSettings(SectionModel):
+    """Entity resolution of mentions against existing Records (design 9.2 step 9)."""
+
+    trigram_threshold: float = Field(
+        default=0.45, ge=0, le=1, description="Least trigram similarity of a fuzzy title match."
+    )
+    match_threshold: float = Field(
+        default=0.6,
+        ge=0,
+        le=1,
+        description="Least score for a mention to be matched to a Record; below, the mention "
+        "is a new-Record candidate (when its Kind allows it) or dropped.",
+    )
+    ambiguity_margin: float = Field(
+        default=0.05,
+        ge=0,
+        le=1,
+        description="Candidates scoring within this margin of the best one make the match "
+        "ambiguous; the reviewer picks.",
+    )
+    candidates: int = Field(default=5, ge=1, le=50, description="Candidates kept per mention.")
+    dictionary_max_titles: int = Field(
+        default=20000,
+        ge=0,
+        description="Titles of existing Records loaded per job for tier 0 dictionary matching; "
+        "Kinds beyond the budget are matched by keys, tier 1 spans and trigrams only.",
+    )
+    min_title_chars: int = Field(
+        default=4, ge=1, description="Shorter titles are not used for dictionary matching."
+    )
+
+
+class ConfidenceSettings(SectionModel):
+    """Confidence given to candidates by the deterministic tiers and tier 2."""
+
+    key: float = Field(default=1.0, ge=0, le=1, description="A key found by its pattern.")
+    title: float = Field(default=0.95, ge=0, le=1, description="An exact title match.")
+    normalized_title: float = Field(
+        default=0.9, ge=0, le=1, description="A title equal after case and punctuation folding."
+    )
+    value_max: float = Field(
+        default=0.85,
+        ge=0,
+        le=1,
+        description="Ceiling of rule-detected Attribute values (closer cues score higher).",
+    )
+    cooccurrence_max: float = Field(
+        default=0.7, ge=0, le=1, description="Ceiling of Links proposed from co-occurrence."
+    )
+    heading: float = Field(
+        default=0.6,
+        ge=0,
+        le=1,
+        description="Links between the Record a section heading names and the Records its "
+        "text mentions.",
+    )
+    tier2: float = Field(
+        default=0.8, ge=0, le=1, description="Facts returned by tier 2 with verified evidence."
+    )
+
+
+class ParserSettings(SectionModel):
+    """Limits of the process that turns PDF and DOCX bytes into text (design 9.2 step 2).
+
+    A crafted file can make a parser loop or allocate without bound, so PDF and DOCX are parsed
+    in a child process with no network access, limited in wall-clock time, CPU time and address
+    space; a document over `max_input_bytes` is not parsed at all. A document that breaks a
+    limit fails its extraction job with a message naming the setting. Markdown and plain text
+    are decoded in the worker under the same size limit.
+    """
+
+    isolated: bool = Field(
+        default=True,
+        description="Parse PDF and DOCX in a child process with the limits below. Off: in a "
+        "worker thread, bounded only by max_input_bytes.",
+    )
+    timeout_s: float = Field(
+        default=60.0, gt=0, description="Wall-clock time a document may take to parse."
+    )
+    cpu_s: int = Field(
+        default=60, ge=1, description="CPU seconds of the parser process (RLIMIT_CPU, POSIX)."
+    )
+    memory_mb: int = Field(
+        default=1024,
+        ge=128,
+        description="Address space of the parser process in MiB (RLIMIT_AS; Linux and other "
+        "POSIX systems that enforce it).",
+    )
+    max_input_bytes: int = Field(
+        default=50 * 1024 * 1024, ge=1, description="Largest document parsed, in bytes."
+    )
+
+
+class DiscoverySettings(SectionModel):
+    """Schema suggestions (design 9.2 step 10): what extraction reports when it finds types,
+    fields or relations the schema does not have. How suggestions are aggregated, listed and
+    matched against the schema is set in `schema.suggestions`."""
+
+    enabled: bool = Field(
+        default=True, description="Record schema suggestions from extraction jobs."
+    )
+    rules: bool = Field(
+        default=True,
+        description="Tier 0: `**Label:** value` field lines and table columns no Attribute "
+        "matches, and headings naming a thing by a code no Kind's key pattern knows.",
+    )
+    span_labels: tuple[str, ...] = Field(
+        default=(),
+        description="Tier 1: extra labels for the span extractor; spans found with them "
+        "suggest a Kind of that name (e.g. certificate, warehouse).",
+    )
+    relation_labels: tuple[str, ...] = Field(
+        default=(),
+        description="Tier 1: extra labels for the relation extractor; relations found with "
+        "them between Records of known Kinds suggest a Relation of that name.",
+    )
+    tier2: bool = Field(
+        default=True,
+        description="Tier 2: the chat model may report facts outside the schema (`unlisted`) "
+        "with verified evidence, and its answers outside the schema are kept as observations.",
+    )
+    llm_pass: bool = Field(
+        default=False,
+        description="Run a dedicated discovery call per chunk through the "
+        "`inference.llm.tasks.schema_discovery` route (any chat model; the extraction route "
+        "when the task has no override) that proposes Kinds and Attributes from the document "
+        "text. Needs no tier 2. It runs after tier 2 of each document and has its own token cap "
+        "(`llm_max_tokens`), so it never takes tokens from `max_tokens_per_job`.",
+    )
+    llm_max_tokens: int = Field(
+        default=200_000,
+        ge=0,
+        description="The dedicated discovery calls stop once a job used this many tokens "
+        "(input and output, estimated before each call); 0 means no limit. Separate from "
+        "`max_tokens_per_job`, which tier 2 alone spends.",
+    )
+    llm_max_quote_chars: int = Field(
+        default=300,
+        ge=16,
+        description="A proposal of the dedicated call whose evidence quote is longer is discarded.",
+    )
+    llm_max_quote_share: float = Field(
+        default=0.5,
+        gt=0,
+        le=1,
+        description="A proposal whose evidence quote covers more than this share of the "
+        "chunk is discarded: it quotes the text instead of pointing at a fact. Applies only to "
+        "chunks of at least `llm_quote_share_min_chunk_chars` characters.",
+    )
+    llm_quote_share_min_chunk_chars: int = Field(
+        default=400,
+        ge=0,
+        description="Shorter chunks are exempt from `llm_max_quote_share`: a legitimate quote "
+        "can cover most of a short chunk.",
+    )
+    llm_max_items: int = Field(
+        default=20, ge=1, description="Proposals of each list the dedicated call may return."
+    )
+    min_confidence: float = Field(
+        default=0.3, ge=0, le=1, description="Weaker observations are dropped."
+    )
+    field_confidence: float = Field(
+        default=0.6, ge=0, le=1, description="Confidence of a field line or table column."
+    )
+    heading_confidence: float = Field(
+        default=0.5, ge=0, le=1, description="Confidence of a Kind named by a heading."
+    )
+    max_value_chars: int = Field(
+        default=120, ge=8, description="Sample values are cut to this length."
+    )
+    vault_match_max_words: int = Field(
+        default=12,
+        ge=1,
+        le=64,
+        description="Longest word sequence of a quote compared with the keyed fingerprints of "
+        "vaulted values of any Kind; a quote holding one is kept without its evidence and sample.",
+    )
+    vault_match_max_text_words: int = Field(
+        default=400,
+        ge=1,
+        description="A quote or sample with more words is not compared and is treated as "
+        "holding a vaulted value (its evidence is withheld).",
+    )
+    vault_match_max_text_chars: int = Field(
+        default=4000,
+        ge=1,
+        description="Same, by characters.",
+    )
+    vault_match_max_candidates: int = Field(
+        default=200_000,
+        ge=1,
+        description="Distinct word sequences compared per job; past it the remaining quotes "
+        "are treated as holding a vaulted value.",
+    )
+    max_observations_per_job: int = Field(
+        default=1000, ge=1, description="Observations beyond this many are dropped and counted."
+    )
+
+
+class ExtractionSettings(SectionModel):
+    """Document extraction into reviewable Proposals (design 9.2)."""
+
+    tiers: tuple[Tier, ...] = Field(
+        default=(0, 1),
+        min_length=1,
+        description="Tiers a job runs unless the request names others: 0 rules and "
+        "dictionaries, 1 local span and relation models, 2 the configured chat model. Add 2 "
+        "once inference.llm points at a model.",
+    )
+    auto_accept_threshold: float = Field(
+        default=0.9,
+        ge=0,
+        le=1.01,
+        description="Items at or above this confidence are accepted when the Proposal opens "
+        "and committed on the extraction branch; the rest stay pending for review. Above 1 "
+        "accepts nothing automatically.",
+    )
+    min_confidence: float = Field(
+        default=0.3, ge=0, le=1, description="Candidates below this confidence are dropped."
+    )
+    new_record_kinds: tuple[str, ...] | None = Field(
+        default=None,
+        description="Kind api names that may receive new-Record candidates; null allows every "
+        "Kind with a natural key, an empty list allows none.",
+    )
+    value_cue_words: int = Field(
+        default=8,
+        ge=1,
+        description="A rule-detected value counts only when a cue word of its Attribute "
+        "precedes it by at most this many words in the sentence.",
+    )
+    max_items_per_job: int = Field(
+        default=5000, ge=1, description="Items beyond this many are dropped and counted."
+    )
+    max_document_chars: int = Field(
+        default=2_000_000, ge=1, description="Parsed text beyond this length is not extracted."
+    )
+    prompt_version: str = Field(
+        default="extract-v1",
+        min_length=1,
+        description="Recorded in Lineage and in the cache key; change it when prompts change.",
+    )
+    cache: bool = Field(
+        default=True, description="Reuse tier 2 answers for unchanged chunks, schema and model."
+    )
+    max_tokens_per_job: int = Field(
+        default=2_000_000,
+        ge=0,
+        description="Tier 2 stops calling the model once a job used this many tokens; 0 means "
+        "no limit.",
+    )
+    parser: ParserSettings = Field(default_factory=ParserSettings)
+    chunking: ChunkSettings = Field(default_factory=ChunkSettings)
+    matching: MatchSettings = Field(default_factory=MatchSettings)
+    confidence: ConfidenceSettings = Field(default_factory=ConfidenceSettings)
+    discovery: DiscoverySettings = Field(default_factory=DiscoverySettings)

lorebase_scribe/extraction/text.py

@@ -0,0 +1,179 @@
+"""Chunks and sentences of parsed text, with offsets into that text (design 9.2 step 3).
+
+Chunks follow the structure: paragraphs, headings, list items and table rows are kept whole
+when they fit, longer ones are cut at sentence ends and then at whitespace. Consecutive chunks
+overlap by at most `overlap_chars`, starting at a whitespace boundary. A chunk's text is always
+`text[start:end]`, so evidence offsets found in a chunk map back to the document by adding its
+start.
+
+Sentences end at `.`, `!` or `?` followed by whitespace and a character that is not a
+lowercase letter (so `Sp. z o.o.` stays whole), at blank lines, and at line breaks before a
+block marker (`#`, `|`, `-`, `*`, `>`, `1.`) or after a heading or table row. Lines wrapped
+inside a paragraph stay in one sentence.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+
+_BLOCK_START = re.compile(r"\s*(#|\||[-*>]\s|\*\*|\d+[.)]\s)")
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[^\sa-z])")
+_HEADING = re.compile(r"\s*#{1,6}\s")
+
+
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    seq: int
+    start: int
+    end: int
+    text: str
+
+    @property
+    def hash(self) -> bytes:
+        return hashlib.sha256(self.text.encode("utf-8")).digest()
+
+
+def _lines(text: str) -> list[tuple[int, int]]:
+    """`(start, end)` of every line, newline excluded."""
+    bounds: list[tuple[int, int]] = []
+    start = 0
+    for index, char in enumerate(text):
+        if char == "\n":
+            bounds.append((start, index))
+            start = index + 1
+    bounds.append((start, len(text)))
+    return bounds
+
+
+def _groups(text: str) -> list[tuple[int, int]]:
+    """Blocks of lines: paragraphs, headings, list items and table rows."""
+    groups: list[tuple[int, int]] = []
+    current: list[int] | None = None
+    previous_single = False
+    for start, end in _lines(text):
+        line = text[start:end]
+        if not line.strip():
+            if current is not None:
+                groups.append((current[0], current[1]))
+            current, previous_single = None, False
+            continue
+        starts_block = bool(_BLOCK_START.match(line))
+        if current is None or starts_block or previous_single:
+            if current is not None:
+                groups.append((current[0], current[1]))
+            current = [start, end]
+        else:
+            current[1] = end
+        stripped = line.lstrip()
+        previous_single = stripped.startswith(("#", "|"))
+    if current is not None:
+        groups.append((current[0], current[1]))
+    return [_trim(text, start, end) for start, end in groups if text[start:end].strip()]
+
+
+def _trim(text: str, start: int, end: int) -> tuple[int, int]:
+    while start < end and text[start].isspace():
+        start += 1
+    while end > start and text[end - 1].isspace():
+        end -= 1
+    return start, end
+
+
+def sentence_bounds(text: str) -> list[tuple[int, int]]:
+    """`(start, end)` of the sentences of `text`, whitespace trimmed."""
+    sentences: list[tuple[int, int]] = []
+    for start, end in _groups(text):
+        position = start
+        for match in _SENTENCE_END.finditer(text, start, end):
+            sentences.append(_trim(text, position, match.start()))
+            position = match.end()
+        sentences.append(_trim(text, position, end))
+    return [bound for bound in sentences if bound[1] > bound[0]]
+
+
+@dataclass(frozen=True, slots=True)
+class Section:
+    """A Markdown heading and its section, up to the next heading of the same or a higher
+    level; `heading_end` ends the heading line."""
+
+    start: int
+    heading_end: int
+    end: int
+    level: int
+
+
+_HEADING_LINE = re.compile(r"^(#{1,6})\s", re.MULTILINE)
+
+
+def heading_sections(text: str) -> list[Section]:
+    headings = [(m.start(), len(m.group(1))) for m in _HEADING_LINE.finditer(text)]
+    sections: list[Section] = []
+    for index, (start, level) in enumerate(headings):
+        end = next(
+            (later for later, later_level in headings[index + 1 :] if later_level <= level),
+            len(text),
+        )
+        line_end = text.find("\n", start)
+        sections.append(Section(start, len(text) if line_end == -1 else line_end, end, level))
+    return sections
+
+
+def is_heading(text: str, offset: int) -> bool:
+    """Whether `offset` lies on a Markdown heading line."""
+    line_start = text.rfind("\n", 0, offset) + 1
+    return bool(_HEADING.match(text, line_start))
+
+
+def _pieces(text: str, start: int, end: int, size: int) -> list[tuple[int, int]]:
+    """Cut `[start, end)` into pieces of at most `size` characters at sentence ends, then at
+    whitespace, then anywhere."""
+    if end - start <= size:
+        return [(start, end)]
+    units = [(s, e) for s, e in sentence_bounds(text[start:end])]
+    units = [(start + s, start + e) for s, e in units] or [(start, end)]
+    pieces: list[tuple[int, int]] = []
+    for unit_start, unit_end in units:
+        position = unit_start
+        while unit_end - position > size:
+            cut = text.rfind(" ", position + 1, position + size)
+            if cut <= position:
+                cut = position + size
+            pieces.append(_trim(text, position, cut))
+            position = cut
+        pieces.append(_trim(text, position, unit_end))
+    return [piece for piece in pieces if piece[1] > piece[0]]
+
+
+def chunk_text(text: str, *, max_chars: int, overlap_chars: int) -> list[Chunk]:
+    """Chunks of at most `max_chars` characters covering every non-blank part of `text`."""
+    if overlap_chars * 2 >= max_chars:
+        raise ValueError("overlap_chars must be less than half of max_chars")
+    unit_size = max_chars - overlap_chars
+    units = [
+        piece for start, end in _groups(text) for piece in _pieces(text, start, end, unit_size)
+    ]
+    chunks: list[Chunk] = []
+    index = 0
+    previous_end: int | None = None
+    while index < len(units):
+        start = units[index][0]
+        if previous_end is not None and overlap_chars:
+            boundary = text.find(" ", max(previous_end - overlap_chars, 0), previous_end)
+            if boundary != -1:
+                start = min(start, boundary + 1)
+        end = units[index][1]
+        index += 1
+        while index < len(units) and units[index][1] - start <= max_chars:
+            end = units[index][1]
+            index += 1
+        start, end = _trim(text, start, end)
+        chunks.append(Chunk(len(chunks), start, end, text[start:end]))
+        previous_end = end
+    return chunks
+
+
+def normalize_space(text: str) -> str:
+    """Whitespace runs folded to one space, ends stripped (evidence comparison)."""
+    return " ".join(text.split())