structured-data-extraction 0.1.0__py3-none-any.whl

structured_data_extraction/tables.py

@@ -0,0 +1,406 @@
+"""Table complexity scoring and vision-based refinement.
+
+Docling's table parser sometimes garbles complex tables: merged cells get lost,
+columns misalign, and symbol-heavy rows (units, ±, subscripts) collapse into
+nonsense. This module scores each table's structural complexity and, for tables
+above VISION_COMPLEXITY_THRESHOLD, sends the original PDF crop to a vision-
+capable LLM to reconstruct clean HTML.
+
+Two refinement passes exist:
+  1. Pre-embed (refine_symbol_only_tables) — tables whose cells are almost
+     entirely symbols (word-cell ratio < 10%) are refined *before* embedding.
+     Without real words their embeddings would be meaningless and search could
+     never retrieve them.
+  2. Post-search (refine_matched_tables) — tables that search actually matched
+     are refined lazily, so the LLM cost is paid only for tables the user's
+     keys care about.
+
+The complexity score combines sparsity, ragged column counts, vertical merges,
+hierarchy (group-header rows), table width, and HTML keywords. Vision refinement
+renders the PDF bounding box via PyMuPDF, sends the image to the vision model,
+and expands any remaining rowspan/colspan into flat repeated cells so downstream
+extraction sees a regular grid.
+"""
+import base64
+import concurrent.futures
+import json
+import logging
+import os
+import re
+from collections import Counter
+from html.parser import HTMLParser
+from pathlib import Path
+import fitz
+from openai import OpenAI
+from . import config
+
+logger = logging.getLogger(__name__)
+
+
+def _ramp(x, lo, hi):
+    """Return a value between 0.0 and 1.0, ramping linearly from lo to hi."""
+    if x <= lo:
+        return 0.0
+    if x >= hi:
+        return 1.0
+    return (x - lo) / (hi - lo)
+
+
+def _keyword_score(text):
+    """Return a score between 0.0 and 1.0 based on the presence of keywords that indicate complexity."""
+    keywords = ("colspan", "rowspan", "<figure", "<img", "nested")
+    return sum(0.15 for kw in keywords if kw in text)
+
+
+def _pipe_stats(content):
+    """Return (ragged, sparsity, hierarchy, ncols, merges) for a pipe-delimited table,
+    or None if the content has no pipe rows."""
+    pipe_rows = [line for line in content.split("\n") if "|" in line]
+    if not pipe_rows:
+        return None
+
+    delims = [row.count("|") for row in pipe_rows]
+    mode = Counter(delims).most_common(1)[0][0]
+    ragged = sum(1 for d in delims if d != mode) / len(delims)
+
+    # sparsity + hierarchy over data rows (separators skipped)
+    data = []
+    for row in pipe_rows:
+        inner = [c.strip() for c in row.split("|")[1:-1]]
+        if inner and all(set(c) <= {"-", ":", " ", ""} for c in inner):
+            continue
+        data.append(inner)
+
+    cells = sum(len(r) for r in data)
+    empty = sum(1 for r in data for c in r if not c)
+    sparsity = empty / cells if cells else 0.0
+    group_rows = sum(1 for r in data if r and r[0] and not any(r[1:]))
+    hierarchy = group_rows / len(data) if data else 0.0
+    ncols = max(mode - 1, 1)
+
+    # vertical merges: empty cells below a filled cell in the same column
+    spanned = 0
+    width = max((len(r) for r in data), default=0)
+    for col in range(width):
+        filled_above = False
+        for r in data:
+            cell = r[col] if col < len(r) else ""
+            if cell:
+                filled_above = True
+            elif filled_above:
+                spanned += 1
+    merges = spanned / cells if cells else 0.0
+
+    return ragged, sparsity, hierarchy, ncols, merges
+
+
+def score_table_complexity(content):
+    text = content.lower()
+    score = _keyword_score(text)
+
+    stats = _pipe_stats(content)
+    if stats is not None:
+        ragged, sparsity, hierarchy, ncols, merges = stats
+        score += _ramp(sparsity, 0.15, 0.60) * 0.40    # sparse grid
+        score += ragged * 0.30                         # broken rows
+        score += min(hierarchy / 0.30, 1.0) * 0.20     # nested groups
+        score += _ramp(ncols, 4, 12) * 0.10            # wide tables
+        score += _ramp(merges, 0.05, 0.35) * 0.35      # vertically merged cells
+
+    return min(score, 1.0)
+
+
+def _render(pdf_path, page_number, bbox=None, dpi=200):
+    """Render a page (or a bounding box on a page) of a PDF to a base64-encoded PNG."""
+    doc = fitz.open(pdf_path)
+    idx = max(0, min(page_number - 1, len(doc) - 1))
+    page = doc[idx]
+
+    clip = None
+    if bbox:
+        h = page.rect.height
+        # docling uses BOTTOMLEFT origin, PyMuPDF uses top-left
+        origin = bbox.get("coord_origin", "BOTTOMLEFT")
+        if "BOTTOM" in origin:
+            clip = fitz.Rect(bbox["l"], h - bbox["t"], bbox["r"], h - bbox["b"])
+        else:
+            clip = fitz.Rect(bbox["l"], bbox["t"], bbox["r"], bbox["b"])
+
+    pix = page.get_pixmap(dpi=dpi, clip=clip)
+    b64 = base64.b64encode(pix.tobytes("png")).decode("utf-8")
+    doc.close()
+    return b64
+
+
+TABLE_VISION_PROMPT = (
+    "You are a table-to-HTML conversion expert. Look at the table in the provided image "
+    "and convert it to clean, semantic HTML. Use proper <thead>, <tbody>, <th>, <td> tags. "
+    "If a cell is merged/spanned in the original, REPEAT its value in every row and column it "
+    "covers — do NOT use rowspan or colspan; every <td> must be filled. "
+    "Preserve ALL values exactly as shown — do not omit or invent data. "
+    "Do NOT add any explanation, just output the HTML table.\n"
+)
+
+TABLE_TEXT_PROMPT = (
+    "The following is a table extracted from a PDF by an automated tool, so it may be "
+    "garbled: columns misaligned, cells split or merged wrongly, headers out of place. "
+    "Reconstruct it as clean, semantic HTML. Use proper <thead>, <tbody>, <th>, <td> tags. "
+    "If a cell is merged/spanned, REPEAT its value in every row and column it covers — "
+    "do NOT use rowspan or colspan; every <td> must be filled. "
+    "Preserve ALL values exactly as shown — do not omit or invent data. "
+    "Do NOT add any explanation, just output the HTML table.\n\n"
+)
+
+
+def strip_code_fence(text):
+    """Strip a code fence from the start and end of a string, if present."""
+    text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
+    text = re.sub(r"\n?```$", "", text)
+    return text
+
+
+_ollama_client = None
+
+
+def _client():
+    global _ollama_client
+    if _ollama_client is None:
+        _ollama_client = OpenAI(base_url=os.environ["LLM_BASE_URL"], api_key=os.environ["LLM_API_KEY"])
+    return _ollama_client
+
+
+def _call_model(model, messages):
+    resp = _client().chat.completions.create(
+        model=model,
+        messages=messages,
+        temperature=0.0,
+    )
+    return strip_code_fence(resp.choices[0].message.content.strip())
+
+
+def _call_vision_model(b64_img):
+    messages = [{
+        "role": "user",
+        "content": [
+            {"type": "text", "text": TABLE_VISION_PROMPT},
+            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64_img}"}},
+        ],
+    }]
+    return _call_model(os.environ["VISION_MODEL"], messages)
+
+
+def _call_text_model(table_md):
+    messages = [{"role": "user", "content": TABLE_TEXT_PROMPT + table_md}]
+    # single-model task → use the primary (first) extraction model
+    return _call_model(config.EXTRACTION_MODELS[0], messages)
+
+
+class _TableParser(HTMLParser):
+    """Parse an HTML table into a list of rows, each row being a list of (tag, text, colspan, rowspan)."""
+    def __init__(self):
+        super().__init__()
+        self.rows = []
+        self._row = None
+        self._cell = None
+        self._attrs = {}
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "tr":
+            self._row = []
+        elif tag in ("td", "th") and self._row is not None:
+            self._attrs = dict(attrs)
+            self._cell = []
+
+    def handle_data(self, data):
+        if self._cell is not None:
+            self._cell.append(data)
+
+    def handle_endtag(self, tag):
+        if tag in ("td", "th") and self._cell is not None:
+            self._row.append((tag, "".join(self._cell).strip(),
+                              int(self._attrs.get("colspan") or 1),
+                              int(self._attrs.get("rowspan") or 1)))
+            self._cell = None
+        elif tag == "tr" and self._row is not None:
+            self.rows.append(self._row)
+            self._row = None
+
+
+def _expand_spans(html):
+    """Expand an HTML table so that every <td> is filled, repeating values for merged cells."""
+    parser = _TableParser()
+    parser.feed(html)
+    if not parser.rows:
+        return html
+    grid = {}
+    for r, row in enumerate(parser.rows):
+        c = 0
+        for tag, text, colspan, rowspan in row:
+            while (r, c) in grid:
+                c += 1
+            for dr in range(rowspan):
+                for dc in range(colspan):
+                    grid[(r + dr, c + dc)] = (tag, text)
+            c += colspan
+    width = max((c for _, c in grid), default=-1) + 1
+    lines = ["<table>"]
+    for r in range(len(parser.rows)):
+        lines.append("  <tr>")
+        for c in range(width):
+            tag, text = grid.get((r, c), ("td", ""))
+            lines.append(f"    <{tag}>{text}</{tag}>")
+        lines.append("  </tr>")
+    lines.append("</table>")
+    return "\n".join(lines)
+
+
+def process_tables(sections_path):
+    """Process table sections from a JSON file, scoring their complexity and saving the results."""
+    sections_path = Path(sections_path)
+    if not sections_path.exists():
+        raise FileNotFoundError(f"Sections not found: {sections_path}")
+
+    stem = sections_path.stem.replace("_sections", "") 
+    out_path = Path(config.INTERMEDIATE_DIR) / f"{stem}_tables.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if out_path.exists() and not config.FORCE:
+        logger.info(f"Skipping tables — already exists")
+        return json.loads(out_path.read_text(encoding="utf-8"))
+
+    sections = json.loads(sections_path.read_text(encoding="utf-8"))
+    table_sections = [(i, s) for i, s in enumerate(sections) if s["type"] == "table"]
+
+    if not table_sections:
+        logger.info("No tables found")
+        out_path.write_text("[]", encoding="utf-8")
+        return []
+
+    logger.info(f"Scoring {len(table_sections)} tables...")
+    results = []
+
+    for i, (section_idx, section) in enumerate(table_sections):
+        content = section["content"]
+        complexity = score_table_complexity(content)
+        page = section.get("page") or 1
+
+        logger.info(f"  Table {i}: complexity={complexity:.2f}")
+
+        results.append({
+            "section_idx": section_idx,
+            "complexity_score": round(complexity, 4),
+            "page": page,
+            "content": content,
+            "bbox": section.get("bbox"),
+        })
+
+    out_path.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8")
+    logger.info(f"Scored {len(results)} tables → {out_path}")
+    return results
+
+
+def _refine_one_table(table, pdf_path):
+    """Refine a single table section using the vision model, returning the updated HTML."""
+    idx = table["section_idx"]
+    page = table.get("page", 1)
+    content = table.get("content", "")
+    bbox = table.get("bbox")
+
+    html = None
+    used_vision = False
+    if pdf_path and Path(pdf_path).exists():
+        try:
+            image_b64 = _render(pdf_path, page, bbox)
+            html = _call_vision_model(image_b64)
+            used_vision = True
+        except Exception as e:
+            logger.warning(f"  Section {idx}: vision failed ({e}), falling back to text LLM")
+
+    if html is None:
+        try:
+            html = _call_text_model(content)
+        except Exception as e:
+            logger.error(f"  Section {idx}: failed — {e}")
+            return idx, None, False
+
+    return idx, _expand_spans(html), used_vision
+
+
+_WORD_RE = re.compile(r"[A-Za-z]{4,}")
+
+
+def _word_cell_ratio(content):
+    cells = [c.strip() for line in content.splitlines() for c in line.split("|") # skip empty lines and separator rows
+             if c.strip() and set(c.strip()) != {"-"}]
+    if len(cells) < 12:
+        return 1.0
+    return sum(1 for c in cells if _WORD_RE.search(c)) / len(cells) # fraction of cells containing at least one word of 4+ letters
+
+
+def refine_symbol_only_tables(tables_path, pdf_path=None):
+    threshold = config.VISION_COMPLEXITY_THRESHOLD
+    tables = json.loads(Path(tables_path).read_text(encoding="utf-8"))
+    idxs = {t["section_idx"] for t in tables
+            if t.get("complexity_score", 0) >= threshold
+            and "|" in t.get("content", "") and _word_cell_ratio(t["content"]) < 0.10} 
+    if idxs:
+        logger.info(f"Pre-refining {len(idxs)} symbol-only table(s) before embedding: {sorted(idxs)}")
+        refine_matched_tables(tables_path, idxs, pdf_path=pdf_path)
+
+
+def refine_matched_tables(tables_path, matched_section_indices, pdf_path=None):
+    """Refine matched tables using the vision model, updating the JSON file in place.
+    Arguments:
+    tables_path: Path to the JSON file containing table sections.
+    matched_section_indices: A set of section indices to refine.
+    pdf_path: Optional path to the PDF file for rendering images.
+    Returns:
+    The updated list of table sections with refined HTML content.
+    """
+    tables_path = Path(tables_path)
+    if not tables_path.exists():
+        return []
+
+    tables = json.loads(tables_path.read_text(encoding="utf-8"))
+
+    if not pdf_path:
+        candidate = Path(config.PDF_DIR) / f"{tables_path.stem.replace('_tables', '')}.pdf"
+        pdf_path = str(candidate) if candidate.exists() else None
+
+    threshold = config.VISION_COMPLEXITY_THRESHOLD
+    matched = [t for t in tables if t.get("section_idx") in matched_section_indices]
+    to_refine = [t for t in matched
+                 if t.get("complexity_score", 0) >= threshold
+                 and not t.get("refined")]
+
+    skipped = len(matched) - len(to_refine)
+    if skipped:
+        logger.info(f"  Refining {len(to_refine)} matched tables above complexity "
+                    f"{threshold}; {skipped} simpler ones left as markdown")
+
+    if not to_refine:
+        return tables
+
+    workers = max(1, min(config.VISION_PARALLELISM, len(to_refine)))
+    logger.info(f"  Refining {len(to_refine)} tables ({workers} workers in parallel)...")
+
+    by_idx = {table["section_idx"]: table for table in tables}
+    vision_count = 0
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: # use threads for tasks (PDF rendering, LLM calls)
+        futures = [pool.submit(_refine_one_table, table, pdf_path) for table in to_refine]
+        for fut in concurrent.futures.as_completed(futures):
+            idx, html, used_vision = fut.result()
+            if html is None:
+                continue
+            by_idx[idx]["original_content"] = by_idx[idx].get("content", "")
+            by_idx[idx]["content"] = html
+            by_idx[idx]["refined"] = True
+            by_idx[idx]["refined_format"] = "html"
+            if used_vision:
+                vision_count += 1
+            logger.info(f"  Section {idx}: refined ({len(html)} chars, vision={used_vision})")
+
+    tables_path.write_text(json.dumps(tables, indent=2, ensure_ascii=False), encoding="utf-8")
+    logger.info(f"Refined {vision_count} tables via vision model (of {len(to_refine)} attempted)")
+    return tables

structured_data_extraction/templates/extraction.j2

@@ -0,0 +1,51 @@
+You are a document extraction specialist. Extract ALL value variants for each requested key from the document sections below.
+
+## Document Type
+{{ document_type }}
+
+## Source File
+{{ source_file }}
+
+## Domain Vocabulary
+{{ vocab_terms }}
+
+## Extraction Schema
+{{ schema }}
+
+## Requested Keys
+{{ user_keys }}
+
+## Relevant Document Sections
+
+{{ section_content }}
+
+---
+
+## Instructions
+
+For each field in the schema above, extract ALL value variants found in the document sections. Produce a JSON object where each schema field name maps to a list of records.
+
+The structure of each record is defined by the **fields** property in the schema for that field. Follow that structure exactly — do NOT add extra sub-fields or omit required ones.
+
+When a schema sub-field has a **unit** property, a companion `{field_name}_unit` sub-field will be present in the output structure. Fill it with the unit value from the schema (or from the document if it differs). For example, if the schema defines `smw: {type: "number", unit: "cm³ cm⁻³"}`, the output record will have both `smw` (the value) and `smw_unit` (the unit string).
+
+Every record must include a `metadata` object with:
+- **source**: "{{ source_file }}"
+- **section_type**: type of section (e.g., "table", "list", "text")
+- **specified**: true if explicitly stated, false if inferred
+
+## Rules
+1. Extract EVERY occurrence — a key may appear in multiple tables under different categories, conditions, or columns. Each variant gets its own record.
+2. Match synonyms using the domain vocabulary and schema descriptions.
+3. **One record per category.** If a value applies to multiple distinct categories (e.g., different grades, models, segments, periods), create a SEPARATE record for each. NEVER combine distinct categories into a single field (e.g., do not write "GradeA, GradeB" — create two records).
+4. Use exact values from the document. Do not infer or fabricate.
+5. If a key is not found, produce an empty list for that field.
+6. When a value comes from a labeled table column (e.g., [Min], [Typ], [Max]), use that label in the appropriate sub-field for that field's structure.
+7. **Row-label match required, plus sub-rows that refine it**: Only extract values from rows whose parameter name or column header directly matches the schema field (or a true synonym). Additionally, rows that *immediately follow* a matched parameter row and describe a refinement of it (e.g., a row labeled "Over Temperature" or "Average TC" sitting right under "Input Bias Current") DO belong to that matched parameter — extract them as separate records. For such a sub-row, set `condition` to the sub-row label combined with the Test Conditions/Comments cell when both are present (e.g., `condition = "Over Temperature, T_A = -40°C to +85°C"`); keep `qualifier` as the column label (Min/Typ/Max); keep `category` as the grade/group from the column header. Stop as soon as the next row names a different parameter ("Input X" vs "Output X" are separate fields — never cross that boundary).
+8. **Use the surrounding section heading for category when the table lost it**: If a table is prefixed with a "Surrounding section heading: …" line  and the column headers no longer carry a grade/category label, use that heading as the `category` for rows extracted from that table. Do this only when the column header is missing the label — never override an explicit column label.
+9. **Variant/differentiator columns**: When the schema defines a field that identifies the column-group variant (e.g. named "differentiator", "variant", "series", or "grade"), set it to the header of the column group the value sits under — a part number (e.g. "LT1167AI/LT1167AI-1"), product grade, fiscal period, or segment. Emit a SEPARATE record for every variant group, even when the parameter, condition, and qualifier are identical across groups (only the value and the differentiator change).
+10. **Value fields must contain only the bare measured value.** Strip all surrounding words, labels, and narration — no units, no method/location qualifiers (e.g. `"7"` not `"7mL/well"`, `"-20"` not `"-20C shared freezer"`). Units belong exclusively in the companion `*_unit` sub-field; contextual narration belongs in `condition` or nowhere.
+11. **Fill every defined sub-field that the source supports.** If the document provides information for a sub-field the schema defines, populate it — do not leave it blank. An empty string is only acceptable when the information is genuinely absent from the source text.
+12. **No placeholder records.** Only emit a record for an occurrence the document actually gives a value for. If an entity or parameter is merely named again without its own stated quantity (e.g. mentioned in a processing/dilution step that gives no number), do NOT create a record for it, and do NOT backfill the value sub-field with `0` or an empty string as a stand-in. Every record must carry at least one real value taken from the document — a genuine `0` read from the source is fine, but never fabricate `0`/empty just to satisfy the structure.
+
+Respond with a JSON object matching the schema structure.

structured_data_extraction/templates/schema_discovery.j2

@@ -0,0 +1,91 @@
+You are a document analysis expert. Your task is to discover the extraction schema for this document.
+
+## Context
+
+The user wants to extract the following keys from this document:
+{{ user_keys }}
+
+## Domain Vocabulary
+
+These are the most relevant terms found in this document (use these to understand the domain and recognize synonyms/abbreviations):
+{{ vocab_terms }}
+
+## Relevant Document Sections
+
+The following sections were identified as potentially containing the desired values:
+
+{{ section_content }}
+
+## Document Preview
+
+{{ document_preview }}
+
+---
+
+## Instructions
+
+For each user key, define a schema field. The field name MUST be the user's key converted to a valid Python identifier (lowercase, spaces replaced with underscores, special characters removed). For example:
+- "Revenue" → "revenue"
+- "Net Income" → "net_income"
+- "Supply Current" → "supply_current"
+
+Each field definition must include:
+1. **description**: Describe what the field represents, where to find it, and note any synonyms or abbreviations from the domain vocabulary. The description should help the extraction model locate the right values.
+2. **type**: Always "list" — each key may appear in multiple places (different tables, categories, conditions, columns).
+3. **fields**: An object defining the sub-fields for each record in this field's list. Tailor the sub-fields to what this specific field actually needs based on the document content. Every record MUST include a `metadata` field. Include a `value` field ONLY when the record has a single primary quantity (or identifying label) that the other sub-fields don't already capture — `value` then holds that one scalar/short token. Do NOT add `value` when the record's structured sub-fields already carry all of its information: a `value` that merely restates the other sub-fields as a prose sentence (e.g. "300 xg for 3 minutes" alongside separate speed/duration sub-fields) is wrong and must be omitted. Beyond that, only add sub-fields that are meaningful for this field. Common patterns:
+   - Column-labeled specs: "qualifier" (column label like Min/Typ/Max), "condition" (operating condition), "value", "metadata"
+   - Parallel variant columns: the SAME parameter repeated across several side-by-side column groups — each group a distinct product variant, part-number family, grade, or reporting period sitting above its own value or Min/Typ/Max sub-columns. Add a "differentiator" sub-field that records which group a value came from (the group's header text, e.g. a part number like "LT1167AI/LT1167AI-1", a grade, or "FY2024"), alongside "qualifier"/"condition". Name this sub-field after what the groups represent when they carry an explicit label (e.g. headers reading "AR Grade"/"BR Grade" → "grade"); otherwise call it "differentiator".
+   - Categorical comparisons: "category" (the row label or group), "value", "metadata"
+   - Simple single-value lookups: just "value", "metadata"
+   - Multi-column rows: each numeric column becomes its own sub-field (e.g., `smw`, `smfcf`)
+   Each sub-field needs a **name**, **type** ("string", "number", or "any"), **description**, and optionally **unit** (the unit of measurement for this sub-field). Always include the unit for any numeric sub-field that carries a physical quantity.
+
+**Important**: Only list synonyms that refer to the EXACT same measured quantity. Terms like "Discrepancy" vs "Difference from observed" may sound similar but can refer to different calculations in different tables. Only include a synonym if the value would be interchangeable with the user's key in the same context.
+
+In addition to the per-key schema fields, include a top-level `_document_type` key with a short free-form label describing what kind of document this is (e.g. "op-amp datasheet", "quarterly earnings report", "experimental physics paper", "soil survey"). Derive it from the vocabulary and document preview — do not pick from a fixed list.
+
+Respond with ONLY a JSON object. No explanation, no markdown fences. Example:
+
+{
+  "_document_type": "instrumentation amplifier datasheet",
+  "temperature": {
+    "type": "list",
+    "description": "Temperature readings from the measurement table. May appear as 'Temp', 'Temperature', 'T'.",
+    "fields": {
+      "qualifier": {"type": "string", "description": "Column label: 'minimum', 'typical', or 'maximum'"},
+      "condition": {"type": "string", "description": "Condition or location (e.g., 'outdoor', 'indoor')"},
+      "value": {"type": "any", "description": "Temperature value", "unit": "°C"},
+      "metadata": {"type": "object", "description": "Source and provenance"}
+    }
+  },
+  "common_mode_rejection": {
+    "type": "list",
+    "description": "CMRR spec repeated across several part-number column groups in the characteristics table.",
+    "fields": {
+      "differentiator": {"type": "string", "description": "Which variant column group the value sits under — the group's header, e.g. a part number 'LT1167AI/LT1167AI-1', a grade, or a reporting period"},
+      "qualifier": {"type": "string", "description": "Column label: 'min', 'typ', or 'max'"},
+      "condition": {"type": "string", "description": "Operating condition (e.g., 'G = 10')"},
+      "value": {"type": "any", "description": "CMRR value", "unit": "dB"},
+      "metadata": {"type": "object", "description": "Source and provenance"}
+    }
+  },
+  "survival_rate": {
+    "type": "list",
+    "description": "Survival rate comparison across categories from the results table.",
+    "fields": {
+      "category": {"type": "string", "description": "Group or model name (e.g., 'Control', 'Treatment A')"},
+      "value": {"type": "any", "description": "Survival rate", "unit": "%"},
+      "metadata": {"type": "object", "description": "Source and provenance"}
+    }
+  },
+  "soil_parameters": {
+    "type": "list",
+    "description": "Soil classification parameters from the properties table. Each row is a soil type.",
+    "fields": {
+      "value": {"type": "string", "description": "Soil type name (e.g., 'Medium loam')"},
+      "smw": {"type": "number", "description": "Wilting point", "unit": "cm³ cm⁻³"},
+      "smfcf": {"type": "number", "description": "Field capacity", "unit": "cm³ cm⁻³"},
+      "metadata": {"type": "object", "description": "Source and provenance"}
+    }
+  }
+}

structured_data_extraction/vocab.py

@@ -0,0 +1,47 @@
+"""Domain vocabulary extraction via YAKE.
+
+Runs unsupervised keyword extraction on the document's Markdown text to surface
+domain-specific terms (e.g. "supply voltage", "output impedance"). The resulting
+vocab list is used in embed.py to expand user keys with related terms before
+embedding, improving recall when the user's key wording doesn't exactly match
+the document's phrasing.
+
+YAKE was chosen because it is unsupervised (no training corpus needed), fast,
+and works well on technical English. The top-N and max n-gram size are
+configurable via VOCAB_TOP_N and VOCAB_MAX_NGRAM in .env.
+"""
+import argparse
+import logging
+from pathlib import Path
+import yake
+from . import config
+from .paths import inter, cached
+
+logger = logging.getLogger(__name__)
+
+
+def extract_vocab(md_path):
+    md_path = Path(md_path)
+
+    def build():
+        logger.info(f"Extracting vocabulary from {md_path.name}...")
+        kw = yake.KeywordExtractor(lan="en", n=config.VOCAB_MAX_NGRAM,
+                                   dedupLim=0.7, top=config.VOCAB_TOP_N)
+        # lower score = more important in YAKE
+        return [{"term": term, "score": round(score, 4)}
+                for term, score in kw.extract_keywords(md_path.read_text(encoding="utf-8"))]
+
+    return cached(inter(md_path.stem, "vocab.json"), build)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract domain vocabulary")
+    parser.add_argument("--input", required=True)
+    args = parser.parse_args()
+
+    result = extract_vocab(args.input)
+    print(f"Extracted {len(result)} keywords")
+
+
+if __name__ == "__main__":
+    main()