valcyon 0.1.0__tar.gz

valcyon-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+.venv/
+__pycache__/
+*.egg-info/
+dist/
+*.db
+.pytest_cache/
+.env
+out/
+out_vlm/
+.venv312/

valcyon-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.5
+Name: valcyon
+Version: 0.1.0
+Summary: Accurate, traceable retrieval for PDFs.
+Author: Osama AKESKOUS
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: numpy
+Provides-Extra: cloud
+Requires-Dist: requests; extra == 'cloud'
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Provides-Extra: local
+Requires-Dist: sentence-transformers; extra == 'local'
+Description-Content-Type: text/markdown
+
+# Valcyon
+
+<p align="center">
+  <img src="assets/kingfisher-logo.png" alt="pdftree logo" width="220">
+</p>
+
+Deterministic hierarchical PDF extraction and retrieval toolkit. This project is in early development.

valcyon-0.1.0/README.md

@@ -0,0 +1,7 @@
+# Valcyon
+
+<p align="center">
+  <img src="assets/kingfisher-logo.png" alt="pdftree logo" width="220">
+</p>
+
+Deterministic hierarchical PDF extraction and retrieval toolkit. This project is in early development.

valcyon-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "valcyon"
+version = "0.1.0"
+description = "Accurate, traceable retrieval for PDFs."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Osama AKESKOUS" }]
+dependencies = ["numpy"]
+
+[project.optional-dependencies]
+local = ["sentence-transformers"]
+cloud = ["requests"]
+dev = ["pytest"]
+
+[project.scripts]
+valcyon = "valcyon.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/valcyon"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/src/valcyon",
+  "/README.md",
+  "/LICENSE",
+  "/pyproject.toml",
+]

valcyon-0.1.0/src/valcyon/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

valcyon-0.1.0/src/valcyon/adapters/base.py

@@ -0,0 +1,7 @@
+from typing import Protocol
+
+from valcyon.models import Element
+
+
+class Parser(Protocol):
+    def parse(self, source: str) -> list[Element]: ...

valcyon-0.1.0/src/valcyon/adapters/errors.py

@@ -0,0 +1,2 @@
+class MinerUError(RuntimeError):
+    """No token. Put MINERU_TOKEN in a .env file or set it as an environment variable."""

valcyon-0.1.0/src/valcyon/adapters/mineru.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import json
+
+from valcyon.models import Element
+
+# block type in middle.json -> element type in Valcyon
+_TYPE_MAP = {
+    "title": "heading",
+    "text": "text",
+    "list": "text",
+    "index": "text",
+    "code": "text",
+    "table": "table",          # to verify on a real file
+    "image": "figure",         # to verify on a real file
+    "interline_equation": "equation",  # to verify on a real file
+}
+
+
+def _span_text(span: dict) -> str:
+    """Text of one span. Tables/equations may use other keys than 'content'."""
+    return span.get("content") or span.get("html") or span.get("latex") or ""
+
+
+def _block_text(block: dict) -> str:
+    """Text of a block: its lines joined by spaces, or its sub-blocks if nested."""
+    if "lines" in block:
+        lines = []
+        for line in block["lines"]:
+            parts = [_span_text(s) for s in line.get("spans", [])]
+            lines.append("".join(parts).strip())
+        return " ".join(x for x in lines if x)
+    # image/table blocks nest their content in sub-blocks (caption, body...)
+    sub = [_block_text(b) for b in block.get("blocks", [])]
+    return "\n".join(x for x in sub if x)
+
+
+def _bbox(block: dict):
+    b = block.get("bbox")
+    if b and len(b) == 4:
+        return tuple(float(x) for x in b)
+    return None
+
+
+class MinerUParser:
+    """Reads MinerU's middle.json.
+
+    Assumed shape:
+        {"pdf_info": [ {"page_idx": 0, "preproc_blocks": [ block, ... ]}, ... ]}
+        block = {"type", "bbox", "index", "level" (titles), "lines": [...]}
+    """
+
+    def parse(self, source: str) -> list[Element]:
+        with open(source, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        elements: list[Element] = []
+        for page_pos, page in enumerate(data.get("pdf_info", [])):
+            page_no = int(page.get("page_idx", page_pos)) + 1  # 1-based
+            blocks = page.get("preproc_blocks", [])
+            # explicit sort key -> same file always gives same order
+            ordered = sorted(enumerate(blocks), key=lambda t: (t[1].get("index", t[0]), t[0]))
+
+            for _, block in ordered:
+                kind = _TYPE_MAP.get(block.get("type"))
+                if kind is None:
+                    continue  # unknown block type: ignore for now
+                text = _block_text(block)
+                if not text:
+                    continue
+                level = int(block.get("level", 1)) if kind == "heading" else None
+                elements.append(Element(
+                    type=kind, text=text, page=page_no, order=len(elements),
+                    bbox=_bbox(block), raw_level=level,
+                ))
+        return elements

valcyon-0.1.0/src/valcyon/adapters/mineru_cloud.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import io
+import os
+import time
+import zipfile
+from pathlib import Path, PurePosixPath
+from valcyon.config import get_setting
+
+from valcyon.adapters.errors import MinerUError
+
+API = "https://mineru.net/api/v4"
+
+
+class CloudMinerURunner:
+    """Sends a PDF to the MinerU cloud API and saves its middle.json.
+
+    Flow (from MinerU's docs):
+      1. POST /file-urls/batch      -> batch_id + presigned upload URL
+      2. PUT  <upload url>          -> the PDF bytes (no Content-Type header)
+      3. GET  /extract-results/batch/{batch_id}   until state == "done"
+      4. download full_zip_url      -> the cloud names middle.json "layout.json"
+    """
+
+    def __init__(self, token: str | None = None, model_version: str = "pipeline",
+                 language: str = "ch", is_ocr: bool = False,
+                 timeout: float = 600, interval: float = 3,
+                 base_url: str = API, session=None, sleep=time.sleep):
+        self.token = token or get_setting("MINERU_TOKEN")
+        if not self.token:
+            raise MinerUError("No token. Set the MINERU_TOKEN environment variable.")
+        self.model_version = model_version
+        self.language = language
+        self.is_ocr = is_ocr
+        self.timeout = timeout
+        self.interval = interval
+        self.base_url = base_url.rstrip("/")
+        self.sleep = sleep
+        self._session = session
+
+    # -- helpers ---------------------------------------------------------
+    def _http(self):
+        if self._session is None:
+            try:
+                import requests
+            except ImportError:
+                raise MinerUError("Install the cloud extra: pip install valcyon[cloud]")
+            self._session = requests.Session()
+        return self._session
+
+    @staticmethod
+    def _data(resp) -> dict:
+        """Check an API response and return its 'data' part."""
+        if resp.status_code != 200:
+            raise MinerUError(f"HTTP {resp.status_code} from MinerU")
+        body = resp.json()
+        if body.get("code") != 0:
+            raise MinerUError(f"MinerU error {body.get('code')}: {body.get('msg')}")
+        return body.get("data") or {}
+
+    @staticmethod
+    def _pick(result, file_name: str) -> dict | None:
+        """extract_result is a list in the docs; accept a dict too."""
+        if isinstance(result, dict):
+            return result
+        for row in result or []:
+            if row.get("file_name") == file_name:
+                return row
+        return (result or [None])[0]
+
+    @staticmethod
+    def _find_middle(names: list[str]) -> str | None:
+        found = [n for n in names
+                 if PurePosixPath(n).name in ("layout.json",)
+                 or PurePosixPath(n).name.endswith("middle.json")]
+        return sorted(found)[0] if found else None
+
+    # -- main ------------------------------------------------------------
+    def run(self, pdf_path: str, out_dir: str) -> str:
+        pdf = Path(pdf_path)
+        if not pdf.is_file():
+            raise MinerUError(f"PDF not found: {pdf}")
+        out = Path(out_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        http = self._http()
+        auth = {"Authorization": f"Bearer {self.token}"}
+
+        # 1. ask for an upload URL
+        body = {
+            "files": [{"name": pdf.name, "is_ocr": self.is_ocr}],
+            "model_version": self.model_version,
+            "language": self.language,
+            "enable_table": True,
+            "enable_formula": True,
+        }
+        data = self._data(http.post(f"{self.base_url}/file-urls/batch",
+                                    headers={**auth, "Content-Type": "application/json"},
+                                    json=body, timeout=60))
+        batch_id = data.get("batch_id")
+        urls = data.get("file_urls") or []
+        if not batch_id or not urls:
+            raise MinerUError("MinerU did not return a batch_id and an upload URL")
+
+        # 2. upload the PDF (deliberately no headers: the URL is signed)
+        with open(pdf, "rb") as f:
+            put = http.put(urls[0], data=f, timeout=300)
+        if put.status_code not in (200, 201):
+            raise MinerUError(f"Upload failed: HTTP {put.status_code}")
+
+        # 3. poll until done
+        deadline = time.monotonic() + self.timeout
+        while True:
+            data = self._data(http.get(f"{self.base_url}/extract-results/batch/{batch_id}",
+                                       headers=auth, timeout=60))
+            row = self._pick(data.get("extract_result"), pdf.name) or {}
+            state = row.get("state")
+            if state == "done":
+                zip_url = row.get("full_zip_url")
+                if not zip_url:
+                    raise MinerUError("Task is done but has no full_zip_url")
+                break
+            if state == "failed":
+                raise MinerUError(f"MinerU failed: {row.get('err_msg') or 'unknown reason'}")
+            if time.monotonic() >= deadline:
+                raise MinerUError(f"Timed out (state={state}). batch_id={batch_id}")
+            self.sleep(self.interval)
+
+        # 4. download the ZIP and keep only the middle.json inside it
+        resp = http.get(zip_url, timeout=300)
+        if resp.status_code != 200:
+            raise MinerUError(f"Download failed: HTTP {resp.status_code}")
+        with zipfile.ZipFile(io.BytesIO(resp.content)) as z:
+            member = self._find_middle(z.namelist())
+            if member is None:
+                raise MinerUError(f"No layout.json / middle.json in ZIP: {z.namelist()}")
+            target = out / f"{pdf.stem}_middle.json"
+            target.write_bytes(z.read(member))
+        return str(target)

valcyon-0.1.0/src/valcyon/chunking.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Callable
+
+from valcyon.ids import sha
+from valcyon.models import BBox, Node
+
+
+def approx_tokens(text: str) -> int:
+    """Cheap token estimate: whitespace-separated words."""
+    return len(text.split())
+
+
+@dataclass
+class Chunk:
+    id: str
+    doc_id: str
+    section_id: str
+    text: str
+    breadcrumb: str               # "Expérience > Assistant projet – Stage"
+    path: list[str]
+    page_start: int
+    page_end: int
+    bboxes: list[tuple[int, BBox]]   # (page, bbox) of each source block
+    chunk_type: str               # text | table | figure | equation
+    order: int                    # position inside its section
+    token_count: int
+
+    @property
+    def embed_text(self) -> str:
+        """What we will embed: the breadcrumb gives the chunk its context."""
+        return f"{self.breadcrumb}\n\n{self.text}" if self.breadcrumb else self.text
+
+
+def _split_words(sentence: str, max_tokens: int, count: Callable[[str], int]) -> list[str]:
+    """Last resort for a single sentence longer than max_tokens."""
+    out: list[str] = []
+    cur: list[str] = []
+    for w in sentence.split():
+        if cur and count(" ".join(cur + [w])) > max_tokens:
+            out.append(" ".join(cur))
+            cur = []
+        cur.append(w)
+    if cur:
+        out.append(" ".join(cur))
+    return out
+
+
+def _split_text(text: str, max_tokens: int, overlap: int,
+                count: Callable[[str], int]) -> list[str]:
+    """Split on sentence boundaries; each piece repeats up to `overlap` tokens of the last."""
+    pieces: list[str] = []
+    for s in re.split(r"(?<=[.!?])\s+", text.strip()):
+        if not s:
+            continue
+        pieces.extend([s] if count(s) <= max_tokens else _split_words(s, max_tokens, count))
+
+    chunks: list[str] = []
+    cur: list[str] = []
+    for p in pieces:
+        if cur and count(" ".join(cur + [p])) > max_tokens:
+            chunks.append(" ".join(cur))
+            carry: list[str] = []
+            total = 0
+            for q in reversed(cur):          # keep trailing pieces as overlap
+                c = count(q)
+                if total + c > overlap:
+                    break
+                carry.insert(0, q)
+                total += c
+            cur = carry
+            if cur and count(" ".join(cur + [p])) > max_tokens:
+                cur = []                     # overlap would not fit: drop it
+        cur.append(p)
+    if cur:
+        chunks.append(" ".join(cur))
+    return chunks
+
+
+def _chunk_section(section: Node, count: Callable[[str], int],
+                   max_tokens: int, overlap: int) -> list[Chunk]:
+    out: list[Chunk] = []
+    breadcrumb = " > ".join(section.path)
+    buffer: list[Node] = []
+
+    def emit(text: str, blocks: list[Node], kind: str) -> None:
+        idx = len(out)
+        pages = [b.page_start for b in blocks]
+        out.append(Chunk(
+            id=sha(f"{section.doc_id}|{section.id}|{idx}|{sha(text)}"),
+            doc_id=section.doc_id, section_id=section.id, text=text,
+            breadcrumb=breadcrumb, path=list(section.path),
+            page_start=min(pages), page_end=max(pages),
+            bboxes=[(b.page_start, b.bbox) for b in blocks if b.bbox is not None],
+            chunk_type=kind, order=idx, token_count=count(text),
+        ))
+
+    def flush() -> None:
+        if buffer:
+            emit("\n".join(b.text for b in buffer), buffer, "text")
+            buffer.clear()
+
+    for block in (c for c in section.children if c.type == "block"):
+        text = block.text or ""
+        if not text.strip():
+            continue
+        kind = block.element_type or "text"
+
+        if kind != "text":                   # tables, figures, equations: one chunk, never split
+            flush()
+            emit(text, [block], kind)
+        elif count(text) > max_tokens:       # too big for one chunk: split it
+            flush()
+            for piece in _split_text(text, max_tokens, overlap, count):
+                emit(piece, [block], "text")
+        else:                                # merge small blocks together
+            if buffer and count("\n".join([b.text for b in buffer] + [text])) > max_tokens:
+                flush()
+            buffer.append(block)
+    flush()
+    return out
+
+
+def chunk_tree(root: Node, max_tokens: int = 400, overlap_tokens: int = 40,
+               count_tokens: Callable[[str], int] = approx_tokens) -> list[Chunk]:
+    """Turn a tree into chunks, in document order. Chunks never cross a section."""
+    if max_tokens <= 0 or not (0 <= overlap_tokens < max_tokens):
+        raise ValueError("need max_tokens > 0 and 0 <= overlap_tokens < max_tokens")
+    chunks: list[Chunk] = []
+    for node in root.walk():
+        if node.type != "block":             # the document root holds any preamble blocks
+            chunks.extend(_chunk_section(node, count_tokens, max_tokens, overlap_tokens))
+    return chunks

valcyon-0.1.0/src/valcyon/cli.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+DEFAULT_DB = "valcyon.db"
+
+
+def _ingest(args: argparse.Namespace) -> int:
+    from valcyon.embedders.bge_m3 import BgeM3Embedder
+    from valcyon.pipeline import ingest_parsed
+    from valcyon.store import Store
+
+    src = Path(args.source)
+    if src.suffix.lower() == ".pdf":
+        from valcyon.adapters.mineru_cloud import CloudMinerURunner
+
+        out_dir = src.parent / "out"
+        CloudMinerURunner().run(str(src), str(out_dir))
+        found = sorted(out_dir.rglob("*_middle.json"))
+        if not found:
+            print(f"No *_middle.json found in {out_dir}", file=sys.stderr)
+            return 1
+        middle = found[0]
+    else:
+        middle = src
+
+    stats = ingest_parsed(str(middle), Store(args.db), BgeM3Embedder())
+    issues = stats.pop("issues")
+    print(stats)
+    for code, message in issues:
+        print(f"[{code}] {message}")
+    return 0
+
+
+def _query(args: argparse.Namespace) -> int:
+    from valcyon.embedders.bge_m3 import BgeM3Embedder
+    from valcyon.retrieve.dense import DenseRetriever
+    from valcyon.retrieve.hybrid import HybridRetriever
+    from valcyon.retrieve.keyword import KeywordRetriever
+    from valcyon.store import Store
+
+    store = Store(args.db)
+    dense = DenseRetriever(store, BgeM3Embedder())
+    retriever = HybridRetriever(dense, KeywordRetriever(store))
+    for result in retriever.search(args.question, k=args.k):
+        chunk = result.chunk
+        print(f"p.{chunk.page_start}  {chunk.breadcrumb}")
+        print("      ", chunk.text[:120].replace("\n", " "))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="valcyon",
+        description="Structure-aware retrieval for PDFs (early development).",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    ingest = subparsers.add_parser(
+        "ingest",
+        help="parse, chunk, index a PDF or a MinerU *_middle.json",
+    )
+    ingest.add_argument("source")
+    ingest.add_argument("--db", default=DEFAULT_DB)
+    ingest.set_defaults(func=_ingest)
+
+    query = subparsers.add_parser("query", help="search an indexed database")
+    query.add_argument("question")
+    query.add_argument("--db", default=DEFAULT_DB)
+    query.add_argument("-k", type=int, default=3)
+    query.set_defaults(func=_query)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

valcyon-0.1.0/src/valcyon/config.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def _parse_env(path: Path) -> dict[str, str]:
+    """Read KEY=VALUE lines. Ignores blanks and # comments, strips quotes."""
+    values: dict[str, str] = {}
+    if not path.is_file():
+        return values
+    # utf-8-sig: Notepad on Windows may add an invisible BOM at the start
+    for line in path.read_text(encoding="utf-8-sig").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, val = line.partition("=")
+        val = val.strip()
+        if len(val) >= 2 and val[0] == val[-1] and val[0] in "\"'":
+            val = val[1:-1]
+        values[key.strip()] = val
+    return values
+
+
+def get_setting(name: str, env_file: str = ".env") -> str | None:
+    """A real environment variable wins; otherwise look in the .env file."""
+    return os.environ.get(name) or _parse_env(Path(env_file)).get(name) or None

valcyon-0.1.0/src/valcyon/embedders/base.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+import numpy as np
+
+
+class Embedder(Protocol):
+    name: str    # stored in the index: vectors from different models must never be mixed
+
+    def embed(self, texts: list[str], kind: str = "doc") -> np.ndarray:
+        """Return a float32 array (len(texts), dim), L2-normalized,
+        so cosine similarity is a plain dot product. kind is "doc" or "query"."""
+        ...

valcyon-0.1.0/src/valcyon/embedders/bge_m3.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import numpy as np
+
+
+class BgeM3Embedder:
+    """Local multilingual embedder. bge-m3 needs no query/document prefix,
+    so `kind` is accepted for the interface but unused."""
+
+    def __init__(self, model_name: str = "BAAI/bge-m3", device: str | None = None) -> None:
+        self.name = model_name
+        self.device = device
+        self._model = None
+
+    def _load(self):
+        if self._model is None:                       # lazy: the import and download happen once
+            from sentence_transformers import SentenceTransformer
+            self._model = SentenceTransformer(self.name, device=self.device)
+        return self._model
+
+    def embed(self, texts: list[str], kind: str = "doc") -> np.ndarray:
+        vecs = self._load().encode(texts, normalize_embeddings=True,
+                                   convert_to_numpy=True, batch_size=16)
+        return np.asarray(vecs, dtype=np.float32)

valcyon-0.1.0/src/valcyon/eval/metrics.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import time
+
+
+def is_relevant(chunk_text: str, needles: list[str]) -> bool:
+    low = chunk_text.lower()
+    return any(n.lower() in low for n in needles)
+
+
+def evaluate(retriever, cases: list[dict], k: int = 5) -> dict:
+    """cases: [{"query": str, "relevant": [str, ...]}].
+    hit_rate: share of queries with a relevant chunk in the top k.
+    hit1: share of queries whose first result is relevant.
+    mrr: mean of 1/rank of the first relevant chunk (0 if not in the top k)."""
+    hits = 0
+    first = 0
+    rr_sum = 0.0
+    failures: list[str] = []
+    ranks: list = []
+    t0 = time.perf_counter()
+    for case in cases:
+        results = retriever.search(case["query"], k=k)
+        rank = next((i for i, r in enumerate(results, 1)
+            if is_relevant(r.chunk.embed_text, case["relevant"])), None)
+        ranks.append((case["query"], rank))
+        if rank is None:
+            failures.append(case["query"])
+        else:
+            hits += 1
+            rr_sum += 1 / rank
+            first += rank == 1
+    elapsed = time.perf_counter() - t0
+    n = len(cases)
+    return {"n": n, "k": k,
+            "hit_rate": hits / n if n else 0.0,
+            "hit1": first / n if n else 0.0,
+            "mrr": rr_sum / n if n else 0.0,
+            "ms_per_query": 1000 * elapsed / n if n else 0.0,
+            "failures": failures, "ranks": ranks}

valcyon-0.1.0/src/valcyon/ids.py

@@ -0,0 +1,20 @@
+import hashlib
+
+
+def sha(text: str) -> str:
+    """Short, stable fingerprint of a string."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
+
+
+def doc_id_from_file(path: str) -> str:
+    """Fingerprint of a file's bytes: same file -> same id."""
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(1 << 20), b""):
+            h.update(block)
+    return h.hexdigest()[:16]
+
+
+def node_id(doc_id: str, position: str) -> str:
+    """Id of a tree node, from its document and its position like '0.2.1'."""
+    return sha(f"{doc_id}|{position}")