esbi-cli 0.2.1__py3-none-any.whl

esbi_cli/extract/__init__.py

@@ -0,0 +1,107 @@
+"""Turn a URL or a local file into clean markdown text."""
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urlparse
+
+import httpx
+
+from esbi_cli.netguard import CannotResolve, UnsafeURL, safe_get
+
+# Identify honestly with a contact URL: Wikipedia (and others) answer 403 to generic browser-like agents
+USER_AGENT = "esbi-cli/0.1 (personal knowledge tool; +https://github.com/RubenAmaury/esbi-cli)"
+
+
+class ExtractError(RuntimeError):
+    pass
+
+
+@dataclass
+class Figure:
+    """An image taken from a PDF (as PNG), with the caption printed next to it if there is one."""
+
+    data: bytes
+    page: int
+    caption: str | None = None
+
+
+@dataclass
+class ExtractedDoc:
+    title: str
+    text: str
+    kind: str  # article | paper
+    url: str | None = None  # canonical URL, or None for local files
+    pdf_bytes: bytes | None = None  # original PDF, saved into raw/
+    figures: list[Figure] = field(default_factory=list)  # PDF images: copied into the vault
+    image_links: list[tuple[str, str]] = field(
+        default_factory=list
+    )  # web images: (alt, url), linked
+    image_bytes: bytes | None = None  # the original image file, saved into raw/
+    image_suffix: str = ""  # its extension, e.g. ".jpg"
+    warnings: list[str] = field(default_factory=list)  # said to the user (e.g. OCR page cap)
+
+
+def is_url(target: str) -> bool:
+    return urlparse(target).scheme in ("http", "https")
+
+
+_GITHUB_REPO = re.compile(r"^https?://(?:www\.)?github\.com/([\w.-]+)/([\w.-]+?)(?:\.git)?/?$")
+
+
+def _get(url: str) -> httpx.Response:
+    try:
+        resp = safe_get(url, headers={"User-Agent": USER_AGENT}, timeout_seconds=30)
+        resp.raise_for_status()
+    except CannotResolve as exc:
+        raise ExtractError(f"Could not fetch {url}: {exc}") from exc
+    except UnsafeURL as exc:
+        raise ExtractError(f"Refused to fetch {url}: {exc}") from exc
+    except httpx.HTTPError as exc:
+        raise ExtractError(f"Could not fetch {url}: {exc}") from exc
+    return resp
+
+
+def extract_source(target: str, ocr=None, max_ocr_pages: int = 10) -> ExtractedDoc:
+    """Dispatch on the target: http(s) URL, local .pdf, .md or image path. `ocr` is the model that
+    reads images and scanned PDFs; without one, those are rejected."""
+    from esbi_cli.extract import clip, html, image, pdf
+
+    if is_url(target):
+        if repo := _GITHUB_REPO.match(target):
+            # the repo page is mostly navigation; its README is the content
+            owner, name = repo.groups()
+            try:
+                readme = _get(f"https://raw.githubusercontent.com/{owner}/{name}/HEAD/README.md")
+            except ExtractError:
+                readme = None  # no README: read the page like any other
+            if readme is not None and len(readme.text.strip()) >= 200:
+                return ExtractedDoc(f"{owner}/{name}", readme.text.strip(), "article", target)
+        resp = _get(target)
+        final_url = str(resp.url)
+        content_type = resp.headers.get("content-type", "")
+        if "application/pdf" in content_type or final_url.lower().endswith(".pdf"):
+            doc = pdf.extract_pdf_bytes(
+                resp.content,
+                fallback_title=Path(urlparse(final_url).path).stem,
+                ocr=ocr,
+                max_ocr_pages=max_ocr_pages,
+            )
+            doc.url = target
+            return doc
+        doc = html.extract_html(resp.text, url=final_url)
+        doc.url = target
+        return doc
+
+    path = Path(target).expanduser()
+    if not path.is_file():
+        raise ExtractError(f"{target} is neither an http(s) URL nor an existing file")
+    if path.suffix.lower() == ".pdf":
+        return pdf.extract_pdf_bytes(
+            path.read_bytes(), fallback_title=path.stem, ocr=ocr, max_ocr_pages=max_ocr_pages
+        )
+    if path.suffix.lower() == ".md":
+        return clip.extract_clip(path)
+    if path.suffix.lower() in image.IMAGE_SUFFIXES:
+        return image.extract_image(path, ocr)
+    raise ExtractError(f"Unsupported file type: {path.suffix or path.name}")

esbi_cli/extract/clip.py

@@ -0,0 +1,30 @@
+"""Markdown notes, typically Obsidian Web Clipper output: the clipped body is the content."""
+
+import re
+from pathlib import Path
+
+from esbi_cli.extract import ExtractedDoc, ExtractError
+from esbi_cli.vault import parse_page
+
+KINDS = ("article", "paper", "email", "video")
+MIN_CHARS = 40  # social posts are short; anything below this is an empty clip
+
+
+def extract_clip(path: Path) -> ExtractedDoc:
+    page = parse_page(path, path.read_text(encoding="utf-8"))
+    text = page.body.strip()
+    if len(text) < MIN_CHARS:
+        raise ExtractError(f"{path.name} has almost no text ({len(text)} chars)")
+    url = page.meta.get("source") or page.meta.get("url")
+    title = str(page.meta.get("title") or path.stem).strip()
+    kind = page.meta.get("kind")
+    if (
+        kind == "video"
+    ):  # the Clipper writes one transcript line per row: make each its own paragraph
+        text = re.sub(r"\n(?=\*\*\d+(?::\d{2}){1,2}\*\* · )", "\n\n", text)
+    return ExtractedDoc(
+        title=title,
+        text=text,
+        kind=kind if kind in KINDS else "article",
+        url=str(url) if url else None,
+    )

esbi_cli/extract/html.py

@@ -0,0 +1,60 @@
+import re
+from urllib.parse import urljoin, urlparse
+
+import lxml.html
+import trafilatura
+
+from esbi_cli.extract import ExtractedDoc, ExtractError
+
+MAX_IMAGES = 5
+NOT_CONTENT = re.compile(r"logo|icon|avatar|sprite|pixel|tracking|badge|button", re.I)
+
+
+def _image_links(html: str, base_url: str) -> list[tuple[str, str]]:
+    """(alt, url) of the article's content images. Linked, never downloaded."""
+    try:
+        tree = lxml.html.fromstring(html)
+    except (lxml.etree.ParserError, ValueError):
+        return []
+    region = next(iter(tree.xpath("//article") or tree.xpath("//main") or [tree]))
+    links, seen = [], set()
+    for img in region.iter("img"):
+        src = (img.get("src") or "").strip()
+        if (
+            not src
+            or src.startswith("data:")
+            or urlparse(src).path.lower().endswith((".svg", ".gif"))
+        ):
+            continue
+        try:
+            width_px, height_px = int(img.get("width") or 0), int(img.get("height") or 0)
+        except ValueError:
+            width_px = height_px = 0
+        if (width_px and width_px < 100) or (height_px and height_px < 100):
+            continue  # a pixel, an icon
+        if NOT_CONTENT.search(src) or NOT_CONTENT.search(img.get("class") or ""):
+            continue
+        url = urljoin(base_url, src)
+        if urlparse(url).scheme in ("http", "https") and url not in seen:
+            seen.add(url)
+            links.append((" ".join((img.get("alt") or "").split()), url))
+            if len(links) == MAX_IMAGES:
+                break
+    return links
+
+
+def extract_html(html: str, url: str) -> ExtractedDoc:
+    text = trafilatura.extract(
+        html, url=url, output_format="markdown", include_links=False, include_tables=True
+    )
+    if not text or len(text.strip()) < 200:
+        raise ExtractError(f"No readable article content found at {url}")
+    meta = trafilatura.extract_metadata(html, default_url=url)
+    title = (meta.title if meta and meta.title else None) or url
+    return ExtractedDoc(
+        title=title.strip(),
+        text=text.strip(),
+        kind="article",
+        url=url,
+        image_links=_image_links(html, url),
+    )

esbi_cli/extract/image.py

@@ -0,0 +1,58 @@
+"""Images (screenshots, photos of slides, scanned pages): an OCR model reads the text."""
+
+import io
+from pathlib import Path
+
+from PIL import Image, ImageOps
+
+from esbi_cli.extract import ExtractedDoc, ExtractError, Figure
+from esbi_cli.llm.adapter import LLMError
+
+IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff")
+MAX_SIDE_PX = 1600  # px: measured, a dense page at this size reads as well as at twice the size
+MIN_CHARS = 40  # less than this is a photo or a diagram, not text worth a note
+NO_OCR = (
+    "add an [llm.ocr] model to config.toml to read images and scanned PDFs "
+    "(`sb init --ocr`, https://rubenamaury.github.io/esbi-cli/docs/reference/configuration/)"
+)
+
+
+def read_text(ocr, png: bytes, name: str) -> str:
+    try:
+        return ocr.read_image(png)
+    except LLMError as exc:  # this source fails; the text model and the other sources are fine
+        raise ExtractError(f"The OCR model could not read {name}: {exc}") from exc
+
+
+def to_png(data: bytes, name: str) -> bytes:
+    """Upright (EXIF), RGB, at most MAX_SIDE_PX on the long side: what the model sees and the note shows."""
+    try:
+        img = ImageOps.exif_transpose(Image.open(io.BytesIO(data))).convert("RGB")
+    except Exception as exc:  # Pillow raises several unrelated error types
+        raise ExtractError(f"Could not open {name} as an image: {exc}") from exc
+    img.thumbnail((MAX_SIDE_PX, MAX_SIDE_PX))
+    out = io.BytesIO()
+    img.save(out, "PNG")
+    return out.getvalue()
+
+
+def extract_image(path: Path, ocr) -> ExtractedDoc:
+    if ocr is None:
+        raise ExtractError(f"{path.name}: {NO_OCR}")
+    data = path.read_bytes()
+    png = to_png(data, path.name)
+    text = read_text(ocr, png, path.name).strip()
+    if len(text) < MIN_CHARS:
+        raise ExtractError(f"{path.name} has no readable text (a photo or a diagram?)")
+    return ExtractedDoc(
+        title=path.stem.replace("-", " ").replace("_", " ").strip(),
+        text=text,
+        kind="article",
+        image_bytes=data,
+        image_suffix=path.suffix.lower(),
+        figures=[image_figure(png)],
+    )
+
+
+def image_figure(png: bytes) -> Figure:
+    return Figure(data=png, page=0)  # page 0: it is not a page of anything

esbi_cli/extract/pdf.py

@@ -0,0 +1,109 @@
+import hashlib
+import re
+
+import pymupdf
+import pymupdf4llm
+
+from esbi_cli.extract import ExtractedDoc, ExtractError, Figure
+from esbi_cli.extract.image import MAX_SIDE_PX, MIN_CHARS, NO_OCR, read_text
+
+CAPTION = re.compile(r"\s*(figure|fig\.?|figura)\s*\d+", re.I)
+MAX_FIGURES = 8
+MIN_WIDTH_POINTS, MIN_HEIGHT_POINTS = (
+    100,
+    80,
+)  # PDF points: smaller images are icons, logos, bullets
+MAX_ASPECT = 6  # thinner strips are rules and banners, not figures
+
+
+def _caption_for(captions: list, rect: pymupdf.Rect) -> str | None:
+    """The figure caption printed just below the image (or just above it)."""
+    below = [b for b in captions if rect.y1 - 5 <= b[1] < rect.y1 + 90]
+    above = [b for b in captions if 0 <= rect.y0 - b[3] < 60]
+    nearest = sorted(below, key=lambda b: b[1] - rect.y1) or above
+    return " ".join(nearest[0][4].split())[:200] if nearest else None
+
+
+MAX_FIGURE_PX = 3000
+
+
+def _dpi_for(width_points: float, height_points: float) -> int:
+    """150 dpi, lowered so that no side passes MAX_FIGURE_PX: a hostile PDF can claim a huge image."""
+    return max(
+        1, int(min(150, MAX_FIGURE_PX * 72 / max(width_points, height_points, 1)))
+    )  # pymupdf wants an int
+
+
+def _figures(doc: pymupdf.Document) -> list[Figure]:
+    """Up to MAX_FIGURES real figures: the captioned ones first, then the biggest."""
+    candidates = []  # (page number, rect, caption)
+    for number, page in enumerate(doc, 1):
+        captions = [b for b in page.get_text("blocks") if len(b) > 4 and CAPTION.match(b[4])]
+        for image in page.get_images(full=True):
+            for rect in page.get_image_rects(image[0]):
+                w, h = rect.width, rect.height
+                if w < MIN_WIDTH_POINTS or h < MIN_HEIGHT_POINTS or max(w / h, h / w) > MAX_ASPECT:
+                    continue
+                candidates.append((number, rect, _caption_for(captions, rect)))
+    candidates.sort(key=lambda c: (c[2] is None, -c[1].get_area()))
+    figures, seen = [], set()
+    for number, rect, caption in candidates:
+        if len(figures) == MAX_FIGURES:
+            break
+        # rendering the image's area (not extracting the raw stream) keeps masks and overlays right
+        data = (
+            doc[number - 1]
+            .get_pixmap(clip=rect, dpi=_dpi_for(rect.width, rect.height))
+            .tobytes("png")
+        )
+        digest = hashlib.sha1(data).hexdigest()
+        if digest not in seen:
+            seen.add(digest)
+            figures.append(Figure(data=data, page=number, caption=caption))
+    return sorted(figures, key=lambda f: f.page)  # reading order
+
+
+def _ocr_pages(doc: pymupdf.Document, ocr, max_pages: int) -> tuple[str, list[str]]:
+    """A scanned PDF has no text layer: render its first `max_pages` pages and read them."""
+    pages = []
+    for number, page in enumerate(doc, 1):
+        if number > max_pages:
+            break
+        dpi = max(1, int(min(130, MAX_SIDE_PX * 72 / max(page.rect.width, page.rect.height, 1))))
+        png = page.get_pixmap(dpi=dpi).tobytes("png")
+        pages.append(read_text(ocr, png, f"page {number}").strip())
+    text = "\n\n".join(p for p in pages if p)
+    if len(text) < MIN_CHARS:
+        raise ExtractError("PDF has no readable text, even with OCR")
+    if doc.page_count <= max_pages:
+        return text, []
+    return text, [f"OCR read the first {max_pages} of {doc.page_count} pages ([run].ocr_max_pages)"]
+
+
+def extract_pdf_bytes(
+    data: bytes, fallback_title: str, ocr=None, max_ocr_pages: int = 10
+) -> ExtractedDoc:
+    try:
+        doc = pymupdf.open(stream=data, filetype="pdf")
+    except Exception as exc:  # pymupdf raises several unrelated error types
+        raise ExtractError(f"Could not open PDF: {exc}") from exc
+    with doc:
+        text = pymupdf4llm.to_markdown(doc)
+        meta_title = (doc.metadata or {}).get("title", "") or ""
+        warnings: list[str] = []
+        if len(text.strip()) < 200:
+            if ocr is None:
+                raise ExtractError(f"PDF has almost no extractable text (scanned?): {NO_OCR}")
+            text, warnings = _ocr_pages(doc, ocr, max_ocr_pages)
+            figures = []  # the pages are images: showing them all as figures would only add noise
+        else:
+            figures = _figures(doc)
+    title = meta_title.strip() or fallback_title.replace("-", " ").replace("_", " ").strip()
+    return ExtractedDoc(
+        title=title,
+        text=text.strip(),
+        kind="paper",
+        pdf_bytes=data,
+        figures=figures,
+        warnings=warnings,
+    )

esbi_cli/gitops.py

@@ -0,0 +1,101 @@
+import subprocess
+from pathlib import Path
+
+# What the worker commits. SCHEMA.md and the golden questions are the user's own, but they are
+# versioned so that a restore from the remote does not lose them.
+VAULT_MANAGED = (
+    "wiki",
+    "raw",
+    "attachments",
+    "index.md",
+    "log.md",
+    "Home.md",
+    "SCHEMA.md",
+    ".esbi/golden.jsonl",
+)
+# The state folder is ignored, but for the golden questions.
+STATE_IGNORE = (".esbi/*", "!.esbi/golden.jsonl")
+GIT_PUSH_TIMEOUT_SECONDS = 120  # an unreachable remote must not hold a run
+
+
+class GitError(RuntimeError):
+    pass
+
+
+def has_git() -> bool:
+    """Is a working `git` installed? (On a Mac without the developer tools, /usr/bin/git exists
+    but fails, so asking it is the only honest test.)"""
+    try:
+        return subprocess.run(["git", "--version"], capture_output=True).returncode == 0
+    except OSError:
+        return False
+
+
+def _git(root: Path, *args: str) -> subprocess.CompletedProcess:
+    try:
+        return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True)
+    except FileNotFoundError:
+        raise GitError("git is not installed") from None
+
+
+def _has_files(path: Path) -> bool:
+    """A managed path is committable only if it holds a file: git cannot track an empty folder,
+    and naming one in `git commit -- <path>` fails ("pathspec did not match")."""
+    return path.is_file() or (path.is_dir() and any(f.is_file() for f in path.rglob("*")))
+
+
+def ignore_state(root: Path) -> None:
+    """Make `.gitignore` ignore the state folder except the golden questions. A vault from before
+    this (a plain `.esbi/` line, or none after the `.secondbrain` rename) is upgraded in place."""
+    ignore = root / ".gitignore"
+    if not ignore.exists():
+        return
+    lines = ignore.read_text().splitlines()
+    if STATE_IGNORE[0] in lines:
+        return
+    at = lines.index(".esbi/") if ".esbi/" in lines else len(lines)
+    lines[at : at + 1] = STATE_IGNORE
+    ignore.write_text("\n".join(lines) + "\n")
+
+
+def commit_vault(root: Path, message: str) -> bool:
+    """Commit only worker-managed paths, leaving the user's own edits (.obsidian etc.) alone.
+
+    Returns True if a commit was made, False if there was nothing to commit.
+    """
+    if not (root / ".git").exists():  # history is optional: no repository, nothing to commit
+        return False
+    ignore_state(root)
+    paths = [p for p in VAULT_MANAGED if _has_files(root / p)]
+    # a path the user's own rules ignore would make `git add` fail and stop every commit
+    hidden = _git(root, "check-ignore", "--", *paths).stdout.split() if paths else []
+    paths = [p for p in paths if p not in hidden]
+    if not paths:
+        return False
+    added = _git(root, "add", "--", *paths)
+    if added.returncode != 0:
+        raise GitError(added.stderr.strip())
+    if _git(root, "diff", "--cached", "--quiet", "--", *paths).returncode == 0:
+        return False
+    done = _git(root, "commit", "-m", message, "--", *paths)
+    if done.returncode != 0:
+        raise GitError(done.stderr.strip() or done.stdout.strip())
+    return True
+
+
+def push_vault(root: Path) -> str | None:
+    """Back the vault up to its `origin`, if it has one. Returns None when done or when there is
+    no remote, else the reason: a backup that is unreachable must never stop a run."""
+    if not (root / ".git").exists() or _git(root, "remote", "get-url", "origin").returncode != 0:
+        return None
+    try:
+        done = subprocess.run(
+            ["git", "push", "origin", "HEAD", "--tags"],
+            cwd=root,
+            capture_output=True,
+            text=True,
+            timeout=GIT_PUSH_TIMEOUT_SECONDS,
+        )
+    except subprocess.TimeoutExpired:
+        return "git push timed out"
+    return None if done.returncode == 0 else (done.stderr.strip() or done.stdout.strip())