bmlib 0.4.0__py3-none-any.whl

bmlib/__init__.py

@@ -0,0 +1,19 @@
+# bmlib — shared library for biomedical literature tools
+# Copyright (C) 2024-2026 Dr Horst Herb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""bmlib — shared library for biomedical literature tools."""
+
+__version__ = "0.4.0"

bmlib/agents/__init__.py

@@ -0,0 +1,21 @@
+# bmlib — shared library for biomedical literature tools
+# Copyright (C) 2024-2026 Dr Horst Herb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Agent base class for LLM-powered tasks."""
+
+from bmlib.agents.base import BaseAgent
+
+__all__ = ["BaseAgent"]

bmlib/agents/base.py

@@ -0,0 +1,269 @@
+# bmlib — shared library for biomedical literature tools
+# Copyright (C) 2024-2026 Dr Horst Herb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Base agent class for LLM-powered tasks.
+
+Provides shared infrastructure for agents that call LLMs:
+- Model/provider resolution from externally-supplied configuration
+- Helper methods for building messages
+- JSON response parsing
+
+Unlike the bmlibrarian_lite ``LiteBaseAgent``, this class does **not**
+read config from a hardcoded path.  The calling application passes in
+the model string and LLM client explicitly.
+
+Usage::
+
+    class ScoringAgent(BaseAgent):
+        def score(self, title: str, abstract: str, interests: list[str]) -> dict:
+            prompt = self.render_template("scoring.txt", ...)
+            response = self.chat(
+                [self.system_msg("You are ..."), self.user_msg(prompt)],
+                json_mode=True,
+            )
+            return self.parse_json(response.content)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any
+
+from bmlib.llm import LLMClient, LLMMessage, LLMResponse
+from bmlib.llm.json_repair import extract_and_repair_json
+from bmlib.llm.utils import extract_json
+from bmlib.templates import TemplateEngine
+
+logger = logging.getLogger(__name__)
+
+# Provider stop_reason values that mean the output hit the max_tokens ceiling:
+# Anthropic reports "max_tokens", OpenAI-compatible providers "length".
+_TRUNCATION_STOP_REASONS = ("max_tokens", "length")
+
+
+class BaseAgent:
+    """Base class for LLM-powered agents.
+
+    Args:
+        llm: The LLM client to use.
+        model: Full model string (``"provider:model_name"``).
+        template_engine: Template engine for loading prompt files.
+        temperature: Default sampling temperature.
+        max_tokens: Default max tokens.
+    """
+
+    def __init__(
+        self,
+        llm: LLMClient,
+        model: str,
+        template_engine: TemplateEngine | None = None,
+        temperature: float = 0.3,
+        max_tokens: int = 4096,
+    ) -> None:
+        self.llm = llm
+        self.model = model
+        self.templates = template_engine
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+
+    # --- Message helpers ---
+
+    @staticmethod
+    def system_msg(content: str) -> LLMMessage:
+        return LLMMessage(role="system", content=content)
+
+    @staticmethod
+    def user_msg(content: str) -> LLMMessage:
+        return LLMMessage(role="user", content=content)
+
+    @staticmethod
+    def assistant_msg(content: str) -> LLMMessage:
+        return LLMMessage(role="assistant", content=content)
+
+    # --- LLM interaction ---
+
+    def chat(
+        self,
+        messages: list[LLMMessage],
+        *,
+        json_mode: bool = False,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+        **kwargs: object,
+    ) -> LLMResponse:
+        """Send a chat request through the LLM client."""
+        return self.llm.chat(
+            messages=messages,
+            model=self.model,
+            temperature=temperature if temperature is not None else self.temperature,
+            max_tokens=max_tokens if max_tokens is not None else self.max_tokens,
+            json_mode=json_mode,
+            **kwargs,
+        )
+
+    def chat_json(
+        self,
+        messages: list[LLMMessage],
+        *,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+        max_retries: int = 3,
+        **kwargs: object,
+    ) -> dict:
+        """Chat with JSON mode, retry on empty/unparseable responses.
+
+        Combines :meth:`chat` with :meth:`parse_json` and exponential
+        backoff retry.  Empty responses are treated as transport/model
+        errors (WARNING).  Unparseable responses are logged at ERROR
+        with the full model output for diagnosis.
+
+        A response that stopped because it hit the ``max_tokens``
+        ceiling is reported as truncation, not "unparseable response".
+        At temperature 0 the retry is provably futile (greedy sampling
+        reproduces the identical truncation), so it raises immediately;
+        at temperature > 0 a retry may sample a shorter completion that
+        fits, so truncation still gets the normal retries, but the
+        final error names the real cause.
+
+        Returns the parsed dict.  Raises :class:`ValueError` on
+        truncation or after all retries are exhausted.
+        """
+        last_error: str | None = None
+        for attempt in range(max_retries):
+            if attempt > 0:
+                delay = 2 ** (attempt - 1)  # 1s, 2s, 4s …
+                logger.warning(
+                    "Retry %d/%d after %.0fs (previous: %s)",
+                    attempt + 1,
+                    max_retries,
+                    delay,
+                    last_error,
+                )
+                time.sleep(delay)
+
+            response = self.chat(
+                messages,
+                json_mode=True,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                **kwargs,
+            )
+
+            content = response.content.strip()
+
+            if response.stop_reason in _TRUNCATION_STOP_REASONS:
+                parsed = self._try_parse(content)
+                if parsed is not None:
+                    # The JSON happens to be complete despite hitting the
+                    # ceiling — usable as-is.
+                    return parsed
+                budget = max_tokens if max_tokens is not None else self.max_tokens
+                truncated = (
+                    f"response truncated at max_tokens={budget} "
+                    f"(stop_reason={response.stop_reason!r}) — raise max_tokens "
+                    "or request less output"
+                )
+                logger.error(
+                    "LLM response truncated (attempt %d/%d), full response: %s",
+                    attempt + 1,
+                    max_retries,
+                    content,
+                )
+                effective_temperature = temperature if temperature is not None else self.temperature
+                if effective_temperature == 0.0:
+                    # Greedy sampling reproduces the identical truncation;
+                    # retrying only pays for it again.
+                    raise ValueError(truncated) from None
+                # A retry at temperature > 0 may sample a shorter completion
+                # that fits.
+                last_error = truncated
+                continue
+
+            if not content:
+                last_error = "empty response from model"
+                logger.warning(
+                    "LLM returned empty response (attempt %d/%d)",
+                    attempt + 1,
+                    max_retries,
+                )
+                continue
+
+            try:
+                return self.parse_json(content)
+            except ValueError:
+                last_error = "unparseable response"
+                logger.error(
+                    "LLM returned unparseable response (attempt %d/%d), full response: %s",
+                    attempt + 1,
+                    max_retries,
+                    content,
+                )
+                continue
+
+        raise ValueError(f"Failed after {max_retries} attempts: {last_error}")
+
+    # --- Template rendering ---
+
+    def render_template(self, template_name: str, **variables: Any) -> str:
+        """Render a prompt template.  Raises if no template engine configured."""
+        if self.templates is None:
+            raise RuntimeError(f"No template engine configured — cannot render {template_name!r}")
+        return self.templates.render(template_name, **variables)
+
+    # --- JSON parsing ---
+
+    @classmethod
+    def _try_parse(cls, text: str) -> dict | None:
+        """:meth:`parse_json`, but ``None`` instead of ``ValueError`` on failure."""
+        if not text:
+            return None
+        try:
+            return cls.parse_json(text)
+        except ValueError:
+            return None
+
+    @staticmethod
+    def parse_json(text: str) -> dict:
+        """Extract and parse JSON from LLM response text.
+
+        Handles responses wrapped in markdown code blocks.
+        """
+        # Try direct parse
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            pass
+
+        # Fall back to the shared extractor (code-block aware + balanced-brace
+        # scanning that picks the first parseable object).
+        candidate = extract_json(text)
+        if candidate != text:
+            try:
+                return json.loads(candidate)
+            except json.JSONDecodeError:
+                pass
+
+        # Last resort: repair common LLM JSON defects (single quotes, trailing
+        # commas, truncation, unquoted keys) after extracting the JSON span.
+        try:
+            repaired, _ = extract_and_repair_json(text)
+            return json.loads(repaired)
+        except (ValueError, json.JSONDecodeError):
+            pass
+
+        raise ValueError(f"Could not parse JSON from LLM response: {text[:200]!r}")

bmlib/db/__init__.py

@@ -0,0 +1,57 @@
+# bmlib — shared library for biomedical literature tools
+# Copyright (C) 2024-2026 Dr Horst Herb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Thin database abstraction — pure functions over DB-API connections.
+
+Supports SQLite (built-in) and PostgreSQL (optional, via psycopg2).
+
+Usage::
+
+    from bmlib.db import connect_sqlite, fetch_all, execute, transaction
+
+    conn = connect_sqlite("~/.myapp/data.db")
+    with transaction(conn):
+        execute(conn, "INSERT INTO papers (doi, title) VALUES (?, ?)", ("10.1101/x", "A paper"))
+    rows = fetch_all(conn, "SELECT * FROM papers")
+"""
+
+from bmlib.db.connection import connect_postgresql, connect_sqlite
+from bmlib.db.migrations import Migration, run_migrations
+from bmlib.db.operations import (
+    create_tables,
+    execute,
+    executemany,
+    fetch_all,
+    fetch_one,
+    fetch_scalar,
+    table_exists,
+)
+from bmlib.db.transactions import transaction
+
+__all__ = [
+    "connect_sqlite",
+    "connect_postgresql",
+    "execute",
+    "executemany",
+    "fetch_one",
+    "fetch_all",
+    "fetch_scalar",
+    "table_exists",
+    "create_tables",
+    "transaction",
+    "Migration",
+    "run_migrations",
+]

bmlib/db/connection.py

@@ -0,0 +1,100 @@
+# bmlib — shared library for biomedical literature tools
+# Copyright (C) 2024-2026 Dr Horst Herb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Database connection factories.
+
+Each function returns a standard DB-API 2.0 connection.  SQLite uses the
+built-in ``sqlite3`` module; PostgreSQL uses ``psycopg2`` (optional
+dependency).
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def connect_sqlite(
+    path: str | Path,
+    *,
+    wal_mode: bool = True,
+    foreign_keys: bool = True,
+) -> sqlite3.Connection:
+    """Open (or create) a SQLite database and return a connection.
+
+    Args:
+        path: File path (``":memory:"`` for in-memory).
+        wal_mode: Enable WAL journal mode for better concurrent access.
+        foreign_keys: Enforce foreign key constraints.
+    """
+    path = str(Path(path).expanduser()) if path != ":memory:" else ":memory:"
+
+    if path != ":memory:":
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+
+    conn = sqlite3.connect(path, check_same_thread=False)
+    conn.row_factory = sqlite3.Row
+
+    if wal_mode and path != ":memory:":
+        conn.execute("PRAGMA journal_mode=WAL")
+    if foreign_keys:
+        conn.execute("PRAGMA foreign_keys=ON")
+
+    logger.debug("SQLite connection opened: %s", path)
+    return conn
+
+
+def connect_postgresql(
+    dsn: str | None = None,
+    *,
+    host: str = "localhost",
+    port: int = 5432,
+    database: str = "bmlib",
+    user: str = "bmlib",
+    password: str = "",
+) -> Any:
+    """Open a PostgreSQL connection via psycopg2.
+
+    Either provide a full *dsn* string, or individual parameters.
+
+    Returns:
+        A ``psycopg2`` connection with ``RealDictCursor`` as the default
+        cursor factory.
+    """
+    try:
+        import psycopg2
+        import psycopg2.extras
+    except ImportError:
+        raise ImportError("psycopg2 not installed. Install with: pip install bmlib[postgresql]")
+
+    if dsn:
+        conn = psycopg2.connect(dsn, cursor_factory=psycopg2.extras.RealDictCursor)
+    else:
+        conn = psycopg2.connect(
+            host=host,
+            port=port,
+            database=database,
+            user=user,
+            password=password,
+            cursor_factory=psycopg2.extras.RealDictCursor,
+        )
+
+    logger.debug("PostgreSQL connection opened: %s:%s/%s", host, port, database)
+    return conn

bmlib/db/migrations.py

@@ -0,0 +1,141 @@
+# bmlib — shared library for biomedical literature tools
+# Copyright (C) 2024-2026 Dr Horst Herb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Idempotent database migration runner.
+
+Provides a simple, sequential migration system that tracks applied
+migrations in a ``schema_version`` table.  Each migration is a Python
+function that receives a DB-API connection.
+
+Usage::
+
+    from bmlib.db.migrations import Migration, run_migrations
+
+    def _m001_create_users(conn):
+        create_tables(conn, "CREATE TABLE IF NOT EXISTS users (...);")
+
+    MIGRATIONS = [Migration(1, "create_users", _m001_create_users)]
+
+    run_migrations(conn, MIGRATIONS)
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from bmlib.db.operations import create_tables, execute, fetch_all, table_exists
+from bmlib.db.transactions import transaction
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Migration:
+    """A single database migration.
+
+    Attributes:
+        version: Sequential integer (1, 2, 3, ...). Must be unique.
+        name: Short descriptive name (e.g. ``"initial_schema"``).
+        up: Callable that takes a DB-API connection and applies the DDL.
+    """
+
+    version: int
+    name: str
+    up: Callable[[Any], None]
+
+
+def _is_sqlite(conn: Any) -> bool:
+    """Return True if the connection is SQLite."""
+    return "sqlite3" in type(conn).__module__
+
+
+def _placeholder(conn: Any) -> str:
+    """Return the correct parameter placeholder for this connection."""
+    return "?" if _is_sqlite(conn) else "%s"
+
+
+def _ensure_version_table(conn: Any) -> None:
+    """Create the ``schema_version`` table if it does not exist."""
+    if table_exists(conn, "schema_version"):
+        return
+
+    if _is_sqlite(conn):
+        ddl = """\
+CREATE TABLE schema_version (
+    version INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+"""
+    else:
+        ddl = """\
+CREATE TABLE schema_version (
+    version INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    applied_at TIMESTAMP NOT NULL DEFAULT NOW()
+);
+"""
+    create_tables(conn, ddl)
+    logger.info("Created schema_version table")
+
+
+def get_applied_versions(conn: Any) -> set[int]:
+    """Return the set of migration version numbers already applied.
+
+    Returns an empty set if the ``schema_version`` table does not exist.
+    """
+    if not table_exists(conn, "schema_version"):
+        return set()
+
+    rows = fetch_all(conn, "SELECT version FROM schema_version")
+    return {r[0] if not isinstance(r, dict) else r["version"] for r in rows}
+
+
+def run_migrations(conn: Any, migrations: list[Migration]) -> int:
+    """Apply all pending migrations in version order.
+
+    Args:
+        conn: A DB-API connection (sqlite3 or psycopg2).
+        migrations: Ordered list of ``Migration`` objects.
+
+    Returns:
+        Number of migrations applied.
+    """
+    _ensure_version_table(conn)
+    applied = get_applied_versions(conn)
+    ph = _placeholder(conn)
+
+    count = 0
+    for migration in sorted(migrations, key=lambda m: m.version):
+        if migration.version in applied:
+            continue
+
+        logger.info("Applying migration %d: %s", migration.version, migration.name)
+        with transaction(conn):
+            migration.up(conn)
+            execute(
+                conn,
+                f"INSERT INTO schema_version (version, name) VALUES ({ph}, {ph})",
+                (migration.version, migration.name),
+            )
+        count += 1
+
+    if count:
+        logger.info("Applied %d migration(s)", count)
+    return count