ingestlib 0.1.0__py3-none-any.whl

ingestlib/__init__.py

@@ -0,0 +1,4 @@
+"""ingestlib — self-hosted document intelligence library."""
+from ingestlib.utils.logger import _auto_configure as _auto_configure_logger
+
+_auto_configure_logger()

ingestlib/config.py

@@ -0,0 +1,231 @@
+"""Typed configuration for ingestlib — loaded lazily on first use.
+
+Importing ingestlib never touches the filesystem; config loads when the
+first operation needs it. Discovery order for config.yaml:
+
+    1. INGESTLIB_CONFIG environment variable (explicit path)
+    2. config.yaml in the current working directory, then each parent
+
+Secrets never live in config.yaml: a .env sitting next to the discovered
+config file is loaded automatically (JINA_API_KEY, PINECONE_API_KEY,
+QDRANT_URL/_API_KEY), and AWS credentials resolve via the profile field
+against ~/.aws/credentials — the standard boto3 chain.
+"""
+import os
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+from dotenv import load_dotenv
+
+
+_CONFIG_ENV_VAR = "INGESTLIB_CONFIG"
+_CONFIG_FILENAME = "config.yaml"
+
+_lock = threading.Lock()
+
+
+@dataclass(frozen=True)
+class AWSConfig:
+    profile: str
+    region: str
+    account_id: str
+
+
+@dataclass(frozen=True)
+class BedrockConfig:
+    llm_model_id: str
+    embedding_model_id: str
+    rerank_model_id: str
+    rerank_region: str          # separate region for rerank (amazon.rerank-v1:0 not in us-east-1)
+
+
+@dataclass(frozen=True)
+class JinaConfig:
+    api_key: str                # from JINA_API_KEY env var
+    base_url: str
+    rerank_model_id: str
+
+
+@dataclass(frozen=True)
+class PaddleVLConfig:
+    backend: str                # mlx-vlm-server (Apple Silicon) | vllm-server (NVIDIA)
+    server_url: str             # VLM inference server URL
+    api_model_name: str
+
+
+@dataclass(frozen=True)
+class S3Config:
+    bucket: str                 # globally unique bucket name for pipeline artifacts
+
+
+@dataclass(frozen=True)
+class PineconeConfig:
+    api_key: str                # from PINECONE_API_KEY env var
+    index_name: str
+    sparse_index_name: str      # lexical half of hybrid search (created on first upsert)
+    sparse_model_id: str        # Pinecone-hosted sparse embedding model (no corpus state)
+    cloud: str                  # serverless cloud provider (aws | gcp | azure)
+    region: str
+
+
+@dataclass(frozen=True)
+class QdrantConfig:
+    api_key: str                # from QDRANT_API_KEY env var ("" for a local/unsecured server)
+    url: str                    # from QDRANT_URL env var (e.g. http://localhost:6333 or a cloud URL)
+    collection_name: str
+
+
+@dataclass(frozen=True)
+class IngestConfig:
+    vector_store: str           # which connector the services default to (pinecone | qdrant)
+    aws: AWSConfig
+    bedrock: BedrockConfig
+    jina: JinaConfig
+    paddle_vl: PaddleVLConfig
+    s3: S3Config
+    pinecone: PineconeConfig
+    qdrant: QdrantConfig
+
+
+def _find_config_path() -> Path:
+    """Locate config.yaml: INGESTLIB_CONFIG first, then CWD and its parents."""
+    explicit = os.environ.get(_CONFIG_ENV_VAR)
+    if explicit:
+        path = Path(explicit).expanduser()
+        if not path.is_file():
+            raise FileNotFoundError(
+                f"{_CONFIG_ENV_VAR} points to {path}, which does not exist"
+            )
+        return path
+    cwd = Path.cwd()
+    for directory in (cwd, *cwd.parents):
+        candidate = directory / _CONFIG_FILENAME
+        if candidate.is_file():
+            return candidate
+    raise FileNotFoundError(
+        f"{_CONFIG_FILENAME} not found in {cwd} or any parent directory — "
+        f"copy config.example.yaml from https://github.com/LangModule/ingestlib "
+        f"into your project, or set {_CONFIG_ENV_VAR}=/path/to/config.yaml"
+    )
+
+
+def _load_config() -> IngestConfig:
+    config_path = _find_config_path()
+    # secrets conventionally sit next to the config file
+    load_dotenv(config_path.parent / ".env")
+
+    with open(config_path, "r") as f:
+        data = yaml.safe_load(f)
+
+    aws_data = data["aws"]
+    aws_config = AWSConfig(
+        profile=aws_data["profile"],
+        region=aws_data["region"],
+        account_id=str(aws_data["account_id"]),
+    )
+
+    bedrock_data = data["bedrock"]
+    bedrock_config = BedrockConfig(
+        llm_model_id=bedrock_data["llm_model_id"],
+        embedding_model_id=bedrock_data["embedding_model_id"],
+        rerank_model_id=bedrock_data["rerank_model_id"],
+        rerank_region=bedrock_data["rerank_region"],
+    )
+
+    jina_data = data["jina"]
+    jina_config = JinaConfig(
+        api_key=os.environ.get("JINA_API_KEY", ""),
+        base_url=jina_data.get("base_url", "https://api.jina.ai/v1"),
+        rerank_model_id=jina_data["rerank_model_id"],
+    )
+
+    paddle_vl_data = data["paddle_vl"]
+    paddle_vl_config = PaddleVLConfig(
+        backend=paddle_vl_data.get("backend", "mlx-vlm-server"),
+        server_url=paddle_vl_data.get("server_url", "http://localhost:8111/"),
+        api_model_name=paddle_vl_data["api_model_name"],
+    )
+
+    s3_data = data.get("s3", {})
+    s3_config = S3Config(
+        bucket=s3_data.get("bucket", f"ingestlib-{aws_config.account_id}"),
+    )
+
+    pinecone_data = data.get("pinecone", {})
+    index_name = pinecone_data.get("index_name", "ingestlib")
+    pinecone_config = PineconeConfig(
+        api_key=os.environ.get("PINECONE_API_KEY", ""),
+        index_name=index_name,
+        sparse_index_name=pinecone_data.get("sparse_index_name", f"{index_name}-sparse"),
+        sparse_model_id=pinecone_data.get("sparse_model_id", "pinecone-sparse-english-v0"),
+        cloud=pinecone_data.get("cloud", "aws"),
+        region=pinecone_data.get("region", "us-east-1"),
+    )
+
+    qdrant_data = data.get("qdrant", {})
+    qdrant_config = QdrantConfig(
+        api_key=os.environ.get("QDRANT_API_KEY", ""),
+        url=os.environ.get("QDRANT_URL", "http://localhost:6333"),
+        collection_name=qdrant_data.get("collection_name", "ingestlib"),
+    )
+
+    return IngestConfig(
+        vector_store=data.get("vector_store", "pinecone"),
+        aws=aws_config,
+        bedrock=bedrock_config,
+        jina=jina_config,
+        paddle_vl=paddle_vl_config,
+        s3=s3_config,
+        pinecone=pinecone_config,
+        qdrant=qdrant_config,
+    )
+
+
+_config: IngestConfig | None = None
+
+
+def get_config() -> IngestConfig:
+    """Full typed configuration, loaded lazily from config.yaml + .env and cached."""
+    global _config
+    if _config is None:
+        with _lock:
+            if _config is None:
+                _config = _load_config()
+    return _config
+
+
+def get_aws_config() -> AWSConfig:
+    """AWS profile/region/account settings."""
+    return get_config().aws
+
+
+def get_bedrock_config() -> BedrockConfig:
+    """Bedrock model IDs and rerank region."""
+    return get_config().bedrock
+
+
+def get_jina_config() -> JinaConfig:
+    """Jina reranker endpoint and API key."""
+    return get_config().jina
+
+
+def get_paddle_vl_config() -> PaddleVLConfig:
+    """PaddleOCR-VL inference server settings."""
+    return get_config().paddle_vl
+
+
+def get_s3_config() -> S3Config:
+    """S3 artifact bucket settings."""
+    return get_config().s3
+
+
+def get_pinecone_config() -> PineconeConfig:
+    """Pinecone index settings and API key."""
+    return get_config().pinecone
+
+
+def get_qdrant_config() -> QdrantConfig:
+    """Qdrant endpoint, API key, and collection settings."""
+    return get_config().qdrant

ingestlib/foundations/__init__.py

@@ -0,0 +1,10 @@
+"""Foundation layers the operations are built on.
+
+Sub-packages:
+    llm — Bedrock Nova (chat / thinking / structured output), multimodal
+          embeddings, and reranking (Jina primary, Amazon retained)
+    ocr — PaddleOCR-VL engine (layout + recognition) and the shared region types
+
+Operations (parse / classify / split) compose these; nothing here knows the
+operations exist.
+"""

ingestlib/foundations/llm/__init__.py

@@ -0,0 +1,48 @@
+"""Public LLM surface.
+
+- LLM/embedding primitives come from Bedrock (Nova family).
+- Rerank is exposed twice with explicit provider suffixes — no ambiguous default:
+    aws_rerank / aws_arerank  → amazon.rerank-v1:0 (quota-limited, keep for later)
+    jina_rerank / jina_arerank → Jina Reranker API (primary)
+"""
+from ingestlib.foundations.llm.bedrock import (
+    Image,
+    achat,
+    achat_structured,
+    achat_with_thinking,
+    aembed_image,
+    aembed_text,
+    chat,
+    chat_structured,
+    chat_with_thinking,
+    embed_image,
+    embed_text,
+    get_llm,
+    get_llm_with_thinking,
+    reset_clients,
+)
+from ingestlib.foundations.llm.bedrock import arerank as aws_arerank
+from ingestlib.foundations.llm.bedrock import rerank as aws_rerank
+from ingestlib.foundations.llm.jina import arerank as jina_arerank
+from ingestlib.foundations.llm.jina import rerank as jina_rerank
+
+__all__ = [
+    "Image",
+    "get_llm",
+    "get_llm_with_thinking",
+    "chat",
+    "chat_with_thinking",
+    "achat",
+    "chat_structured",
+    "achat_structured",
+    "achat_with_thinking",
+    "embed_text",
+    "embed_image",
+    "aembed_text",
+    "aembed_image",
+    "aws_rerank",
+    "aws_arerank",
+    "jina_rerank",
+    "jina_arerank",
+    "reset_clients",
+]

ingestlib/foundations/llm/bedrock/__init__.py

@@ -0,0 +1,39 @@
+"""Bedrock backend: Nova LLM/embedding + amazon.rerank-v1:0. Clients built in factory.py."""
+from ingestlib.foundations.llm.bedrock.embedding import (
+    aembed_image,
+    aembed_text,
+    embed_image,
+    embed_text,
+)
+from ingestlib.foundations.llm.bedrock.factory import reset_clients
+from ingestlib.foundations.llm.bedrock.nova import (
+    Image,
+    achat,
+    achat_structured,
+    achat_with_thinking,
+    chat,
+    chat_structured,
+    chat_with_thinking,
+    get_llm,
+    get_llm_with_thinking,
+)
+from ingestlib.foundations.llm.bedrock.rerank import arerank, rerank
+
+__all__ = [
+    "Image",
+    "get_llm",
+    "get_llm_with_thinking",
+    "chat",
+    "chat_with_thinking",
+    "achat",
+    "chat_structured",
+    "achat_structured",
+    "achat_with_thinking",
+    "embed_text",
+    "embed_image",
+    "aembed_text",
+    "aembed_image",
+    "rerank",
+    "arerank",
+    "reset_clients",
+]

ingestlib/foundations/llm/bedrock/embedding.py

@@ -0,0 +1,127 @@
+"""Nova 2 multimodal embeddings — text and image embed primitives (sync and async)."""
+import asyncio
+import base64
+import json
+import time
+from typing import Any, Literal
+
+from ingestlib.config import get_bedrock_config
+from ingestlib.foundations.llm.bedrock.factory import get_runtime_client
+from ingestlib.utils.logger import get_logger
+
+
+logger = get_logger(__name__)
+
+EmbeddingPurpose = Literal["GENERIC_INDEX", "GENERIC_RETRIEVAL", "DOCUMENT_RETRIEVAL"]
+EmbeddingDimension = Literal[256, 384, 1024, 3072]
+ImageFormat = Literal["jpeg", "png", "webp", "gif"]
+ImageDetailLevel = Literal["STANDARD_IMAGE", "DOCUMENT_IMAGE"]
+
+DEFAULT_DIMENSION: EmbeddingDimension = 1024
+SUPPORTED_DIMENSIONS: tuple[int, ...] = (256, 384, 1024, 3072)
+
+
+def _validate_dimension(dimension: int) -> None:
+    if dimension not in SUPPORTED_DIMENSIONS:
+        raise ValueError(
+            f"dimension must be one of {SUPPORTED_DIMENSIONS}, got {dimension}"
+        )
+
+
+def _invoke(body: dict[str, Any]) -> dict[str, Any]:
+    client = get_runtime_client()
+    cfg = get_bedrock_config()
+    response = client.invoke_model(
+        modelId=cfg.embedding_model_id,
+        body=json.dumps(body),
+        contentType="application/json",
+        accept="application/json",
+    )
+    return json.loads(response["body"].read())
+
+
+def _embed(
+    *,
+    content: dict[str, Any],
+    purpose: EmbeddingPurpose,
+    dimension: EmbeddingDimension,
+) -> list[float]:
+    _validate_dimension(dimension)
+    body = {
+        "taskType": "SINGLE_EMBEDDING",
+        "singleEmbeddingParams": {
+            "embeddingPurpose": purpose,
+            "embeddingDimension": dimension,
+            **content,
+        },
+    }
+    return _invoke(body)["embeddings"][0]["embedding"]
+
+
+def embed_text(
+    text: str,
+    purpose: EmbeddingPurpose = "GENERIC_INDEX",
+    dimension: EmbeddingDimension = DEFAULT_DIMENSION,
+) -> list[float]:
+    """Embed text → vector of `dimension` floats (truncates overlong input at the end)."""
+    logger.info(
+        "embed_text: purpose=%s dim=%d input_len=%d", purpose, dimension, len(text),
+    )
+    t0 = time.perf_counter()
+    result = _embed(
+        content={"text": {"truncationMode": "END", "value": text}},
+        purpose=purpose,
+        dimension=dimension,
+    )
+    logger.info("embed_text done: %.2fs returned_dim=%d", time.perf_counter() - t0, len(result))
+    return result
+
+
+def embed_image(
+    data: bytes,
+    format: ImageFormat,
+    purpose: EmbeddingPurpose = "GENERIC_INDEX",
+    dimension: EmbeddingDimension = DEFAULT_DIMENSION,
+    detail_level: ImageDetailLevel = "STANDARD_IMAGE",
+) -> list[float]:
+    """Embed an image → vector of `dimension` floats. DOCUMENT_IMAGE detail for doc pages."""
+    logger.info(
+        "embed_image: purpose=%s dim=%d format=%s detail_level=%s size=%d bytes",
+        purpose, dimension, format, detail_level, len(data),
+    )
+    t0 = time.perf_counter()
+    result = _embed(
+        content={
+            "image": {
+                "format": format,
+                "detailLevel": detail_level,
+                "source": {"bytes": base64.b64encode(data).decode()},
+            }
+        },
+        purpose=purpose,
+        dimension=dimension,
+    )
+    logger.info("embed_image done: %.2fs returned_dim=%d", time.perf_counter() - t0, len(result))
+    return result
+
+
+async def aembed_text(
+    text: str,
+    purpose: EmbeddingPurpose = "GENERIC_INDEX",
+    dimension: EmbeddingDimension = DEFAULT_DIMENSION,
+) -> list[float]:
+    """Async embed_text() — runs the sync Bedrock client in a worker thread."""
+    return await asyncio.to_thread(embed_text, text, purpose, dimension)
+
+
+async def aembed_image(
+    data: bytes,
+    format: ImageFormat,
+    purpose: EmbeddingPurpose = "GENERIC_INDEX",
+    dimension: EmbeddingDimension = DEFAULT_DIMENSION,
+    detail_level: ImageDetailLevel = "STANDARD_IMAGE",
+) -> list[float]:
+    """Async embed_image() — runs the sync Bedrock client in a worker thread."""
+    return await asyncio.to_thread(
+        embed_image, data, format, purpose, dimension, detail_level
+    )

ingestlib/foundations/llm/bedrock/factory.py

@@ -0,0 +1,101 @@
+"""Shared boto3 session, Bedrock clients, and generation-keyed model cache."""
+import threading
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ProfileNotFound
+
+from ingestlib.config import get_aws_config, get_bedrock_config
+from ingestlib.utils.logger import get_logger
+
+
+logger = get_logger(__name__)
+_lock = threading.Lock()
+
+_session: boto3.Session | None = None
+_runtime_client = None              # bedrock-runtime      (LLM inference + embeddings)
+_rerank_agent_client = None         # bedrock-agent-runtime in cfg.rerank_region (for rerank)
+_client_generation: int = 0
+_model_cache: dict[str, tuple[object, int]] = {}
+
+
+def _build_clients() -> None:
+    global _session, _runtime_client, _rerank_agent_client, _client_generation
+
+    aws = get_aws_config()
+    bedrock = get_bedrock_config()
+    logger.info(
+        "building Bedrock clients: profile=%r region=%s rerank_region=%s",
+        aws.profile, aws.region, bedrock.rerank_region,
+    )
+
+    profile = (aws.profile or "").strip()
+    try:
+        _session = (
+            boto3.Session(profile_name=profile, region_name=aws.region)
+            if profile
+            else boto3.Session(region_name=aws.region)
+        )
+    except ProfileNotFound:
+        logger.warning("profile %r not found, falling back to default session", profile)
+        _session = boto3.Session(region_name=aws.region)
+
+    retry_cfg = Config(
+        retries={"total_max_attempts": 6, "mode": "standard"},
+        connect_timeout=10,
+        read_timeout=3600,
+    )
+    _runtime_client = _session.client(
+        "bedrock-runtime", region_name=aws.region, config=retry_cfg
+    )
+    _rerank_agent_client = _session.client(
+        "bedrock-agent-runtime", region_name=bedrock.rerank_region, config=retry_cfg
+    )
+    _client_generation += 1
+    _model_cache.clear()
+    logger.debug("clients built (generation=%d)", _client_generation)
+
+
+def _ensure() -> None:
+    if _runtime_client is None:
+        _build_clients()
+
+
+def get_runtime_client():
+    """Return the shared boto3 bedrock-runtime client (LLM + embeddings)."""
+    with _lock:
+        _ensure()
+        return _runtime_client
+
+
+def get_rerank_agent_client():
+    """Return the boto3 bedrock-agent-runtime client bound to cfg.rerank_region."""
+    with _lock:
+        _ensure()
+        return _rerank_agent_client
+
+
+def reset_clients() -> None:
+    """Force client recreation on the next call (e.g. after credential rotation)."""
+    global _session, _runtime_client, _rerank_agent_client
+    with _lock:
+        logger.info("resetting Bedrock clients (next call will rebuild)")
+        _session = None
+        _runtime_client = None
+        _rerank_agent_client = None
+
+
+def get_model(key: str) -> object | None:
+    """Return a cached model instance if still valid for the current client generation."""
+    with _lock:
+        _ensure()
+        entry = _model_cache.get(key)
+        if entry is not None and entry[1] == _client_generation:
+            return entry[0]
+        return None
+
+
+def cache_model(key: str, model: object) -> None:
+    """Store a model instance keyed to the current client generation."""
+    with _lock:
+        _model_cache[key] = (model, _client_generation)