qdrant-loader-core 0.7.1__py3-none-any.whl

qdrant_loader_core/__init__.py

@@ -0,0 +1,25 @@
+# qdrant-loader-core package root
+
+from .llm import (
+    ChatClient,
+    EmbeddingPolicy,
+    EmbeddingsClient,
+    LLMProvider,
+    LLMSettings,
+    RateLimitPolicy,
+    RequestPolicy,
+    TokenCounter,
+    create_provider,
+)
+
+__all__ = [
+    "EmbeddingsClient",
+    "ChatClient",
+    "TokenCounter",
+    "LLMProvider",
+    "LLMSettings",
+    "RequestPolicy",
+    "RateLimitPolicy",
+    "EmbeddingPolicy",
+    "create_provider",
+]

qdrant_loader_core/llm/__init__.py

@@ -0,0 +1,17 @@
+# Re-export core interfaces for convenience
+
+from .factory import create_provider
+from .settings import EmbeddingPolicy, LLMSettings, RateLimitPolicy, RequestPolicy
+from .types import ChatClient, EmbeddingsClient, LLMProvider, TokenCounter
+
+__all__ = [
+    "EmbeddingsClient",
+    "ChatClient",
+    "TokenCounter",
+    "LLMProvider",
+    "LLMSettings",
+    "RequestPolicy",
+    "RateLimitPolicy",
+    "EmbeddingPolicy",
+    "create_provider",
+]

qdrant_loader_core/llm/errors.py

@@ -0,0 +1,22 @@
+class LLMError(Exception):
+    pass
+
+
+class TimeoutError(LLMError):
+    pass
+
+
+class RateLimitedError(LLMError):
+    pass
+
+
+class InvalidRequestError(LLMError):
+    pass
+
+
+class AuthError(LLMError):
+    pass
+
+
+class ServerError(LLMError):
+    pass

qdrant_loader_core/llm/factory.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from .providers.ollama import OllamaProvider
+from .providers.openai import OpenAIProvider
+
+try:
+    from .providers.azure_openai import AzureOpenAIProvider  # type: ignore
+except Exception:  # pragma: no cover - optional dependency surface
+    AzureOpenAIProvider = None  # type: ignore
+from .settings import LLMSettings
+from .types import ChatClient, EmbeddingsClient, LLMProvider, TokenCounter
+
+
+class _NoopEmbeddings(EmbeddingsClient):
+    async def embed(self, inputs: list[str]) -> list[list[float]]:
+        raise NotImplementedError("Embeddings provider not implemented")
+
+
+class _NoopChat(ChatClient):
+    async def chat(self, messages, **kwargs):  # type: ignore[no-untyped-def]
+        raise NotImplementedError("Chat provider not implemented")
+
+
+class _NoopTokenizer(TokenCounter):
+    def count(self, text: str) -> int:  # naive char-count fallback
+        return len(text)
+
+
+class _NoopProvider(LLMProvider):
+    def embeddings(self) -> EmbeddingsClient:
+        return _NoopEmbeddings()
+
+    def chat(self) -> ChatClient:
+        return _NoopChat()
+
+    def tokenizer(self) -> TokenCounter:
+        return _NoopTokenizer()
+
+
+def _safe_hostname(url: str | None) -> str | None:
+    if not url:
+        return None
+    try:
+        host = urlparse(url).hostname
+        return host.lower() if host else None
+    except Exception:
+        return None
+
+
+def create_provider(settings: LLMSettings) -> LLMProvider:
+    """Create a provider by settings.
+
+    Phase 0: route OpenAI/OpenAI-compatible to OpenAIProvider when available; otherwise return a noop provider.
+    Ollama returns a stub provider for now.
+    """
+    provider_name = (settings.provider or "").lower()
+    base_url = (settings.base_url or "")
+    base_host = _safe_hostname(base_url)
+
+    # Route Azure before generic OpenAI routing
+    is_azure = (
+        "azure" in provider_name
+        or (
+            base_host is not None
+            and (
+                base_host == "openai.azure.com"
+                or base_host.endswith(".openai.azure.com")
+                or base_host == "cognitiveservices.azure.com"
+                or base_host.endswith(".cognitiveservices.azure.com")
+            )
+        )
+    )
+    if is_azure and AzureOpenAIProvider is not None:  # type: ignore[truthy-bool]
+        try:
+            return AzureOpenAIProvider(settings)  # type: ignore[misc]
+        except Exception:
+            return _NoopProvider()
+
+    if "openai" in provider_name or "openai" in base_url.lower():
+        try:
+            return OpenAIProvider(settings)
+        except Exception:
+            return _NoopProvider()
+
+    if provider_name == "ollama" or (base_host in ("localhost", "127.0.0.1")):
+        return OllamaProvider(settings)
+
+    return _NoopProvider()

qdrant_loader_core/llm/providers/__init__.py

@@ -0,0 +1,8 @@
+# Namespace package for LLM providers
+
+__all__ = [
+    "openai",
+    "ollama",
+]
+
+

qdrant_loader_core/llm/providers/azure_openai.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urlparse
+
+try:
+    from openai import AzureOpenAI  # type: ignore
+except Exception:  # pragma: no cover - optional dependency surface
+    AzureOpenAI = None  # type: ignore
+
+from ...logging import LoggingConfig
+from ..settings import LLMSettings
+from ..types import ChatClient, EmbeddingsClient, LLMProvider, TokenCounter
+from .openai import OpenAIChat, OpenAIEmbeddings, _OpenAITokenCounter
+
+logger = LoggingConfig.get_logger(__name__)
+
+
+def _host_of(url: str | None) -> str | None:
+    if not url:
+        return None
+    try:
+        return urlparse(url).hostname or None
+    except Exception:
+        return None
+
+
+def _validate_azure_settings(settings: LLMSettings) -> None:
+    base_url = settings.base_url or ""
+    if "/openai/deployments" in base_url:
+        raise ValueError(
+            "Azure OpenAI base_url must be the resource root (e.g. https://<resource>.openai.azure.com). Do not include /openai/deployments/... in base_url."
+        )
+    if not (settings.api_version and isinstance(settings.api_version, str)):
+        raise ValueError(
+            "Azure OpenAI requires api_version (e.g. '2024-05-01-preview') in global.llm.api_version"
+        )
+
+
+class AzureOpenAIProvider(LLMProvider):
+    def __init__(self, settings: LLMSettings):
+        self._settings = settings
+        _validate_azure_settings(settings)
+
+        self._base_host = _host_of(settings.base_url)
+        if AzureOpenAI is None:
+            self._client = None
+        else:
+            # Prefer explicit azure_endpoint in provider_options; fallback to base_url
+            provider_opts = settings.provider_options or {}
+            endpoint = provider_opts.get("azure_endpoint") or settings.base_url
+            kwargs: dict[str, Any] = {
+                "api_key": settings.api_key,
+                "api_version": settings.api_version,
+            }
+            if endpoint:
+                kwargs["azure_endpoint"] = endpoint
+            self._client = AzureOpenAI(**{k: v for k, v in kwargs.items() if v is not None})
+
+    def embeddings(self) -> EmbeddingsClient:
+        model = self._settings.models.get("embeddings", "")
+        return OpenAIEmbeddings(self._client, model, self._base_host, provider_label="azure_openai")
+
+    def chat(self) -> ChatClient:
+        model = self._settings.models.get("chat", "")
+        return OpenAIChat(self._client, model, self._base_host, provider_label="azure_openai")
+
+    def tokenizer(self) -> TokenCounter:
+        return _OpenAITokenCounter(self._settings.tokenizer)
+
+

qdrant_loader_core/llm/providers/ollama.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+from typing import Any
+
+try:
+    import httpx  # type: ignore
+except Exception:  # pragma: no cover - optional dependency
+    httpx = None  # type: ignore
+
+from ...logging import LoggingConfig
+from ..errors import (
+    AuthError,
+    InvalidRequestError,
+    RateLimitedError,
+    ServerError,
+)
+from ..errors import (
+    TimeoutError as LLMTimeoutError,
+)
+from ..settings import LLMSettings
+from ..types import ChatClient, EmbeddingsClient, LLMProvider, TokenCounter
+
+logger = LoggingConfig.get_logger(__name__)
+
+
+def _join_url(base: str | None, path: str) -> str:
+    base = (base or "").rstrip("/")
+    path = path.lstrip("/")
+    return f"{base}/{path}" if base else f"/{path}"
+
+
+class OllamaEmbeddings(EmbeddingsClient):
+    def __init__(
+        self,
+        base_url: str | None,
+        model: str,
+        headers: dict[str, str] | None,
+        *,
+        timeout_s: float | None = None,
+        provider_options: dict[str, Any] | None = None,
+    ):
+        self._base_url = (base_url or "http://localhost:11434").rstrip("/")
+        self._model = model
+        self._headers = headers or {}
+        self._timeout_s = float(timeout_s) if timeout_s is not None else 30.0
+        self._provider_options = provider_options or {}
+
+    async def embed(self, inputs: list[str]) -> list[list[float]]:
+        if httpx is None:
+            raise NotImplementedError("httpx not available for Ollama embeddings")
+
+        # Prefer OpenAI-compatible if base_url seems to expose /v1
+        use_v1 = "/v1" in (self._base_url or "")
+        async with httpx.AsyncClient(timeout=self._timeout_s) as client:
+            try:
+                if use_v1:
+                    # OpenAI-compatible embeddings endpoint
+                    url = _join_url(self._base_url, "/embeddings")
+                    payload = {"model": self._model, "input": inputs}
+                    resp = await client.post(url, json=payload, headers=self._headers)
+                    resp.raise_for_status()
+                    data = resp.json()
+                    logger.info(
+                        "LLM request",
+                        provider="ollama",
+                        operation="embeddings",
+                        model=self._model,
+                        base_host=self._base_url,
+                        inputs=len(inputs),
+                        # latency for v1 path hard to compute here; omitted for now
+                    )
+                    return [item["embedding"] for item in data.get("data", [])]
+                else:
+                    # Determine native endpoint preference: embed | embeddings | auto (default)
+                    native_pref = str(self._provider_options.get("native_endpoint", "auto")).lower()
+                    prefer_embed = native_pref != "embeddings"
+
+                    # Try batch embed first when preferred
+                    if prefer_embed:
+                        url = _join_url(self._base_url, "/api/embed")
+                        payload = {"model": self._model, "input": inputs}
+                        try:
+                            resp = await client.post(
+                                url, json=payload, headers=self._headers
+                            )
+                            resp.raise_for_status()
+                            data = resp.json()
+                            vectors = data.get("embeddings")
+                            if not isinstance(vectors, list) or (
+                                len(vectors) != len(inputs)
+                            ):
+                                raise ValueError(
+                                    "Invalid embeddings response from /api/embed"
+                                )
+                            # Normalize to list[list[float]]
+                            norm = [list(vec) for vec in vectors]
+                            logger.info(
+                                "LLM request",
+                                provider="ollama",
+                                operation="embeddings",
+                                model=self._model,
+                                base_host=self._base_url,
+                                inputs=len(inputs),
+                                # latency for native batch path not measured in this stub
+                            )
+                            return norm
+                        except httpx.HTTPStatusError as exc:
+                            status = exc.response.status_code if exc.response else None
+                            # Fallback for servers that don't support /api/embed
+                            if status not in (404, 405, 501):
+                                raise
+
+                    # Per-item embeddings endpoint fallback or preference
+                    url = _join_url(self._base_url, "/api/embeddings")
+                    vectors2: list[list[float]] = []
+                    for text in inputs:
+                        payload = {"model": self._model, "input": text}
+                        resp = await client.post(
+                            url, json=payload, headers=self._headers
+                        )
+                        resp.raise_for_status()
+                        data = resp.json()
+                        emb = data.get("embedding")
+                        if emb is None and isinstance(data.get("data"), dict):
+                            emb = data["data"].get("embedding")
+                        if emb is None:
+                            raise ValueError(
+                                "Invalid embedding response from /api/embeddings"
+                            )
+                        vectors2.append(list(emb))
+                    logger.info(
+                        "LLM request",
+                        provider="ollama",
+                        operation="embeddings",
+                        model=self._model,
+                        base_host=self._base_url,
+                        inputs=len(inputs),
+                        # latency for per-item path not measured in this stub
+                    )
+                    return vectors2
+            except httpx.TimeoutException as exc:
+                raise LLMTimeoutError(str(exc))
+            except httpx.HTTPStatusError as exc:
+                status = exc.response.status_code if exc.response else None
+                if status == 401:
+                    raise AuthError(str(exc))
+                if status == 429:
+                    raise RateLimitedError(str(exc))
+                if status and 400 <= status < 500:
+                    raise InvalidRequestError(str(exc))
+                raise ServerError(str(exc))
+            except httpx.HTTPError as exc:
+                raise ServerError(str(exc))
+
+
+class OllamaChat(ChatClient):
+    def __init__(self, base_url: str | None, model: str, headers: dict[str, str] | None):
+        self._base_url = base_url or "http://localhost:11434"
+        self._model = model
+        self._headers = headers or {}
+
+    async def chat(self, messages: list[dict[str, Any]], **kwargs: Any) -> dict[str, Any]:
+        if httpx is None:
+            raise NotImplementedError("httpx not available for Ollama chat")
+
+        # Prefer OpenAI-compatible if base_url exposes /v1
+        use_v1 = "/v1" in (self._base_url or "")
+        # Flatten messages to a single prompt for native API; preserve roles when possible
+        if use_v1:
+            url = _join_url(self._base_url, "/chat/completions")
+            payload = {"model": self._model, "messages": messages}
+            # Map common kwargs
+            for k in ("temperature", "max_tokens", "top_p", "stop"):
+                if k in kwargs and kwargs[k] is not None:
+                    payload[k] = kwargs[k]
+            async with httpx.AsyncClient(timeout=60.0) as client:
+                try:
+                    from datetime import datetime
+                    started = datetime.utcnow()
+                    resp = await client.post(url, json=payload, headers=self._headers)
+                    resp.raise_for_status()
+                    data = resp.json()
+                    text = ""
+                    choices = data.get("choices") or []
+                    if choices:
+                        msg = (choices[0] or {}).get("message") or {}
+                        text = msg.get("content", "") or ""
+                    duration_ms = int((datetime.utcnow() - started).total_seconds() * 1000)
+                    logger.info(
+                        "LLM request",
+                        provider="ollama",
+                        operation="chat",
+                        model=self._model,
+                        base_host=self._base_url,
+                        messages=len(messages),
+                        latency_ms=duration_ms,
+                    )
+                    return {"text": text, "raw": data, "usage": data.get("usage"), "model": data.get("model", self._model)}
+                except httpx.TimeoutException as exc:
+                    raise LLMTimeoutError(str(exc))
+                except httpx.HTTPStatusError as exc:
+                    status = exc.response.status_code if exc.response else None
+                    if status == 401:
+                        raise AuthError(str(exc))
+                    if status == 429:
+                        raise RateLimitedError(str(exc))
+                    if status and 400 <= status < 500:
+                        raise InvalidRequestError(str(exc))
+                    raise ServerError(str(exc))
+                except httpx.HTTPError as exc:
+                    raise ServerError(str(exc))
+        else:
+            # Native API
+            url = _join_url(self._base_url, "/api/chat")
+            payload = {
+                "model": self._model,
+                "messages": messages,
+                "stream": False,
+            }
+            if "temperature" in kwargs and kwargs["temperature"] is not None:
+                payload["options"] = {"temperature": kwargs["temperature"]}
+            async with httpx.AsyncClient(timeout=60.0) as client:
+                try:
+                    from datetime import datetime
+                    started = datetime.utcnow()
+                    resp = await client.post(url, json=payload, headers=self._headers)
+                    resp.raise_for_status()
+                    data = resp.json()
+                    # Ollama native returns {"message": {"content": "..."}, ...}
+                    text = ""
+                    if isinstance(data.get("message"), dict):
+                        text = data["message"].get("content", "") or ""
+                    duration_ms = int((datetime.utcnow() - started).total_seconds() * 1000)
+                    logger.info(
+                        "LLM request",
+                        provider="ollama",
+                        operation="chat",
+                        model=self._model,
+                        base_host=self._base_url,
+                        messages=len(messages),
+                        latency_ms=duration_ms,
+                    )
+                    return {"text": text, "raw": data, "usage": None, "model": self._model}
+                except httpx.TimeoutException as exc:
+                    raise LLMTimeoutError(str(exc))
+                except httpx.HTTPStatusError as exc:
+                    status = exc.response.status_code if exc.response else None
+                    if status == 401:
+                        raise AuthError(str(exc))
+                    if status == 429:
+                        raise RateLimitedError(str(exc))
+                    if status and 400 <= status < 500:
+                        raise InvalidRequestError(str(exc))
+                    raise ServerError(str(exc))
+                except httpx.HTTPError as exc:
+                    raise ServerError(str(exc))
+
+
+class OllamaTokenizer(TokenCounter):
+    def count(self, text: str) -> int:
+        return len(text)
+
+
+class OllamaProvider(LLMProvider):
+    def __init__(self, settings: LLMSettings):
+        self._settings = settings
+
+    def embeddings(self) -> EmbeddingsClient:
+        model = self._settings.models.get("embeddings", "")
+        timeout = (self._settings.request.timeout_s if self._settings and self._settings.request else 30.0)
+        return OllamaEmbeddings(
+            self._settings.base_url,
+            model,
+            self._settings.headers,
+            timeout_s=timeout,
+            provider_options=self._settings.provider_options,
+        )
+
+    def chat(self) -> ChatClient:
+        model = self._settings.models.get("chat", "")
+        return OllamaChat(self._settings.base_url, model, self._settings.headers)
+
+    def tokenizer(self) -> TokenCounter:
+        return OllamaTokenizer()

qdrant_loader_core/llm/providers/openai.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+from urllib.parse import urlparse
+
+try:
+    from openai import OpenAI  # type: ignore
+    # New-style exception classes (OpenAI Python SDK >=1.x)
+    try:  # nested to avoid failing entirely on older clients
+        from openai import (  # type: ignore
+            APIConnectionError,
+            APIStatusError,
+            APITimeoutError,
+            AuthenticationError,
+            BadRequestError,
+            RateLimitError,
+        )
+    except Exception:  # pragma: no cover - optional dependency surface
+        APIConnectionError = APIStatusError = APITimeoutError = AuthenticationError = BadRequestError = RateLimitError = ()  # type: ignore
+except Exception:  # pragma: no cover - optional dependency at this phase
+    OpenAI = None  # type: ignore
+    APIConnectionError = APIStatusError = APITimeoutError = AuthenticationError = BadRequestError = RateLimitError = ()  # type: ignore
+
+from ...logging import LoggingConfig
+from ..errors import (
+    AuthError,
+    InvalidRequestError,
+    LLMError,
+    RateLimitedError,
+    ServerError,
+)
+from ..errors import (
+    TimeoutError as LLMTimeoutError,
+)
+from ..settings import LLMSettings
+from ..types import ChatClient, EmbeddingsClient, LLMProvider, TokenCounter
+
+logger = LoggingConfig.get_logger(__name__)
+
+
+def _safe_host(url: str | None) -> str | None:
+    if not url:
+        return None
+    try:
+        return urlparse(url).hostname or None
+    except Exception:
+        return None
+
+
+def _map_openai_exception(exc: Exception) -> LLMError:
+    try:
+        # Rate limit
+        if RateLimitError and isinstance(exc, RateLimitError):  # type: ignore[arg-type]
+            return RateLimitedError(str(exc))
+        # Timeout
+        if APITimeoutError and isinstance(exc, APITimeoutError):  # type: ignore[arg-type]
+            return LLMTimeoutError(str(exc))
+        # Auth
+        if AuthenticationError and isinstance(exc, AuthenticationError):  # type: ignore[arg-type]
+            return AuthError(str(exc))
+        # Bad request / invalid params
+        if BadRequestError and isinstance(exc, BadRequestError):  # type: ignore[arg-type]
+            return InvalidRequestError(str(exc))
+        # API status error (typically non-2xx)
+        if APIStatusError and isinstance(exc, APIStatusError):  # type: ignore[arg-type]
+            # Best-effort: check for status code
+            status_code = getattr(exc, "status_code", None) or getattr(
+                getattr(exc, "response", None), "status_code", None
+            )
+            if isinstance(status_code, int) and 400 <= status_code < 500:
+                if status_code == 429:
+                    return RateLimitedError(str(exc))
+                if status_code in (401, 403):
+                    return AuthError(str(exc))
+                return InvalidRequestError(str(exc))
+            return ServerError(str(exc))
+        # Connection-level errors
+        if APIConnectionError and isinstance(exc, APIConnectionError):  # type: ignore[arg-type]
+            return ServerError(str(exc))
+    except Exception:
+        pass
+    # Fallback
+    return ServerError(str(exc))
+
+
+class _OpenAITokenCounter(TokenCounter):
+    def __init__(self, tokenizer: str):
+        self._tokenizer = tokenizer
+
+    def count(self, text: str) -> int:
+        # Phase 0: fallback to naive length; real tiktoken impl to come later
+        return len(text)
+
+
+class OpenAIEmbeddings(EmbeddingsClient):
+    def __init__(
+        self, client: Any, model: str, base_host: str | None, *, provider_label: str = "openai"
+    ):
+        self._client = client
+        self._model = model
+        self._base_host = base_host
+        self._provider_label = provider_label
+
+    async def embed(self, inputs: list[str]) -> list[list[float]]:
+        if not self._client:
+            raise NotImplementedError("OpenAI client not available")
+        # Use thread offloading to keep async interface consistent with sync client
+        import asyncio
+
+        started = datetime.utcnow()
+        try:
+            response = await asyncio.to_thread(
+                self._client.embeddings.create, model=self._model, input=inputs
+            )
+            duration_ms = int((datetime.utcnow() - started).total_seconds() * 1000)
+            try:
+                logger.info(
+                    "LLM request",
+                    provider=self._provider_label,
+                    operation="embeddings",
+                    model=self._model,
+                    base_host=self._base_host,
+                    inputs=len(inputs),
+                    latency_ms=duration_ms,
+                )
+            except Exception:
+                pass
+            return [item.embedding for item in response.data]
+        except Exception as exc:  # Normalize errors
+            mapped = _map_openai_exception(exc)
+            try:
+                logger.warning(
+                    "LLM error",
+                    provider=self._provider_label,
+                    operation="embeddings",
+                    model=self._model,
+                    base_host=self._base_host,
+                    error=type(exc).__name__,
+                )
+            except Exception:
+                pass
+            raise mapped
+
+
+class OpenAIChat(ChatClient):
+    def __init__(
+        self, client: Any, model: str, base_host: str | None, *, provider_label: str = "openai"
+    ):
+        self._client = client
+        self._model = model
+        self._base_host = base_host
+        self._provider_label = provider_label
+
+    async def chat(
+        self, messages: list[dict[str, Any]], **kwargs: Any
+    ) -> dict[str, Any]:
+        if not self._client:
+            raise NotImplementedError("OpenAI client not available")
+
+        # Normalize kwargs to OpenAI python client parameters
+        create_kwargs: dict[str, Any] = {}
+        for key in ("temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "stop", "seed", "response_format"):
+            if key in kwargs and kwargs[key] is not None:
+                create_kwargs[key] = kwargs[key]
+
+        # Allow model override per-call
+        model_name = kwargs.pop("model", self._model)
+
+        import asyncio
+
+        # The OpenAI python client call is sync for chat.completions
+        started = datetime.utcnow()
+        try:
+            response = await asyncio.to_thread(
+                self._client.chat.completions.create,
+                model=model_name,
+                messages=messages,
+                **create_kwargs,
+            )
+            duration_ms = int((datetime.utcnow() - started).total_seconds() * 1000)
+            try:
+                logger.info(
+                    "LLM request",
+                    provider=self._provider_label,
+                    operation="chat",
+                    model=model_name,
+                    base_host=self._base_host,
+                    messages=len(messages),
+                    latency_ms=duration_ms,
+                )
+            except Exception:
+                pass
+
+            # Normalize to provider-agnostic dict
+            choice0 = response.choices[0] if getattr(response, "choices", None) else None
+            text = ""
+            if choice0 is not None:
+                message = getattr(choice0, "message", None)
+                if message is not None:
+                    text = getattr(message, "content", "") or ""
+
+            usage = getattr(response, "usage", None)
+            normalized_usage = None
+            if usage is not None:
+                normalized_usage = {
+                    "prompt_tokens": getattr(usage, "prompt_tokens", None),
+                    "completion_tokens": getattr(usage, "completion_tokens", None),
+                    "total_tokens": getattr(usage, "total_tokens", None),
+                }
+
+            return {
+                "text": text,
+                "raw": response,
+                "usage": normalized_usage,
+                "model": getattr(response, "model", model_name),
+            }
+        except Exception as exc:
+            mapped = _map_openai_exception(exc)
+            try:
+                logger.warning(
+                    "LLM error",
+                    provider=self._provider_label,
+                    operation="chat",
+                    model=model_name,
+                    base_host=self._base_host,
+                    error=type(exc).__name__,
+                )
+            except Exception:
+                pass
+            raise mapped
+
+
+class OpenAIProvider(LLMProvider):
+    def __init__(self, settings: LLMSettings):
+        self._settings = settings
+        self._base_host = _safe_host(settings.base_url)
+        if OpenAI is None:
+            self._client = None
+        else:
+            kwargs: dict[str, Any] = {}
+            if settings.base_url:
+                kwargs["base_url"] = settings.base_url
+            if settings.api_key:
+                kwargs["api_key"] = settings.api_key
+            self._client = OpenAI(**kwargs)
+
+    def embeddings(self) -> EmbeddingsClient:
+        model = self._settings.models.get("embeddings", "")
+        return OpenAIEmbeddings(self._client, model, self._base_host, provider_label="openai")
+
+    def chat(self) -> ChatClient:
+        model = self._settings.models.get("chat", "")
+        return OpenAIChat(self._client, model, self._base_host, provider_label="openai")
+
+    def tokenizer(self) -> TokenCounter:
+        return _OpenAITokenCounter(self._settings.tokenizer)

qdrant_loader_core/llm/ratelimit.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+import asyncio
+
+
+class AsyncRateLimiter:
+    """Minimal async rate limiter placeholder.
+
+    In later phases, enforce RPM/TPM and concurrency.
+    """
+
+    def __init__(self, max_concurrency: int = 5):
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+
+    async def __aenter__(self):
+        await self._semaphore.acquire()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self._semaphore.release()

qdrant_loader_core/llm/settings.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import warnings
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+
+@dataclass
+class RequestPolicy:
+    timeout_s: float = 30.0
+    max_retries: int = 3
+    backoff_s_min: float = 1.0
+    backoff_s_max: float = 30.0
+
+
+@dataclass
+class RateLimitPolicy:
+    rpm: int | None = None
+    tpm: int | None = None
+    concurrency: int = 5
+
+
+@dataclass
+class EmbeddingPolicy:
+    vector_size: int | None = None
+
+
+@dataclass
+class LLMSettings:
+    provider: str
+    base_url: str | None
+    api_key: str | None
+    headers: dict[str, str] | None
+    models: dict[str, str]
+    tokenizer: str
+    request: RequestPolicy
+    rate_limits: RateLimitPolicy
+    embeddings: EmbeddingPolicy
+    api_version: str | None = None
+    provider_options: dict[str, Any] | None = None
+
+    @staticmethod
+    def from_global_config(global_data: Mapping[str, Any]) -> LLMSettings:
+        """Construct settings from a parsed global configuration dict.
+
+        Supports two schemas:
+        - New: global.llm
+        - Legacy: global.embedding and file_conversion.markitdown
+        """
+        llm = (global_data or {}).get("llm") or {}
+        if llm:
+            return LLMSettings(
+                provider=str(llm.get("provider")),
+                base_url=llm.get("base_url"),
+                api_key=llm.get("api_key"),
+                api_version=llm.get("api_version"),
+                headers=dict(llm.get("headers") or {}),
+                models=dict(llm.get("models") or {}),
+                tokenizer=str(llm.get("tokenizer", "none")),
+                request=RequestPolicy(**(llm.get("request") or {})),
+                rate_limits=RateLimitPolicy(**(llm.get("rate_limits") or {})),
+                embeddings=EmbeddingPolicy(**(llm.get("embeddings") or {})),
+                provider_options=dict(llm.get("provider_options") or {}),
+            )
+
+        # Legacy mapping
+        embedding = (global_data or {}).get("embedding") or {}
+        file_conv = (global_data or {}).get("file_conversion") or {}
+        markit = (
+            (file_conv.get("markitdown") or {}) if isinstance(file_conv, dict) else {}
+        )
+
+        endpoint = embedding.get("endpoint")
+        # Detect Azure OpenAI in legacy endpoint to set provider accordingly
+        endpoint_l = (endpoint or "").lower() if isinstance(endpoint, str) else ""
+        host: str | None = None
+        if endpoint_l:
+            try:
+                host = urlparse(endpoint_l).hostname or None
+            except Exception:
+                host = None
+        is_azure = False
+        if host:
+            host_l = host.lower()
+            is_azure = (
+                host_l == "openai.azure.com"
+                or host_l.endswith(".openai.azure.com")
+                or host_l == "cognitiveservices.azure.com"
+                or host_l.endswith(".cognitiveservices.azure.com")
+            )
+        if is_azure:
+            provider = "azure_openai"
+        elif "openai" in endpoint_l:
+            provider = "openai"
+        else:
+            provider = "openai_compat"
+        models = {
+            "embeddings": embedding.get("model"),
+        }
+        if isinstance(markit.get("llm_model"), str):
+            models["chat"] = markit.get("llm_model")
+
+        # Emit deprecation warnings when relying on legacy fields
+        try:
+            if embedding or markit:
+                warnings.warn(
+                    (
+                        "Using legacy configuration fields is deprecated. "
+                        "Please migrate to 'global.llm' (see docs: configuration reference). "
+                        "Mapped from: global.embedding.* and/or file_conversion.markitdown.*"
+                    ),
+                    category=DeprecationWarning,
+                    stacklevel=2,
+                )
+        except Exception:
+            # Best-effort warning; never break mapping
+            pass
+
+        return LLMSettings(
+            provider=provider,
+            base_url=endpoint,
+            api_key=embedding.get("api_key"),
+            api_version=None,
+            headers=None,
+            models=models,
+            tokenizer=str(embedding.get("tokenizer", "none")),
+            request=RequestPolicy(),
+            rate_limits=RateLimitPolicy(),
+            embeddings=EmbeddingPolicy(vector_size=embedding.get("vector_size")),
+            provider_options=None,
+        )

qdrant_loader_core/llm/tokenization.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from .types import TokenCounter
+
+try:  # Optional dependency
+    import tiktoken  # type: ignore
+except Exception:  # pragma: no cover - absence is acceptable
+    tiktoken = None  # type: ignore
+
+
+class CharCountTokenCounter(TokenCounter):
+    def count(self, text: str) -> int:
+        return len(text)
+
+
+class TiktokenTokenCounter(TokenCounter):
+    """Token counter backed by tiktoken; falls back gracefully when unavailable.
+
+    If the requested encoding cannot be loaded or encode fails, falls back to
+    a simple character count to avoid runtime errors.
+    """
+
+    def __init__(self, encoding_name: str):
+        self._encoding_name = encoding_name
+        self._encoding = None
+        if tiktoken is not None:
+            try:
+                self._encoding = tiktoken.get_encoding(encoding_name)
+            except Exception:
+                self._encoding = None
+
+    def count(self, text: str) -> int:
+        if self._encoding is not None:
+            try:
+                return len(self._encoding.encode(text))
+            except Exception:
+                pass
+        return len(text)

qdrant_loader_core/llm/types.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class EmbeddingsClient(Protocol):
+    async def embed(self, inputs: list[str]) -> list[list[float]]: ...
+
+
+@runtime_checkable
+class ChatClient(Protocol):
+    async def chat(
+        self, messages: list[dict[str, Any]], **kwargs: Any
+    ) -> dict[str, Any]: ...
+
+
+@runtime_checkable
+class TokenCounter(Protocol):
+    def count(self, text: str) -> int: ...
+
+
+@runtime_checkable
+class LLMProvider(Protocol):
+    def embeddings(self) -> EmbeddingsClient: ...
+
+    def chat(self) -> ChatClient: ...
+
+    def tokenizer(self) -> TokenCounter: ...

qdrant_loader_core/logging.py

@@ -0,0 +1,279 @@
+"""Unified logging configuration for qdrant-loader ecosystem.
+
+Provides:
+- structlog setup (console/json/file) with redaction
+- stdlib logging bridge with redaction filter
+- optional suppression of noisy third-party logs
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from typing import Any
+
+import structlog
+from structlog.stdlib import LoggerFactory
+
+try:
+    # ExtraAdder is available in structlog >= 20
+    from structlog.stdlib import ExtraAdder  # type: ignore
+except Exception:  # pragma: no cover - fallback when absent
+    ExtraAdder = None  # type: ignore
+
+
+class QdrantVersionFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        try:
+            return "version check" not in record.getMessage().lower()
+        except Exception:
+            return True
+
+
+class ApplicationFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Allow all logs by default; app packages may add their own filters
+        return True
+
+
+class RedactionFilter(logging.Filter):
+    """Redacts obvious secrets from stdlib log records."""
+
+    # Heuristics for tokens/keys in plain strings
+    TOKEN_PATTERNS = [
+        re.compile(r"sk-[A-Za-z0-9_\-]{6,}"),
+        re.compile(r"tok-[A-Za-z0-9_\-]{6,}"),
+        re.compile(r"(?i)(api_key|authorization|token|access_token|secret|password)\s*[:=]\s*([^\s]+)"),
+        re.compile(r"Bearer\s+[A-Za-z0-9_\-\.]+"),
+    ]
+
+    # Keys commonly used for secrets in structlog event dictionaries
+    SENSITIVE_KEYS = {
+        "api_key",
+        "llm_api_key",
+        "authorization",
+        "Authorization",
+        "token",
+        "access_token",
+        "secret",
+        "password",
+    }
+
+    def _redact_text(self, text: str) -> str:
+        def mask(m: re.Match[str]) -> str:
+            s = m.group(0)
+            if len(s) <= 8:
+                return "***REDACTED***"
+            return s[:2] + "***REDACTED***" + s[-2:]
+
+        redacted = text
+        for pat in self.TOKEN_PATTERNS:
+            redacted = pat.sub(mask, redacted)
+        return redacted
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        try:
+            redaction_detected = False
+
+            # Args may contain secrets; best-effort mask strings and detect changes
+            if isinstance(record.args, tuple):
+                new_args = []
+                for a in record.args:
+                    if isinstance(a, str):
+                        red_a = self._redact_text(a)
+                        if red_a != a:
+                            redaction_detected = True
+                        new_args.append(red_a)
+                    else:
+                        new_args.append(a)
+                record.args = tuple(new_args)
+
+            # Redact raw message only when it contains no formatting placeholders
+            # to avoid interfering with %-style or {}-style formatting
+            if isinstance(record.msg, str):
+                try:
+                    has_placeholders = ("%" in record.msg) or ("{" in record.msg)
+                except Exception:
+                    has_placeholders = True
+                if not has_placeholders:
+                    red_msg = self._redact_text(record.msg)
+                    if red_msg != record.msg:
+                        record.msg = red_msg
+                        redaction_detected = True
+
+            # If structlog extras contain sensitive keys, mark as redacted
+            try:
+                if any(
+                    (k in self.SENSITIVE_KEYS and bool(record.__dict__.get(k)))
+                    for k in record.__dict__.keys()
+                ):
+                    redaction_detected = True
+            except Exception:
+                pass
+
+            # Ensure a visible redaction marker appears in the captured message
+            if redaction_detected:
+                try:
+                    if isinstance(record.msg, str) and "***REDACTED***" not in record.msg:
+                        # Append a marker in a way that won't interfere with %-formatting
+                        record.msg = f"{record.msg} ***REDACTED***"
+                except Exception:
+                    pass
+        except Exception:
+            pass
+        return True
+
+
+class CleanFormatter(logging.Formatter):
+    """Formatter that removes ANSI color codes for clean file output."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = super().format(record)
+        try:
+            ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+            return ansi_escape.sub("", message)
+        except Exception:
+            return message
+
+
+def _redact_processor(logger: Any, method_name: str, event_dict: dict[str, Any]) -> dict[str, Any]:
+    """Structlog processor to redact sensitive fields in event_dict."""
+    sensitive_keys = {
+        "api_key",
+        "llm_api_key",
+        "authorization",
+        "Authorization",
+        "token",
+        "access_token",
+        "secret",
+        "password",
+    }
+
+    def mask(value: str) -> str:
+        try:
+            if not isinstance(value, str) or not value:
+                return "***REDACTED***"
+            if len(value) <= 8:
+                return "***REDACTED***"
+            return value[:2] + "***REDACTED***" + value[-2:]
+        except Exception:
+            return "***REDACTED***"
+
+    def deep_redact(obj: Any) -> Any:
+        try:
+            if isinstance(obj, dict):
+                return {k: (mask(v) if k in sensitive_keys and isinstance(v, str) else deep_redact(v)) for k, v in obj.items()}
+            if isinstance(obj, list):
+                return [deep_redact(i) for i in obj]
+            return obj
+        except Exception:
+            return obj
+
+    return deep_redact(event_dict)
+
+
+class LoggingConfig:
+    """Core logging setup with structlog + stdlib redaction and filters."""
+
+    _initialized = False
+
+    @classmethod
+    def setup(
+        cls,
+        *,
+        level: str = "INFO",
+        format: str = "console",  # "console" | "json"
+        file: str | None = None,
+        clean_output: bool = True,
+        suppress_qdrant_warnings: bool = True,
+        disable_console: bool | None = None,
+    ) -> None:
+        # Env override for console toggling (e.g., MCP server)
+        if disable_console is None:
+            disable_console = os.getenv("MCP_DISABLE_CONSOLE_LOGGING", "").lower() == "true"
+
+        try:
+            numeric_level = getattr(logging, level.upper())
+        except AttributeError:
+            raise ValueError(f"Invalid log level: {level}") from None
+
+        # Reset structlog defaults but preserve existing stdlib handlers (e.g., pytest caplog)
+        structlog.reset_defaults()
+
+        handlers: list[logging.Handler] = []
+
+        # Choose timestamp format and final renderer for structlog messages
+        if clean_output and format == "console":
+            ts_fmt = "%H:%M:%S"
+            final_renderer = structlog.dev.ConsoleRenderer(colors=True)
+        else:
+            ts_fmt = "iso"
+            final_renderer = (
+                structlog.processors.JSONRenderer()
+                if format == "json"
+                else structlog.dev.ConsoleRenderer(colors=True)
+            )
+
+        if not disable_console:
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(logging.Formatter("%(message)s"))
+            console_handler.addFilter(ApplicationFilter())
+            console_handler.addFilter(RedactionFilter())
+            handlers.append(console_handler)
+
+        if file:
+            file_handler = logging.FileHandler(file)
+            # Use CleanFormatter to strip ANSI sequences from structlog console renderer output
+            file_handler.setFormatter(CleanFormatter("%(message)s"))
+            file_handler.addFilter(ApplicationFilter())
+            file_handler.addFilter(RedactionFilter())
+            handlers.append(file_handler)
+
+        # Attach our handlers without removing existing ones (so pytest caplog keeps working)
+        root_logger = logging.getLogger()
+        root_logger.setLevel(numeric_level)
+        for h in handlers:
+            root_logger.addHandler(h)
+
+        # Add global filters so captured logs (e.g., pytest caplog) are also redacted
+        # Avoid duplicate filters if setup() is called multiple times
+        has_redaction = any(isinstance(f, RedactionFilter) for f in root_logger.filters)
+        if not has_redaction:
+            root_logger.addFilter(RedactionFilter())
+        has_app_filter = any(isinstance(f, ApplicationFilter) for f in root_logger.filters)
+        if not has_app_filter:
+            root_logger.addFilter(ApplicationFilter())
+
+        # Optional suppressions
+        if suppress_qdrant_warnings:
+            logging.getLogger("qdrant_client").addFilter(QdrantVersionFilter())
+
+        # Quiet noisy libs a bit
+        for name in ("httpx", "httpcore", "urllib3", "gensim"):
+            logging.getLogger(name).setLevel(logging.WARNING)
+
+        # structlog processors – render to a final string directly
+        structlog.configure(
+            processors=[
+                structlog.stdlib.filter_by_level,
+                structlog.stdlib.add_logger_name,
+                structlog.stdlib.add_log_level,
+                structlog.processors.TimeStamper(fmt=ts_fmt),
+                _redact_processor,
+                final_renderer,
+            ],
+            wrapper_class=structlog.make_filtering_bound_logger(numeric_level),
+            logger_factory=LoggerFactory(),
+            cache_logger_on_first_use=False,
+        )
+
+        cls._initialized = True
+
+    @classmethod
+    def get_logger(cls, name: str | None = None) -> structlog.BoundLogger:
+        if not cls._initialized:
+            cls.setup()
+        return structlog.get_logger(name)
+
+

qdrant_loader_core-0.7.1.dist-info/METADATA

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: qdrant-loader-core
+Version: 0.7.1
+Summary: Shared core for provider-agnostic LLM support and configuration mapping for qdrant-loader ecosystem
+Author-email: Martin Papy <martin.papy@cbtw.tech>
+License-Expression: GPL-3.0
+Project-URL: Homepage, https://qdrant-loader.net
+Project-URL: Documentation, https://qdrant-loader.net/docs/packages/core/README.html
+Project-URL: Repository, https://github.com/martin-papy/qdrant-loader
+Project-URL: Issues, https://github.com/martin-papy/qdrant-loader/issues
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Database :: Database Engines/Servers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Environment :: Console
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: openai
+Requires-Dist: openai>=1.3.0; extra == "openai"
+Requires-Dist: tiktoken>=0.5.0; extra == "openai"
+Provides-Extra: ollama
+Requires-Dist: httpx>=0.24.0; extra == "ollama"

qdrant_loader_core-0.7.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+qdrant_loader_core/__init__.py,sha256=9JasHhaZvRdZNL1Mge2I3MtcgZ7QQutCdPG8jJeIGGo,433
+qdrant_loader_core/logging.py,sha256=YCc7ykPLmDX6oOeote1cL3mPd_IkwPecNvYlDf8uqTg,9839
+qdrant_loader_core/llm/__init__.py,sha256=NgYA69RsGaEe9eAIa-4XHZw5ijo0lPF9iiFt7yM0rP4,445
+qdrant_loader_core/llm/errors.py,sha256=OF9jUayFSdXz7Tq0mtD68BuUaBRfKVxbcJY4R2_9eSM,248
+qdrant_loader_core/llm/factory.py,sha256=pfYj0EQ29YGD1Hy8VktH12oKzTtBSwjQOovChrzo2Fo,2825
+qdrant_loader_core/llm/ratelimit.py,sha256=aJOYTc2Y57_yBc-Ea9D_Hdw5LkqwxP73MQPEI1RbYk4,477
+qdrant_loader_core/llm/settings.py,sha256=1OVyOIQ05Mp9370La0ABaMjABIp5kszUzopydWcEwqk,4474
+qdrant_loader_core/llm/tokenization.py,sha256=cptyJevdIIQwMsT1Xqe9kL-PimAr_G2BKIQIv6UMdw4,1141
+qdrant_loader_core/llm/types.py,sha256=z2Zlq6zUt9fTOapBk5g4BDR59uxBC6Ia4YZq3QUjGo0,651
+qdrant_loader_core/llm/providers/__init__.py,sha256=L7JXyBbTI-hkZpKKUsnjFL16YUQkmGbTbqkVPlRW08Y,83
+qdrant_loader_core/llm/providers/azure_openai.py,sha256=MADFASMuZus7STblvk8zp-nXVFY_NFefeK3bah3Ii7E,2598
+qdrant_loader_core/llm/providers/ollama.py,sha256=zU7ce1mV-oyieNtCJabrATLZgiIe97Ay763PiAd-BBA,12764
+qdrant_loader_core/llm/providers/openai.py,sha256=8IG9-2pc3cYNY3Kb9S3ZYQn9qajJfC0HUG63B1C4nnQ,9543
+qdrant_loader_core-0.7.1.dist-info/METADATA,sha256=6GTrW2sghMJRuhTK0lnQQfve7QE9puiVpmaOlMySv74,1628
+qdrant_loader_core-0.7.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+qdrant_loader_core-0.7.1.dist-info/top_level.txt,sha256=IuQX743SRjuKkZNiQd3xqGLuAQkFNKSCFnR5G2hFpck,19
+qdrant_loader_core-0.7.1.dist-info/RECORD,,

qdrant_loader_core-0.7.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

qdrant_loader_core-0.7.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+qdrant_loader_core