unique-search-proxy-core 2026.26.0.dev0__tar.gz

unique_search_proxy_core-2026.26.0.dev0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.3
+Name: unique-search-proxy-core
+Version: 2026.26.0.dev0
+Summary: Shared Pydantic types for the Unique Search Proxy API
+Author: ThePhilAz
+Author-email: ThePhilAz <rami.azouz@philico.com>
+Requires-Dist: pydantic>=2.12.5,<3.0.0
+Requires-Dist: pyhumps>=3.8.0,<4
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# unique-search-proxy-core
+
+Server-free Pydantic models and helpers shared by the Unique Search Proxy HTTP API and SDK.
+
+Install via PyPI as `unique-search-proxy-core`. Consumers of the HTTP client should prefer `unique-search-proxy-sdk`.

unique_search_proxy_core-2026.26.0.dev0/README.md

@@ -0,0 +1,5 @@
+# unique-search-proxy-core
+
+Server-free Pydantic models and helpers shared by the Unique Search Proxy HTTP API and SDK.
+
+Install via PyPI as `unique-search-proxy-core`. Consumers of the HTTP client should prefer `unique-search-proxy-sdk`.

unique_search_proxy_core-2026.26.0.dev0/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "unique-search-proxy-core"
+version = "2026.26.0.dev0"
+description = "Shared Pydantic types for the Unique Search Proxy API"
+authors = [{ name = "ThePhilAz", email = "rami.azouz@philico.com" }]
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "pydantic>=2.12.5,<3.0.0",
+    "pyhumps>=3.8.0,<4",
+]
+
+[dependency-groups]
+dev = [
+    "basedpyright>=1.39.1",
+    "pytest>=9.0.3",
+    "pytest-asyncio>=1.3.0",
+    "ruff>=0.15.10",
+]
+
+[build-system]
+requires = ["uv_build>=0.7.19,<0.8"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-root = "."
+
+[tool.uv]
+exclude-newer = "2 weeks"
+
+[tool.ruff]
+target-version = "py312"
+
+[tool.ruff.lint]
+extend-select = ["I"]
+
+[tool.basedpyright]
+typeCheckingMode = "standard"
+include = ["unique_search_proxy_core"]
+
+[tool.deptry]
+known_first_party = ["unique_search_proxy_core"]
+
+[tool.deptry.per_rule_ignores]
+# pydantic_core ships with (and is version-pinned by) pydantic; importing
+# CoreSchema from it is the canonical way to type __get_pydantic_core_schema__.
+DEP003 = ["pydantic_core"]
+
+[tool.poe.tasks]
+lint = "ruff check ."
+lint-fix = "ruff check . --fix"
+format = "ruff format ."
+test = "pytest"
+typecheck = "basedpyright"
+depcheck = "deptry ."
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers --import-mode=importlib"
+asyncio_mode = "auto"

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/__init__.py

@@ -0,0 +1,41 @@
+"""Shared types for Unique Search Proxy (no FastAPI / server dependencies)."""
+
+from unique_search_proxy_core.errors import (
+    BadRequestProxyError,
+    EmptySearchResultsError,
+    EngineNotConfiguredError,
+    ForbiddenTargetError,
+    ProxyError,
+    RateLimitedError,
+    UpstreamError,
+    UpstreamTimeoutError,
+    ValidationProxyError,
+)
+from unique_search_proxy_core.schema import (
+    CrawlResponse,
+    ErrorDetail,
+    ErrorResponse,
+    ProvidersListResponse,
+    ProxyErrorCode,
+    SearchResponse,
+    WebSearchResult,
+)
+
+__all__ = [
+    "BadRequestProxyError",
+    "CrawlResponse",
+    "EmptySearchResultsError",
+    "EngineNotConfiguredError",
+    "ErrorDetail",
+    "ErrorResponse",
+    "ForbiddenTargetError",
+    "ProvidersListResponse",
+    "ProxyError",
+    "ProxyErrorCode",
+    "RateLimitedError",
+    "SearchResponse",
+    "UpstreamError",
+    "UpstreamTimeoutError",
+    "ValidationProxyError",
+    "WebSearchResult",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/__init__.py

@@ -0,0 +1,37 @@
+from unique_search_proxy_core.crawlers.base import (
+    BaseCrawler,
+    BaseCrawlerConfig,
+    CrawlerRequestT,
+    CrawlerType,
+)
+from unique_search_proxy_core.crawlers.basic.schema import (
+    BasicCrawlerConfig,
+    BasicCrawlerRequest,
+)
+from unique_search_proxy_core.crawlers.config_types import (
+    CrawlerConfigTypes,
+    CrawlRequest,
+    CrawlRequestTypes,
+    build_crawl_request_union,
+    crawler_config_from_request,
+    parse_crawl_request,
+    parse_crawler_config,
+)
+from unique_search_proxy_core.crawlers.params import merge_crawler_config_and_invocation
+
+__all__ = [
+    "BaseCrawler",
+    "BaseCrawlerConfig",
+    "BasicCrawlerConfig",
+    "BasicCrawlerRequest",
+    "CrawlerConfigTypes",
+    "CrawlerRequestT",
+    "CrawlRequest",
+    "CrawlRequestTypes",
+    "CrawlerType",
+    "build_crawl_request_union",
+    "crawler_config_from_request",
+    "merge_crawler_config_and_invocation",
+    "parse_crawl_request",
+    "parse_crawler_config",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/base.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from typing import TYPE_CHECKING, Generic, TypeVar
+
+from pydantic import BaseModel, Field
+
+from unique_search_proxy_core.schema import CrawlUrlResult, camelized_model_config
+
+if TYPE_CHECKING:
+    from httpx import AsyncClient
+
+CrawlerTypeT = TypeVar("CrawlerTypeT", bound="CrawlerType")
+CrawlerRequestT = TypeVar("CrawlerRequestT", bound=BaseModel)
+
+
+class CrawlerType(StrEnum):
+    """Registered crawler ids (JSON discriminator values)."""
+
+    BASIC = "BasicProxyCrawler"
+
+
+class BaseCrawlerConfig(BaseModel, Generic[CrawlerTypeT]):
+    """Shared crawler config; each crawler narrows ``crawler_type`` with a Literal."""
+
+    model_config = camelized_model_config
+
+    crawler_type: CrawlerTypeT
+    timeout: int = Field(
+        default=30,
+        ge=1,
+        le=600,
+        description="Request timeout in seconds",
+    )
+
+
+class BaseCrawler(ABC, Generic[CrawlerRequestT]):
+    """Crawler contract: per-URL outcomes with optional url-safety enforcement."""
+
+    crawler_id: str
+
+    def __init__(
+        self,
+        *,
+        http_client: AsyncClient | None = None,
+    ) -> None:
+        self._http_client = http_client
+
+    @abstractmethod
+    async def crawl(self, request: CrawlerRequestT) -> list[CrawlUrlResult]:
+        """Crawl URLs from a flat request model (``BasicCrawlerRequest``, …)."""

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/basic/__init__.py

@@ -0,0 +1,13 @@
+from unique_search_proxy_core.crawlers.basic.content_types import ContentTypeToggles
+from unique_search_proxy_core.crawlers.basic.schema import (
+    BasicCrawlerCall,
+    BasicCrawlerConfig,
+    BasicCrawlerRequest,
+)
+
+__all__ = [
+    "BasicCrawlerCall",
+    "BasicCrawlerConfig",
+    "BasicCrawlerRequest",
+    "ContentTypeToggles",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/basic/content_types.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from unique_search_proxy_core.crawlers.basic.processing.policy import (
+    ContentTypeHandlerPolicy,
+)
+from unique_search_proxy_core.schema import get_model_config
+
+# Supported media types for the basic crawler (must match registered processors).
+CONTENT_TYPE_TOGGLE_TO_MIME: dict[str, str] = {
+    "html": "text/html",
+    "xhtml": "application/xhtml+xml",
+    "plain_text": "text/plain",
+    "markdown": "text/markdown",
+    "pdf": "application/pdf",
+}
+
+
+class ContentTypeToggles(BaseModel):
+    """Per-type activation flags for basic-crawler content processing."""
+
+    model_config = get_model_config(title="Content Types")
+
+    html: bool = Field(default=True, title="HTML", description="text/html")
+    xhtml: bool = Field(
+        default=True,
+        title="XHTML",
+        description="application/xhtml+xml",
+    )
+    plain_text: bool = Field(default=True, title="Plain text", description="text/plain")
+    markdown: bool = Field(default=True, title="Markdown", description="text/markdown")
+    pdf: bool = Field(default=False, title="PDF", description="application/pdf")
+
+    def to_handlers(self) -> dict[str, ContentTypeHandlerPolicy]:
+        """Map enabled toggles to allow-policies for the processing registry."""
+        return {
+            mime_type: ContentTypeHandlerPolicy.ALLOW
+            for field_name, mime_type in CONTENT_TYPE_TOGGLE_TO_MIME.items()
+            if getattr(self, field_name)
+        }
+
+
+__all__ = [
+    "CONTENT_TYPE_TOGGLE_TO_MIME",
+    "ContentTypeToggles",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/basic/processing/policy.py

@@ -0,0 +1,8 @@
+from enum import StrEnum
+
+
+class ContentTypeHandlerPolicy(StrEnum):
+    """Whether the basic crawler may run the built-in processor for a media type."""
+
+    ALLOW = "allow"
+    FORBID = "forbid"

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/basic/schema.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from unique_search_proxy_core.crawlers.base import BaseCrawlerConfig, CrawlerType
+from unique_search_proxy_core.crawlers.basic.content_types import ContentTypeToggles
+from unique_search_proxy_core.projection import build_crawl_request_model
+from unique_search_proxy_core.schema import get_model_config
+
+
+class BasicCrawlerCall(BaseModel):
+    """LLM-facing call surface for the basic crawler (urls supplied per invocation)."""
+
+    urls: list[str] = Field(
+        ...,
+        min_length=1,
+        description="URLs to fetch and convert to markdown",
+    )
+
+
+class BasicCrawlerConfig(BaseCrawlerConfig[CrawlerType.BASIC]):
+    """Deployment config for the HTTP basic crawler."""
+
+    model_config = get_model_config(title="Basic Proxy Crawler ")
+
+    crawler_type: Literal[CrawlerType.BASIC] = CrawlerType.BASIC
+
+    content_types: ContentTypeToggles = Field(
+        default_factory=ContentTypeToggles,
+        title="Content types",
+        description=(
+            "Enable built-in processing per media type. "
+            "Unchecked types return raw body only."
+        ),
+    )
+    max_concurrent_requests: int = Field(
+        default=10,
+        ge=1,
+        le=50,
+        title="Maximum concurrent HTTP fetches",
+        description="Maximum concurrent HTTP fetches",
+    )
+
+
+def basic_crawler_request_model() -> type[BaseModel]:
+    """Derived ``POST /v1/crawl`` model (cached via ``build_crawl_request_model``)."""
+    return build_crawl_request_model(BasicCrawlerConfig)
+
+
+BasicCrawlerRequest = basic_crawler_request_model()
+
+
+__all__ = [
+    "BasicCrawlerCall",
+    "BasicCrawlerConfig",
+    "BasicCrawlerRequest",
+    "basic_crawler_request_model",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/call_schema.py

@@ -0,0 +1,75 @@
+"""LLM-facing call JSON Schema derived from crawler deployment config (no HTTP)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from pydantic import BaseModel
+
+from unique_search_proxy_core.crawlers.basic.schema import (
+    BasicCrawlerCall,
+    BasicCrawlerConfig,
+)
+from unique_search_proxy_core.crawlers.config_types import (
+    CRAWLER_NAME_TO_CONFIG,
+    CrawlerConfigTypes,
+    parse_crawler_config,
+)
+from unique_search_proxy_core.projection import project_call_schema
+from unique_search_proxy_core.providers.schema import provider_default_config
+
+
+@dataclass(frozen=True)
+class CrawlCallSchemaDescriptor:
+    """Metadata and JSON Schema for the crawler call model on ``POST /v1/crawl``."""
+
+    crawler: str
+    call_schema: dict[str, Any]
+
+
+def _llm_call_schema_for_config(config: CrawlerConfigTypes) -> type[BaseModel]:
+    if isinstance(config, BasicCrawlerConfig):
+        return project_call_schema(BasicCrawlerCall, ["urls"])
+    raise ValueError(f"No LLM call schema for crawler config {type(config).__name__}")
+
+
+def resolve_crawl_call_schema_from_config(
+    crawler_id: str,
+    config: CrawlerConfigTypes,
+) -> CrawlCallSchemaDescriptor:
+    """Project the LLM-visible call surface from a parsed deployment config."""
+    config_cls = CRAWLER_NAME_TO_CONFIG[crawler_id.lower()]
+    if type(config) is not config_cls:
+        raise ValueError(
+            f"Config type {type(config).__name__} does not match crawler {crawler_id!r}",
+        )
+    projected = _llm_call_schema_for_config(config)
+    return CrawlCallSchemaDescriptor(
+        crawler=crawler_id.lower(),
+        call_schema=projected.model_json_schema(),
+    )
+
+
+def resolve_crawl_call_schema(
+    crawler_id: str,
+    *,
+    config: CrawlerConfigTypes | dict[str, Any] | None = None,
+) -> CrawlCallSchemaDescriptor:
+    """Resolve call schema from deployment config or crawler defaults."""
+    if config is not None:
+        parsed = (
+            config if isinstance(config, BaseModel) else parse_crawler_config(config)
+        )
+        return resolve_crawl_call_schema_from_config(crawler_id, parsed)
+
+    defaults = provider_default_config("crawler", crawler_id)
+    parsed = parse_crawler_config(defaults)
+    return resolve_crawl_call_schema_from_config(crawler_id, parsed)
+
+
+__all__ = [
+    "CrawlCallSchemaDescriptor",
+    "resolve_crawl_call_schema",
+    "resolve_crawl_call_schema_from_config",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/config_types.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Annotated, Any, TypeAlias, Union
+
+from pydantic import BaseModel, Field, TypeAdapter
+
+from unique_search_proxy_core.crawlers.base import BaseCrawlerConfig, CrawlerType
+from unique_search_proxy_core.crawlers.basic.schema import BasicCrawlerConfig
+from unique_search_proxy_core.projection import URLS_FIELD, build_crawl_request_model
+
+CrawlerConfigTypes: TypeAlias = BasicCrawlerConfig
+
+CRAWLER_NAME_TO_CONFIG: dict[str, type[BaseCrawlerConfig]] = {
+    CrawlerType.BASIC.value: BasicCrawlerConfig,
+}
+
+_crawler_config_adapter: TypeAdapter[CrawlerConfigTypes] = TypeAdapter(
+    CrawlerConfigTypes,
+)
+
+_CRAWL_REQUEST_EXCLUDED_FIELDS = {URLS_FIELD}
+
+
+def parse_crawler_config(data: object) -> CrawlerConfigTypes:
+    return _crawler_config_adapter.validate_python(data)
+
+
+def build_crawl_request_union() -> Any:
+    """Discriminated union of flat ``POST /v1/crawl`` bodies (``crawler_type`` discriminator)."""
+    members = tuple(CRAWLER_NAME_TO_CONFIG.values())
+    request_models = tuple(
+        build_crawl_request_model(config_cls) for config_cls in members
+    )
+    if len(request_models) == 1:
+        return request_models[0]
+    return Annotated[
+        Union[request_models],  # type: ignore[valid-type]
+        Field(discriminator="crawler_type"),
+    ]
+
+
+CrawlRequestTypes = build_crawl_request_union()
+CrawlRequest = CrawlRequestTypes
+
+_crawl_request_adapter: TypeAdapter[BaseModel] = TypeAdapter(CrawlRequestTypes)  # type: ignore[arg-type]
+
+
+def parse_crawl_request(data: object) -> BaseModel:
+    return _crawl_request_adapter.validate_python(data)
+
+
+def crawler_config_from_request(request: BaseModel) -> CrawlerConfigTypes:
+    """Rebuild deployment config from a flat crawl request (excludes ``urls`` only)."""
+    crawler_id = getattr(request, "crawler_type", None)
+    if not isinstance(crawler_id, str):
+        raise ValueError("Flat crawl request is missing crawler_type discriminator")
+
+    config_cls = CRAWLER_NAME_TO_CONFIG.get(crawler_id.lower())
+    if config_cls is None:
+        raise ValueError(f"No crawler config registered for {crawler_id!r}")
+
+    payload = request.model_dump(
+        exclude=_CRAWL_REQUEST_EXCLUDED_FIELDS,
+        mode="python",
+    )
+    return parse_crawler_config(payload)

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/crawlers/params.py

@@ -0,0 +1,45 @@
+"""Crawler request merge helpers (no HTTP)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from unique_search_proxy_core.projection import build_crawl_request_model
+
+CRAWLER_TYPE_FIELD = "crawler_type"
+URLS_FIELD = "urls"
+TIMEOUT_FIELD = "timeout"
+
+
+def crawler_config_defaults(config: BaseModel) -> dict[str, Any]:
+    """Deployment defaults merged into each flat crawl request."""
+    defaults: dict[str, Any] = {}
+    for field_name in type(config).model_fields:
+        if field_name == CRAWLER_TYPE_FIELD:
+            continue
+        defaults[field_name] = getattr(config, field_name)
+    return defaults
+
+
+def merge_crawler_config_and_invocation(
+    config: BaseModel,
+    invocation: dict[str, Any],
+) -> BaseModel:
+    """Merge deployment config defaults with caller/LLM args into a flat crawl request."""
+    request_model = build_crawl_request_model(type(config))
+    defaults = crawler_config_defaults(config)
+    merged: dict[str, Any] = {**defaults, **invocation}
+    if CRAWLER_TYPE_FIELD in request_model.model_fields:
+        merged[CRAWLER_TYPE_FIELD] = getattr(config, CRAWLER_TYPE_FIELD)
+    return request_model.model_validate(merged)
+
+
+__all__ = [
+    "CRAWLER_TYPE_FIELD",
+    "TIMEOUT_FIELD",
+    "URLS_FIELD",
+    "crawler_config_defaults",
+    "merge_crawler_config_and_invocation",
+]

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/errors.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from typing import Any
+
+from unique_search_proxy_core.schema import (
+    ErrorDetail,
+    ProxyErrorCode,
+)
+
+
+class ProxyError(Exception):
+    """Base exception for proxy failures with a stable error code."""
+
+    code: ProxyErrorCode = ProxyErrorCode.BAD_REQUEST
+    status_code: int = 400
+    retryable: bool = False
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        engine: str | None = None,
+        crawler: str | None = None,
+        retryable: bool | None = None,
+        details: list[dict[str, Any]] | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.message = message
+        self.engine = engine
+        self.crawler = crawler
+        if retryable is not None:
+            self.retryable = retryable
+        self.details = details
+
+    def to_detail(self) -> ErrorDetail:
+        return ErrorDetail(
+            code=self.code.value,
+            message=self.message,
+            engine=self.engine,
+            crawler=self.crawler,
+            retryable=self.retryable,
+            details=self.details,
+        )
+
+
+class BadRequestProxyError(ProxyError):
+    code = ProxyErrorCode.BAD_REQUEST
+    status_code = 400
+
+
+class ValidationProxyError(ProxyError):
+    code = ProxyErrorCode.VALIDATION_ERROR
+    status_code = 422
+
+
+class ForbiddenTargetError(ProxyError):
+    code = ProxyErrorCode.FORBIDDEN_TARGET
+    status_code = 403
+
+
+class RateLimitedError(ProxyError):
+    code = ProxyErrorCode.RATE_LIMITED
+    status_code = 429
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        retry_after_seconds: int | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, retryable=True, **kwargs)
+        self.retry_after_seconds = retry_after_seconds
+
+
+class UpstreamError(ProxyError):
+    code = ProxyErrorCode.UPSTREAM_ERROR
+    status_code = 502
+    retryable = True
+
+
+class EngineNotConfiguredError(ProxyError):
+    code = ProxyErrorCode.ENGINE_NOT_CONFIGURED
+    status_code = 503
+
+    def __init__(self, provider: str, *, kind: str = "engine") -> None:
+        super().__init__(
+            f"{kind.capitalize()} '{provider}' is not registered or not configured",
+            engine=provider if kind == "engine" else None,
+            crawler=provider if kind == "crawler" else None,
+        )
+        self.provider = provider
+        self.kind = kind
+
+
+class UpstreamTimeoutError(ProxyError):
+    code = ProxyErrorCode.UPSTREAM_TIMEOUT
+    status_code = 504
+    retryable = True
+
+
+class EmptySearchResultsError(ProxyError):
+    code = ProxyErrorCode.EMPTY_SEARCH_RESULTS
+    status_code = 404
+    retryable = False

unique_search_proxy_core-2026.26.0.dev0/unique_search_proxy_core/param_policy/__init__.py

@@ -0,0 +1,21 @@
+"""Deployment field policies: ``ExposableParam`` for optional provider knobs."""
+
+from unique_search_proxy_core.param_policy.exposable_param import (
+    ExposableParam,
+    exposable_param_inner_type,
+    flatten_union_args,
+    is_exposable_param_field,
+    is_exposable_param_type,
+    unwrap_exposable_param_value,
+)
+from unique_search_proxy_core.param_policy.policy import QUERY_FIELD
+
+__all__ = [
+    "QUERY_FIELD",
+    "ExposableParam",
+    "exposable_param_inner_type",
+    "flatten_union_args",
+    "is_exposable_param_field",
+    "is_exposable_param_type",
+    "unwrap_exposable_param_value",
+]