antfly-sdk 0.0.2.dev2__py3-none-any.whl

antfly/client_generated/models/embeddings_index_config.py

@@ -0,0 +1,393 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+from ..models.distance_metric import DistanceMetric
+from ..types import UNSET, Unset
+
+if TYPE_CHECKING:
+    from ..models.chunker_config import ChunkerConfig
+    from ..models.embedder_config import EmbedderConfig
+    from ..models.generator_config import GeneratorConfig
+
+
+T = TypeVar("T", bound="EmbeddingsIndexConfig")
+
+
+@_attrs_define
+class EmbeddingsIndexConfig:
+    r"""Unified configuration for embeddings indexes. When sparse is true, creates a sparse vector index (SPLADE inverted
+    index). When sparse is false (default), creates a dense vector index (HNSW). For dense indexes, dimension can be
+    omitted if an embedder is configured — it will be auto-detected.
+
+        Attributes:
+            external (bool | Unset): When true, embeddings are supplied externally via _embeddings and the index does not
+                derive prompts from a field or template. Default: False.
+            sparse (bool | Unset): When true, creates a sparse (SPLADE) inverted index. When false (default), creates a
+                dense (HNSW) vector index. Default: False.
+            dimension (int | Unset): Vector dimension for dense indexes. Required for external dense indexes. Can be omitted
+                for managed dense indexes when an embedder is configured (auto-detected via probe). Ignored for sparse indexes.
+            field (str | Unset): Field to extract embeddings from (managed indexes only; not allowed when external=true)
+            template (str | Unset): Handlebars template for generating prompts (managed indexes only; not allowed when
+                external=true). See https://handlebarsjs.com/guide/ for more information. Example: Hello, {{#if (eq Name
+                "John")}}Johnathan{{else}}{{Name}}{{/if}}! You are {{Age}} years old..
+            distance_metric (DistanceMetric | Unset): Distance metric for the vector index (dense only). Use "cosine" for
+                models trained with cosine similarity (e.g. CLIP, OpenAI). Use "inner_product" for models trained with dot
+                product similarity. Use "l2_squared" (default) for models trained with Euclidean distance.
+            mem_only (bool | Unset): Whether to use in-memory only storage (dense only)
+            embedder (EmbedderConfig | Unset): A unified configuration for an embedding provider.
+
+                Embedders can be configured with templates to customize how documents are
+                converted to text before embedding. Templates use Handlebars syntax and
+                support various built-in helpers.
+
+                **Template System:**
+                - **Syntax**: Handlebars templating (https://handlebarsjs.com/guide/)
+                - **Caching**: Templates are automatically cached with configurable TTL (default: 5 minutes)
+                - **Context**: Templates receive the full document as context
+
+                **Built-in Helpers:**
+
+                1. **scrubHtml** - Remove script/style tags and extract clean text from HTML
+                   ```handlebars
+                   {{scrubHtml html_content}}
+                   ```
+                   - Removes `<script>` and `<style>` tags
+                   - Adds newlines after block elements (p, div, h1-h6, li, etc.)
+                   - Returns plain text with preserved readability
+
+                2. **eq** - Equality comparison for conditionals
+                   ```handlebars
+                   {{#if (eq status "active")}}Active user{{/if}}
+                   {{#if (eq @key "special")}}Special field{{/if}}
+                   ```
+
+                3. **media** - GenKit dotprompt media directive for multimodal content
+                   ```handlebars
+                   {{media url=imageDataURI}}
+                   {{media url=this.image_url}}
+                   {{media url="https://example.com/image.jpg"}}
+                   {{media url="s3://endpoint/bucket/image.png"}}
+                   {{media url="file:///path/to/image.jpg"}}
+                   ```
+
+                   **Supported URL Schemes:**
+                   - `data:` - Base64 encoded data URIs (e.g., `data:image/jpeg;base64,...`)
+                   - `http://` / `https://` - Web URLs with automatic content type detection
+                   - `file://` - Local filesystem paths
+                   - `s3://` - S3-compatible storage (format: `s3://endpoint/bucket/key`)
+
+                   **Automatic Content Processing:**
+                   - **Images**: Downloaded, resized (if needed), converted to data URIs
+                   - **PDFs**: Text extracted or first page rendered as image
+                   - **HTML**: Readable text extracted using Mozilla Readability
+
+                   **Security Controls:**
+                   Downloads are protected by content security settings (see Configuration Reference):
+                   - Allowed host whitelist
+                   - Private IP blocking (prevents SSRF attacks)
+                   - Download size limits (default: 100MB)
+                   - Download timeouts (default: 30s)
+                   - Image dimension limits (default: 2048px, auto-resized)
+
+                   See: https://antfly.io/docs/configuration#security--cors
+
+                4. **encodeToon** - Encode data in TOON format (Token-Oriented Object Notation)
+                   ```handlebars
+                   {{encodeToon this.fields}}
+                   {{encodeToon this.fields lengthMarker=false indent=4}}
+                   {{encodeToon this.fields delimiter="\t"}}
+                   ```
+
+                   **What is TOON?**
+                   TOON is a compact, human-readable format designed for passing structured data to LLMs.
+                   It provides **30-60% token reduction** compared to JSON while maintaining high LLM
+                   comprehension accuracy.
+
+                   **Key Features:**
+                   - Compact syntax using `:` for key-value pairs
+                   - Array length markers: `tags[#3]: ai,search,ml`
+                   - Tabular format for uniform data structures
+                   - Optimized for LLM parsing and understanding
+                   - Maintains human readability
+
+                   **Benefits:**
+                   - **Lower API costs** - Reduced token usage means lower LLM API costs
+                   - **Faster responses** - Less tokens to process
+                   - **More context** - Fit more documents within token limits
+
+                   **Options:**
+                   - `lengthMarker` (bool): Add # prefix to array counts like `[#3]` (default: true)
+                   - `indent` (int): Indentation spacing for nested objects (default: 2)
+                   - `delimiter` (string): Field separator for tabular arrays (default: none, use `"\t"` for tabs)
+
+                   **Example output:**
+                   ```
+                   title: Introduction to Vector Search
+                   author: Jane Doe
+                   tags[#3]: ai,search,ml
+                   metadata:
+                     edition: 2
+                     pages: 450
+                   ```
+
+                   **Default in RAG:** TOON is the default format for document rendering in RAG queries.
+
+                   **References:**
+                   - TOON Specification: https://github.com/toon-format/toon
+                   - Go Implementation: https://github.com/alpkeskin/gotoon
+
+                **Template Examples:**
+
+                Document with metadata:
+                ```handlebars
+                Title: {{metadata.title}}
+                Date: {{metadata.date}}
+                Tags: {{#each metadata.tags}}{{this}}, {{/each}}
+
+                {{content}}
+                ```
+
+                HTML content extraction:
+                ```handlebars
+                Product: {{name}}
+                Description: {{scrubHtml description_html}}
+                Price: ${{price}}
+                ```
+
+                Multimodal with image:
+                ```handlebars
+                Product: {{title}}
+                {{media url=image}}
+                Description: {{description}}
+                ```
+
+                Conditional formatting:
+                ```handlebars
+                {{title}}
+                {{#if author}}By: {{author}}{{/if}}
+                {{#if (eq category "premium")}}⭐ Premium Content{{/if}}
+                {{body}}
+                ```
+
+                **Environment Variables:**
+                - `GEMINI_API_KEY` - API key for Google AI
+                - `OPENAI_API_KEY` - API key for OpenAI
+                - `OPENAI_BASE_URL` - Base URL for OpenAI-compatible APIs
+                - `OLLAMA_HOST` - Ollama server URL (e.g., http://localhost:11434)
+
+                **Importing Pre-computed Embeddings:**
+
+                You can import existing embeddings (from OpenAI, Cohere, or any provider), but only
+                for indexes configured with `external: true`. External indexes accept vectors written
+                directly through the document `_embeddings` field and do not generate prompts from
+                `field` or `template`.
+
+                **Steps:**
+                1. Create an embeddings index with `external: true`
+                2. For dense indexes, set the index `dimension`
+                3. Write documents with `_embeddings: { "<indexName>": [...<embedding>...] }`
+
+                **Example:**
+                ```json
+                {
+                  "title": "My Document",
+                  "content": "Document text...",
+                  "_embeddings": {
+                    "my_vector_index": [0.1, 0.2, 0.3, ...]
+                  }
+                }
+                ```
+
+                **Delete Behavior:**
+                - Use `"_embeddings": { "<indexName>": null }` to delete a stored external vector
+                - Omitting `_embeddings[<indexName>]` leaves the existing vector unchanged
+
+                **Use Cases:**
+                - Migrating from another vector database with existing embeddings
+                - Using embeddings generated by external systems
+                - Importing pre-computed OpenAI, Cohere, or other provider embeddings
+                - Batch processing embeddings offline before ingestion Example: {'provider': 'openai', 'model': 'text-
+                embedding-3-small'}.
+            summarizer (GeneratorConfig | Unset): A unified configuration for a generative AI provider.
+                 Example: {'provider': 'openai', 'model': 'gpt-4.1', 'temperature': 0.7, 'max_tokens': 2048}.
+            chunker (ChunkerConfig | Unset): A unified configuration for a chunking provider. Example: {'provider':
+                'antfly', 'model': 'fixed', 'text': {'target_tokens': 500, 'overlap_tokens': 50}}.
+            top_k (int | Unset): Default number of results to return from search (sparse only) Default: 10.
+            min_weight (float | Unset): Minimum weight threshold for sparse vector entries (sparse only) Default: 0.0.
+            chunk_size (int | Unset): Number of documents per posting list chunk (sparse only) Default: 1024.
+    """
+
+    external: bool | Unset = False
+    sparse: bool | Unset = False
+    dimension: int | Unset = UNSET
+    field: str | Unset = UNSET
+    template: str | Unset = UNSET
+    distance_metric: DistanceMetric | Unset = UNSET
+    mem_only: bool | Unset = UNSET
+    embedder: EmbedderConfig | Unset = UNSET
+    summarizer: GeneratorConfig | Unset = UNSET
+    chunker: ChunkerConfig | Unset = UNSET
+    top_k: int | Unset = 10
+    min_weight: float | Unset = 0.0
+    chunk_size: int | Unset = 1024
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        external = self.external
+
+        sparse = self.sparse
+
+        dimension = self.dimension
+
+        field = self.field
+
+        template = self.template
+
+        distance_metric: str | Unset = UNSET
+        if not isinstance(self.distance_metric, Unset):
+            distance_metric = self.distance_metric.value
+
+        mem_only = self.mem_only
+
+        embedder: dict[str, Any] | Unset = UNSET
+        if not isinstance(self.embedder, Unset):
+            embedder = self.embedder.to_dict()
+
+        summarizer: dict[str, Any] | Unset = UNSET
+        if not isinstance(self.summarizer, Unset):
+            summarizer = self.summarizer.to_dict()
+
+        chunker: dict[str, Any] | Unset = UNSET
+        if not isinstance(self.chunker, Unset):
+            chunker = self.chunker.to_dict()
+
+        top_k = self.top_k
+
+        min_weight = self.min_weight
+
+        chunk_size = self.chunk_size
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update({})
+        if external is not UNSET:
+            field_dict["external"] = external
+        if sparse is not UNSET:
+            field_dict["sparse"] = sparse
+        if dimension is not UNSET:
+            field_dict["dimension"] = dimension
+        if field is not UNSET:
+            field_dict["field"] = field
+        if template is not UNSET:
+            field_dict["template"] = template
+        if distance_metric is not UNSET:
+            field_dict["distance_metric"] = distance_metric
+        if mem_only is not UNSET:
+            field_dict["mem_only"] = mem_only
+        if embedder is not UNSET:
+            field_dict["embedder"] = embedder
+        if summarizer is not UNSET:
+            field_dict["summarizer"] = summarizer
+        if chunker is not UNSET:
+            field_dict["chunker"] = chunker
+        if top_k is not UNSET:
+            field_dict["top_k"] = top_k
+        if min_weight is not UNSET:
+            field_dict["min_weight"] = min_weight
+        if chunk_size is not UNSET:
+            field_dict["chunk_size"] = chunk_size
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        from ..models.chunker_config import ChunkerConfig
+        from ..models.embedder_config import EmbedderConfig
+        from ..models.generator_config import GeneratorConfig
+
+        d = dict(src_dict)
+        external = d.pop("external", UNSET)
+
+        sparse = d.pop("sparse", UNSET)
+
+        dimension = d.pop("dimension", UNSET)
+
+        field = d.pop("field", UNSET)
+
+        template = d.pop("template", UNSET)
+
+        _distance_metric = d.pop("distance_metric", UNSET)
+        distance_metric: DistanceMetric | Unset
+        if isinstance(_distance_metric, Unset):
+            distance_metric = UNSET
+        else:
+            distance_metric = DistanceMetric(_distance_metric)
+
+        mem_only = d.pop("mem_only", UNSET)
+
+        _embedder = d.pop("embedder", UNSET)
+        embedder: EmbedderConfig | Unset
+        if isinstance(_embedder, Unset):
+            embedder = UNSET
+        else:
+            embedder = EmbedderConfig.from_dict(_embedder)
+
+        _summarizer = d.pop("summarizer", UNSET)
+        summarizer: GeneratorConfig | Unset
+        if isinstance(_summarizer, Unset):
+            summarizer = UNSET
+        else:
+            summarizer = GeneratorConfig.from_dict(_summarizer)
+
+        _chunker = d.pop("chunker", UNSET)
+        chunker: ChunkerConfig | Unset
+        if isinstance(_chunker, Unset):
+            chunker = UNSET
+        else:
+            chunker = ChunkerConfig.from_dict(_chunker)
+
+        top_k = d.pop("top_k", UNSET)
+
+        min_weight = d.pop("min_weight", UNSET)
+
+        chunk_size = d.pop("chunk_size", UNSET)
+
+        embeddings_index_config = cls(
+            external=external,
+            sparse=sparse,
+            dimension=dimension,
+            field=field,
+            template=template,
+            distance_metric=distance_metric,
+            mem_only=mem_only,
+            embedder=embedder,
+            summarizer=summarizer,
+            chunker=chunker,
+            top_k=top_k,
+            min_weight=min_weight,
+            chunk_size=chunk_size,
+        )
+
+        embeddings_index_config.additional_properties = d
+        return embeddings_index_config
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

antfly/client_generated/models/embeddings_index_stats.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+from ..models.embeddings_index_stats_index_type import EmbeddingsIndexStatsIndexType
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="EmbeddingsIndexStats")
+
+
+@_attrs_define
+class EmbeddingsIndexStats:
+    """Statistics for an embeddings index (dense or sparse)
+
+    Attributes:
+        index_type (EmbeddingsIndexStatsIndexType): Discriminator for the index stats variant.
+        error (str | Unset): Error message if stats could not be retrieved
+        total_indexed (int | Unset): Number of vectors/documents in the index
+        disk_usage (int | Unset): Size of the index in bytes
+        total_nodes (int | Unset): Total number of nodes in the index (dense only)
+        total_terms (int | Unset): Number of unique terms in the inverted index (sparse only)
+        rebuilding (bool | Unset): Whether the index enricher is currently backfilling
+        wal_backlog (int | Unset): Number of documents pending enrichment in the WAL
+        backfill_progress (float | Unset): Backfill progress as a ratio from 0.0 to 1.0
+        backfill_items_processed (int | Unset): Total items processed during backfill
+    """
+
+    index_type: EmbeddingsIndexStatsIndexType
+    error: str | Unset = UNSET
+    total_indexed: int | Unset = UNSET
+    disk_usage: int | Unset = UNSET
+    total_nodes: int | Unset = UNSET
+    total_terms: int | Unset = UNSET
+    rebuilding: bool | Unset = UNSET
+    wal_backlog: int | Unset = UNSET
+    backfill_progress: float | Unset = UNSET
+    backfill_items_processed: int | Unset = UNSET
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        index_type = self.index_type.value
+
+        error = self.error
+
+        total_indexed = self.total_indexed
+
+        disk_usage = self.disk_usage
+
+        total_nodes = self.total_nodes
+
+        total_terms = self.total_terms
+
+        rebuilding = self.rebuilding
+
+        wal_backlog = self.wal_backlog
+
+        backfill_progress = self.backfill_progress
+
+        backfill_items_processed = self.backfill_items_processed
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "index_type": index_type,
+            }
+        )
+        if error is not UNSET:
+            field_dict["error"] = error
+        if total_indexed is not UNSET:
+            field_dict["total_indexed"] = total_indexed
+        if disk_usage is not UNSET:
+            field_dict["disk_usage"] = disk_usage
+        if total_nodes is not UNSET:
+            field_dict["total_nodes"] = total_nodes
+        if total_terms is not UNSET:
+            field_dict["total_terms"] = total_terms
+        if rebuilding is not UNSET:
+            field_dict["rebuilding"] = rebuilding
+        if wal_backlog is not UNSET:
+            field_dict["wal_backlog"] = wal_backlog
+        if backfill_progress is not UNSET:
+            field_dict["backfill_progress"] = backfill_progress
+        if backfill_items_processed is not UNSET:
+            field_dict["backfill_items_processed"] = backfill_items_processed
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        d = dict(src_dict)
+        index_type = EmbeddingsIndexStatsIndexType(d.pop("index_type"))
+
+        error = d.pop("error", UNSET)
+
+        total_indexed = d.pop("total_indexed", UNSET)
+
+        disk_usage = d.pop("disk_usage", UNSET)
+
+        total_nodes = d.pop("total_nodes", UNSET)
+
+        total_terms = d.pop("total_terms", UNSET)
+
+        rebuilding = d.pop("rebuilding", UNSET)
+
+        wal_backlog = d.pop("wal_backlog", UNSET)
+
+        backfill_progress = d.pop("backfill_progress", UNSET)
+
+        backfill_items_processed = d.pop("backfill_items_processed", UNSET)
+
+        embeddings_index_stats = cls(
+            index_type=index_type,
+            error=error,
+            total_indexed=total_indexed,
+            disk_usage=disk_usage,
+            total_nodes=total_nodes,
+            total_terms=total_terms,
+            rebuilding=rebuilding,
+            wal_backlog=wal_backlog,
+            backfill_progress=backfill_progress,
+            backfill_items_processed=backfill_items_processed,
+        )
+
+        embeddings_index_stats.additional_properties = d
+        return embeddings_index_stats
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

antfly/client_generated/models/embeddings_index_stats_index_type.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class EmbeddingsIndexStatsIndexType(str, Enum):
+    EMBEDDINGS = "embeddings"
+
+    def __str__(self) -> str:
+        return str(self.value)

antfly/client_generated/models/error.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+T = TypeVar("T", bound="Error")
+
+
+@_attrs_define
+class Error:
+    """
+    Attributes:
+        error (str):  Example: An error message.
+    """
+
+    error: str
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        error = self.error
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "error": error,
+            }
+        )
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        d = dict(src_dict)
+        error = d.pop("error")
+
+        error = cls(
+            error=error,
+        )
+
+        error.additional_properties = d
+        return error
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties