pgvector-template 0.3.0__tar.gz → 0.3.2__tar.gz

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgvector-template
-Version: 0.3.0
+Version: 0.3.2
 Summary: Template library for flexible PGVector RAG implementations
 Author-email: DL <v49t9zpqd@mozmail.com>
 License: MIT

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template/core/__init__.py

@@ -1,7 +1,7 @@
 from pgvector_template.core.document import BaseDocument, BaseDocumentMetadata, BaseDocumentOptionalProps
 from pgvector_template.core.embedder import BaseEmbeddingProvider
 from pgvector_template.core.manager import BaseCorpusManager, BaseCorpusManagerConfig, Corpus
-from pgvector_template.core.search import SearchQuery, RetrievalResult, BaseSearchClientConfig, BaseSearchClient
+from pgvector_template.core.search import BaseSearchClientConfig, BaseSearchClient
 
 
 __all__ = [
@@ -16,8 +16,6 @@ __all__ = [
     "BaseCorpusManager",
     "BaseCorpusManagerConfig",
     ### search
-    "SearchQuery",
-    "RetrievalResult",
     "BaseSearchClientConfig",
     "BaseSearchClient",
 ]

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template/core/document.py

@@ -195,9 +195,12 @@ class BaseDocumentMetadata(BaseModel):
     """
 
     document_type: str = Field(
-        ..., description="Description for type of document, e.g. markdown, pdf, etc"
+        ..., description="Original document type/format/extension, e.g. md, pdf, html, json, etc"
+    )
+    schema_version: str = Field(
+        default="1.0",
+        description="Schema version for the metadata. Intended for housekeeping only",
     )
-    schema_version: str = Field(default="1.0", description="Schema version for the metadata")
 
     def to_dict(self) -> dict[str, Any]:
         return self.model_dump()

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template/core/manager.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from logging import getLogger
-from typing import Any, Type
+from typing import Any, Literal, Type
 from uuid import UUID, uuid4
 
 from pydantic import BaseModel, Field
@@ -111,6 +111,7 @@ class BaseCorpusManager(ABC):
         corpus_metadata: dict[str, Any],
         optional_props: BaseDocumentOptionalProps | None = None,
         corpus_id: UUID | str | None = None,
+        update_if_exists: bool = True,
         **kwargs,
     ) -> int:
         """
@@ -130,7 +131,12 @@ class BaseCorpusManager(ABC):
         document_contents = self._split_corpus(content)
         document_embeddings = self.embedding_provider.embed_batch(document_contents)
         return self.insert_documents(
-            corpus_id, document_contents, document_embeddings, corpus_metadata, optional_props
+            corpus_id,
+            document_contents,
+            document_embeddings,
+            corpus_metadata,
+            optional_props,
+            update_if_exists,
         )
 
     def insert_documents(
@@ -140,6 +146,7 @@ class BaseCorpusManager(ABC):
         document_embeddings: list[list[float]],
         corpus_metadata: dict[str, Any],
         optional_props: BaseDocumentOptionalProps | None = None,
+        update_if_exists: bool = True,
         **kwargs,
     ) -> int:
         """
@@ -176,7 +183,12 @@ class BaseCorpusManager(ABC):
                     optional_props=optional_props,
                 )
             )
-        self.session.add_all(documents_to_insert)
+
+        if update_if_exists:
+            for doc in documents_to_insert:
+                self.session.merge(doc)
+        else:
+            self.session.add_all(documents_to_insert)
         self.session.commit()
         return len(documents_to_insert)
 
@@ -189,7 +201,7 @@ class BaseCorpusManager(ABC):
             logger.warning(
                 "Using default _split_corpus. Override this method to improve performance."
             )
-        split_content = [content[i : i + 1000] for i in range(0, len(content), 1000)]
+        split_content = [content[i : i + 1024 * 4] for i in range(0, len(content), 1024 * 4)]
         return [c for c in split_content if len(c.strip()) > 0]
 
     def _join_documents(

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template/core/search.py

@@ -138,7 +138,9 @@ class BaseSearchClient:
 
         # strictly speaking, this validation step is optional. You may override this method to disable
         try:
-            validate_metadata_filters(search_query.metadata_filters, self.config.document_metadata_cls)
+            validate_metadata_filters(
+                search_query.metadata_filters, self.config.document_metadata_cls
+            )
         except ValueError as e:
             logger.warning(f"Metadata filter validation failed: {e}. Query success not guaranteed!")
 

pgvector_template-0.3.2/pgvector_template/models/__init__.py

@@ -0,0 +1 @@
+from pgvector_template.models.search import *

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template/models/search.py

@@ -1,17 +1,17 @@
 from dataclasses import dataclass, asdict
 from datetime import datetime
-from typing import Any, Literal
+from typing import Any, Literal, Type
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-from pgvector_template.core import BaseDocument
+from pgvector_template.core import BaseDocument, BaseDocumentMetadata
 
 
 class MetadataFilter(BaseModel):
     """
     An object acting as a filter for an arbitrary `Metadata` dictionary/map/object.
     """
-    
+
     field_name: str
     """Field path in metadata. Use dot notation for nested fields (e.g., 'publication_info.journal')"""
     condition: Literal["eq", "gt", "gte", "lt", "lte", "contains", "in", "exists"]
@@ -31,14 +31,22 @@ class MetadataFilter(BaseModel):
 
 
 class SearchQuery(BaseModel):
-    """Standardized search query structure. At least 1 search criterion is required."""
+    """
+    Standardized search query structure. At least 1 search criterion is required.
+    """
 
     text: str | None = None
     """String to match against using in a semantic search, i.e. using vector distance."""
     keywords: list[str] = []
     """List of keywords to **exact-match** in a keyword search."""
-    metadata_filters: list[MetadataFilter] = []
-    """List of metadata filter conditions that must be matched."""
+    metadata_filters: list[MetadataFilter] = Field(
+        default=[],
+        json_schema_extra={"metadata_schema": BaseDocumentMetadata.model_json_schema()},
+    )
+    """
+    List of metadata conditions that must be matched.
+    Refer to `metadata_schema` for the expected schema, as it exists in the database.
+    """
     date_range: tuple[datetime, datetime] | None = None
     """Retrieve/limit results based on created_at & updated_at timestamps (i.e. database operations)"""
     limit: int = Field(
@@ -47,7 +55,10 @@ class SearchQuery(BaseModel):
     )
     """Maximum number of results to return."""
 
-    model_config = ConfigDict(use_attribute_docstrings=True)
+    model_config = ConfigDict(
+        use_attribute_docstrings=True,
+        arbitrary_types_allowed=True,
+    )
 
     @model_validator(mode="after")
     def ensure_criterion(self):

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template/service/__init__.py

@@ -4,4 +4,4 @@ Service layer for document operations.
 
 from pgvector_template.service.document_service import DocumentService, DocumentServiceConfig
 
-__all__ = ["DocumentService", "DocumentServiceConfig"]
+__all__ = ["DocumentService", "DocumentServiceConfig"]

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pgvector_template.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgvector-template
-Version: 0.3.0
+Version: 0.3.2
 Summary: Template library for flexible PGVector RAG implementations
 Author-email: DL <v49t9zpqd@mozmail.com>
 License: MIT

{pgvector_template-0.3.0 → pgvector_template-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pgvector-template"
-version = "0.3.0"
+version = "0.3.2"
 description = "Template library for flexible PGVector RAG implementations"
 authors = [{ name="DL", email="v49t9zpqd@mozmail.com" }]
 license = { text = "MIT" }