pgvector-template 0.5__tar.gz → 0.6__tar.gz

{pgvector_template-0.5 → pgvector_template-0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-template
-Version: 0.5
+Version: 0.6
 Summary: Template library for flexible PGVector RAG implementations
 Author-email: DL <v49t9zpqd@mozmail.com>
 License: MIT
@@ -39,38 +39,36 @@ PGVector-Template provides a robust foundation for implementing vector-based doc
 ## Quick Start
 
 ```python
-from pgvector_template import DatabaseManager, DocumentDatabaseManager
+from pgvector_template import DocumentDatabaseManager
 from pgvector_template.core.document import BaseDocument
-from sqlalchemy import create_engine
-from uuid import uuid4
+from pgvector_template.models.search import SearchQuery
+from pgvector_template.service.document_service import DocumentService, DocumentServiceConfig
+
 
 # 1. Define your document model
 class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 
-# 2. Set up database connection
-engine = create_engine("postgresql://user:pass@localhost/mydb")
-db_manager = DatabaseManager(engine)
-db_manager.create_tables([MyDocument])
-
-# 3. Create document manager
-doc_manager = DocumentDatabaseManager(db_manager)
-
-# 4. Insert a document
-with db_manager.get_session() as session:
-    doc = MyDocument.from_props(
-        corpus_id=uuid4(),
-        chunk_index=0,
-        content="Your document content here",
-        embedding=[0.1, 0.2, 0.3, ...],  # Your embedding vector
-    )
-    doc_manager.insert_document(session, doc)
 
-# 5. Search similar documents
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session, MyDocument, query_embedding=[0.1, 0.2, 0.3, ...], limit=5
-    )
+# 2. Set up the database: connects, enables pgvector, creates the schema and tables
+db = DocumentDatabaseManager(
+    database_url="postgresql://user:pass@localhost/mydb",
+    schema_suffix="my_kb",
+    document_classes=[MyDocument],
+)
+db.setup()
+
+# 3. Create a service. `my_embedding_provider` is your `BaseEmbeddingProvider` subclass
+config = DocumentServiceConfig(document_cls=MyDocument, embedding_provider=my_embedding_provider)
+
+with db.get_session() as session:
+    service = DocumentService(session, config)
+
+    # 4. Insert a corpus: it is split into chunks, embedded and stored
+    service.corpus_manager.insert_corpus("Your document content here", {}, corpus_id="doc-1")
+
+    # 5. Search
+    results = service.search_client.search(SearchQuery(text="your query", limit=5))
 ```
 
 ## Key Concepts
@@ -89,6 +87,7 @@ Example: A 50-page PDF (corpus) might be split into 200 documents (chunks), all
 - **Metadata Management**: JSON-based flexible metadata with GIN indexing
 - **Collection Support**: Organize documents into logical collections
 - **Chunk Management**: Handle long content by chunking into retrievable documents
+- **Smart Upserts**: Re-inserting a corpus only re-embeds chunks whose content changed, and skips the write entirely if nothing did
 - **Database Abstraction**: Clean SQLAlchemy-based database layer with schema creation API
 - **Type Safety**: Full Pydantic validation and type hints
 - **Production Ready**: Comprehensive testing and error handling
@@ -142,33 +141,14 @@ CREATE EXTENSION IF NOT EXISTS vector;
 3. **Set up your connection:**
 
 ```python
-from sqlalchemy import create_engine
 from pgvector_template import DatabaseManager
 
-# Option 1: Direct connection string
-engine = create_engine("postgresql://user:password@localhost:5432/mydb")
-db_manager = DatabaseManager(engine)
-
-# Option 2: From environment variable
-import os
-engine = create_engine(os.getenv("DATABASE_URL"))
-db_manager = DatabaseManager(engine)
+db_manager = DatabaseManager("postgresql://user:password@localhost:5432/mydb")
+db_manager.initialize()  # connects and enables the vector extension
 ```
 
-### Production Configuration
-
-```python
-from sqlalchemy import create_engine
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    "postgresql://user:password@localhost:5432/mydb",
-    poolclass=QueuePool,
-    pool_size=10,
-    max_overflow=20,
-    pool_pre_ping=True,
-)
-```
+`create_tables` also adds nullable model columns that are missing from existing tables (for example `content_hash`).
+Only the column name and type are applied: indexes, foreign keys, unique constraints and server defaults are not. This is a stopgap until proper migrations are added.
 
 ### Environment Variables
 
@@ -185,85 +165,77 @@ DATABASE_URL=postgresql://user:password@localhost:5432/dev_db
 from pgvector_template.core.document import BaseDocument, BaseDocumentMetadata
 from pydantic import Field
 
+
 class MyDocumentMetadata(BaseDocumentMetadata):
     source_type: str = Field(..., description="Type of source document")
     author: str = Field(default="unknown", description="Document author")
 
+
 class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 ```
 
-### 2. Insert Documents
+### 2. Insert a Corpus
 
 ```python
 from pgvector_template.core.document import BaseDocumentOptionalProps
-from uuid import uuid4
 
-# Create document with metadata
-metadata = MyDocumentMetadata(source_type="pdf", author="John Doe")
 optional_props = BaseDocumentOptionalProps(
     title="Chapter 1: Introduction",
     collection="textbooks",
     language="en",
-    tags=["education", "intro"]
 )
 
-doc = MyDocument.from_props(
-    corpus_id=uuid4(),
-    chunk_index=0,
-    content="This is the document content...",
-    embedding=your_embedding_vector,  # list[float] with 1024 dimensions
-    metadata=metadata.to_dict(),
-    optional_props=optional_props
-)
+with db.get_session() as session:
+    service = DocumentService(session, config)
+    result = service.corpus_manager.insert_corpus(
+        "This is the document content...",
+        {"source_type": "pdf", "author": "John Doe"},  # corpus metadata
+        optional_props,
+        corpus_id="intro-chapter",
+    )
+```
+
+To insert chunks you have already split and embedded, use `insert_documents()`.
 
-with db_manager.get_session() as session:
-    doc_manager.insert_document(session, doc)
+Calling `insert_corpus()` again with the same `corpus_id` replaces the corpus. Unchanged chunks reuse their stored vector, and the write is skipped if nothing changed:
+
+```python
+result = corpus_manager.insert_corpus(text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=12, reused=0, total=12)
+
+result = corpus_manager.insert_corpus(edited_text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=1, reused=11, total=12)
 ```
 
+Rows written before `content_hash` existed have no hash, so their first upsert re-embeds them.
+
 ### 3. Search Documents
 
 ```python
-# Basic similarity search
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session=session,
-        document_cls=MyDocument,
-        query_embedding=query_vector,
-        limit=10
-    )
+from pgvector_template.models.search import MetadataFilter, SearchQuery
+
+# Semantic search
+results = service.search_client.search(SearchQuery(text="how do I reset my password", limit=10))
 
-# Search with filters
-from pgvector_template.models.search import MetadataFilter
-
-filters = [
-    MetadataFilter(key="source_type", value="pdf"),
-    MetadataFilter(key="author", value="John Doe")
-]
-
-results = doc_manager.search_similar(
-    session=session,
-    document_cls=MyDocument,
-    query_embedding=query_vector,
-    limit=10,
-    metadata_filters=filters,
-    collection="textbooks"
+# Keyword search combined with a metadata filter
+results = service.search_client.search(
+    SearchQuery(
+        keywords=["password", "reset"],
+        metadata_filters=[MetadataFilter(field_name="source_type", condition="eq", value="pdf")],
+        limit=10,
+    )
 )
+
+for r in results:
+    print(r.score, r.document.content)
 ```
 
-### 4. Manage Collections
+### 4. Read a Corpus Back
 
 ```python
-# Get all documents in a collection
-with db_manager.get_session() as session:
-    docs = doc_manager.get_documents_by_collection(
-        session, MyDocument, "textbooks"
-    )
-
-# Get all chunks from a corpus
-    corpus_docs = doc_manager.get_documents_by_corpus_id(
-        session, MyDocument, corpus_id
-    )
+corpus = service.corpus_manager.get_full_corpus("intro-chapter")
+print(corpus.content, corpus.metadata, len(corpus.documents))
 ```
 
 ## Concept Reference
@@ -273,8 +245,11 @@ with db_manager.get_session() as session:
 - **`BaseDocument`**: Abstract SQLAlchemy model for documents with vector embeddings
 - **`BaseDocumentOptionalProps`**: Pydantic model for optional document properties
 - **`BaseDocumentMetadata`**: Base schema for structured document metadata
-- **`DatabaseManager`**: Database connection and session management
-- **`DocumentDatabaseManager`**: High-level document CRUD operations
+- **`BaseEmbeddingProvider`**: Interface to implement for your embedding model
+- **`BaseCorpusManager`**: Splits, embeds and upserts corpora (`insert_corpus`, `get_full_corpus`)
+- **`BaseSearchClient`**: Semantic, keyword and metadata search (`search`)
+- **`DocumentService`**: Bundles a corpus manager and a search client over one session
+- **`DatabaseManager`** / **`DocumentDatabaseManager`**: Connection, schema and table setup
 
 ## Testing
 
@@ -308,7 +283,8 @@ python -m unittest discover -s integ-tests
 
 ```bash
 pip install -e .[dev,test]
-black .  # Format code
+ruff check . && ruff format .
+ty check .
 ```
 
 ## License

{pgvector_template-0.5 → pgvector_template-0.6}/README.md

@@ -9,38 +9,36 @@ PGVector-Template provides a robust foundation for implementing vector-based doc
 ## Quick Start
 
 ```python
-from pgvector_template import DatabaseManager, DocumentDatabaseManager
+from pgvector_template import DocumentDatabaseManager
 from pgvector_template.core.document import BaseDocument
-from sqlalchemy import create_engine
-from uuid import uuid4
+from pgvector_template.models.search import SearchQuery
+from pgvector_template.service.document_service import DocumentService, DocumentServiceConfig
+
 
 # 1. Define your document model
 class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 
-# 2. Set up database connection
-engine = create_engine("postgresql://user:pass@localhost/mydb")
-db_manager = DatabaseManager(engine)
-db_manager.create_tables([MyDocument])
-
-# 3. Create document manager
-doc_manager = DocumentDatabaseManager(db_manager)
-
-# 4. Insert a document
-with db_manager.get_session() as session:
-    doc = MyDocument.from_props(
-        corpus_id=uuid4(),
-        chunk_index=0,
-        content="Your document content here",
-        embedding=[0.1, 0.2, 0.3, ...],  # Your embedding vector
-    )
-    doc_manager.insert_document(session, doc)
 
-# 5. Search similar documents
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session, MyDocument, query_embedding=[0.1, 0.2, 0.3, ...], limit=5
-    )
+# 2. Set up the database: connects, enables pgvector, creates the schema and tables
+db = DocumentDatabaseManager(
+    database_url="postgresql://user:pass@localhost/mydb",
+    schema_suffix="my_kb",
+    document_classes=[MyDocument],
+)
+db.setup()
+
+# 3. Create a service. `my_embedding_provider` is your `BaseEmbeddingProvider` subclass
+config = DocumentServiceConfig(document_cls=MyDocument, embedding_provider=my_embedding_provider)
+
+with db.get_session() as session:
+    service = DocumentService(session, config)
+
+    # 4. Insert a corpus: it is split into chunks, embedded and stored
+    service.corpus_manager.insert_corpus("Your document content here", {}, corpus_id="doc-1")
+
+    # 5. Search
+    results = service.search_client.search(SearchQuery(text="your query", limit=5))
 ```
 
 ## Key Concepts
@@ -59,6 +57,7 @@ Example: A 50-page PDF (corpus) might be split into 200 documents (chunks), all
 - **Metadata Management**: JSON-based flexible metadata with GIN indexing
 - **Collection Support**: Organize documents into logical collections
 - **Chunk Management**: Handle long content by chunking into retrievable documents
+- **Smart Upserts**: Re-inserting a corpus only re-embeds chunks whose content changed, and skips the write entirely if nothing did
 - **Database Abstraction**: Clean SQLAlchemy-based database layer with schema creation API
 - **Type Safety**: Full Pydantic validation and type hints
 - **Production Ready**: Comprehensive testing and error handling
@@ -112,33 +111,14 @@ CREATE EXTENSION IF NOT EXISTS vector;
 3. **Set up your connection:**
 
 ```python
-from sqlalchemy import create_engine
 from pgvector_template import DatabaseManager
 
-# Option 1: Direct connection string
-engine = create_engine("postgresql://user:password@localhost:5432/mydb")
-db_manager = DatabaseManager(engine)
-
-# Option 2: From environment variable
-import os
-engine = create_engine(os.getenv("DATABASE_URL"))
-db_manager = DatabaseManager(engine)
+db_manager = DatabaseManager("postgresql://user:password@localhost:5432/mydb")
+db_manager.initialize()  # connects and enables the vector extension
 ```
 
-### Production Configuration
-
-```python
-from sqlalchemy import create_engine
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    "postgresql://user:password@localhost:5432/mydb",
-    poolclass=QueuePool,
-    pool_size=10,
-    max_overflow=20,
-    pool_pre_ping=True,
-)
-```
+`create_tables` also adds nullable model columns that are missing from existing tables (for example `content_hash`).
+Only the column name and type are applied: indexes, foreign keys, unique constraints and server defaults are not. This is a stopgap until proper migrations are added.
 
 ### Environment Variables
 
@@ -155,85 +135,77 @@ DATABASE_URL=postgresql://user:password@localhost:5432/dev_db
 from pgvector_template.core.document import BaseDocument, BaseDocumentMetadata
 from pydantic import Field
 
+
 class MyDocumentMetadata(BaseDocumentMetadata):
     source_type: str = Field(..., description="Type of source document")
     author: str = Field(default="unknown", description="Document author")
 
+
 class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 ```
 
-### 2. Insert Documents
+### 2. Insert a Corpus
 
 ```python
 from pgvector_template.core.document import BaseDocumentOptionalProps
-from uuid import uuid4
 
-# Create document with metadata
-metadata = MyDocumentMetadata(source_type="pdf", author="John Doe")
 optional_props = BaseDocumentOptionalProps(
     title="Chapter 1: Introduction",
     collection="textbooks",
     language="en",
-    tags=["education", "intro"]
 )
 
-doc = MyDocument.from_props(
-    corpus_id=uuid4(),
-    chunk_index=0,
-    content="This is the document content...",
-    embedding=your_embedding_vector,  # list[float] with 1024 dimensions
-    metadata=metadata.to_dict(),
-    optional_props=optional_props
-)
+with db.get_session() as session:
+    service = DocumentService(session, config)
+    result = service.corpus_manager.insert_corpus(
+        "This is the document content...",
+        {"source_type": "pdf", "author": "John Doe"},  # corpus metadata
+        optional_props,
+        corpus_id="intro-chapter",
+    )
+```
+
+To insert chunks you have already split and embedded, use `insert_documents()`.
 
-with db_manager.get_session() as session:
-    doc_manager.insert_document(session, doc)
+Calling `insert_corpus()` again with the same `corpus_id` replaces the corpus. Unchanged chunks reuse their stored vector, and the write is skipped if nothing changed:
+
+```python
+result = corpus_manager.insert_corpus(text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=12, reused=0, total=12)
+
+result = corpus_manager.insert_corpus(edited_text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=1, reused=11, total=12)
 ```
 
+Rows written before `content_hash` existed have no hash, so their first upsert re-embeds them.
+
 ### 3. Search Documents
 
 ```python
-# Basic similarity search
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session=session,
-        document_cls=MyDocument,
-        query_embedding=query_vector,
-        limit=10
-    )
+from pgvector_template.models.search import MetadataFilter, SearchQuery
+
+# Semantic search
+results = service.search_client.search(SearchQuery(text="how do I reset my password", limit=10))
 
-# Search with filters
-from pgvector_template.models.search import MetadataFilter
-
-filters = [
-    MetadataFilter(key="source_type", value="pdf"),
-    MetadataFilter(key="author", value="John Doe")
-]
-
-results = doc_manager.search_similar(
-    session=session,
-    document_cls=MyDocument,
-    query_embedding=query_vector,
-    limit=10,
-    metadata_filters=filters,
-    collection="textbooks"
+# Keyword search combined with a metadata filter
+results = service.search_client.search(
+    SearchQuery(
+        keywords=["password", "reset"],
+        metadata_filters=[MetadataFilter(field_name="source_type", condition="eq", value="pdf")],
+        limit=10,
+    )
 )
+
+for r in results:
+    print(r.score, r.document.content)
 ```
 
-### 4. Manage Collections
+### 4. Read a Corpus Back
 
 ```python
-# Get all documents in a collection
-with db_manager.get_session() as session:
-    docs = doc_manager.get_documents_by_collection(
-        session, MyDocument, "textbooks"
-    )
-
-# Get all chunks from a corpus
-    corpus_docs = doc_manager.get_documents_by_corpus_id(
-        session, MyDocument, corpus_id
-    )
+corpus = service.corpus_manager.get_full_corpus("intro-chapter")
+print(corpus.content, corpus.metadata, len(corpus.documents))
 ```
 
 ## Concept Reference
@@ -243,8 +215,11 @@ with db_manager.get_session() as session:
 - **`BaseDocument`**: Abstract SQLAlchemy model for documents with vector embeddings
 - **`BaseDocumentOptionalProps`**: Pydantic model for optional document properties
 - **`BaseDocumentMetadata`**: Base schema for structured document metadata
-- **`DatabaseManager`**: Database connection and session management
-- **`DocumentDatabaseManager`**: High-level document CRUD operations
+- **`BaseEmbeddingProvider`**: Interface to implement for your embedding model
+- **`BaseCorpusManager`**: Splits, embeds and upserts corpora (`insert_corpus`, `get_full_corpus`)
+- **`BaseSearchClient`**: Semantic, keyword and metadata search (`search`)
+- **`DocumentService`**: Bundles a corpus manager and a search client over one session
+- **`DatabaseManager`** / **`DocumentDatabaseManager`**: Connection, schema and table setup
 
 ## Testing
 
@@ -278,7 +253,8 @@ python -m unittest discover -s integ-tests
 
 ```bash
 pip install -e .[dev,test]
-black .  # Format code
+ruff check . && ruff format .
+ty check .
 ```
 
 ## License

{pgvector_template-0.5 → pgvector_template-0.6}/pgvector_template/core/__init__.py

@@ -5,21 +5,16 @@ from pgvector_template.core.document import (
 )
 from pgvector_template.core.embedder import BaseEmbeddingProvider
 from pgvector_template.core.manager import BaseCorpusManager, BaseCorpusManagerConfig, Corpus
-from pgvector_template.core.search import BaseSearchClientConfig, BaseSearchClient
-
+from pgvector_template.core.search import BaseSearchClient, BaseSearchClientConfig
 
 __all__ = [
-    ### document
-    "BaseDocumentOptionalProps",
+    "BaseCorpusManager",
+    "BaseCorpusManagerConfig",
     "BaseDocument",
     "BaseDocumentMetadata",
-    "Corpus",
-    ### embedder
+    "BaseDocumentOptionalProps",
     "BaseEmbeddingProvider",
-    ### manager
-    "BaseCorpusManager",
-    "BaseCorpusManagerConfig",
-    ### search
-    "BaseSearchClientConfig",
     "BaseSearchClient",
+    "BaseSearchClientConfig",
+    "Corpus",
 ]

{pgvector_template-0.5 → pgvector_template-0.6}/pgvector_template/core/document.py

@@ -1,21 +1,24 @@
-from datetime import datetime, timezone
-from typing import Any, Type, TypeVar
-from uuid import uuid4, UUID as UuidLiteral
+import hashlib
+import json
+from datetime import UTC, datetime
+from typing import Any, Self
+from uuid import UUID as UuidLiteral
+from uuid import uuid4
 
+from pgvector.sqlalchemy import Vector
 from pydantic import BaseModel, Field
 from sqlalchemy import (
+    Boolean,
     Column,
-    String,
-    Text,
     DateTime,
-    Boolean,
-    Integer,
     Index,
+    Integer,
+    String,
+    Text,
     UniqueConstraint,
 )
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.orm import declarative_base
-from sqlalchemy.dialects.postgresql import UUID, JSONB
-from pgvector.sqlalchemy import Vector
 
 Base = declarative_base()
 
@@ -33,9 +36,6 @@ class BaseDocumentOptionalProps(BaseModel):
     """Language of the content (ISO 639-1 code), e.g., 'en', 'es', 'zh'"""
 
 
-T = TypeVar("T", bound="BaseDocument")
-
-
 class BaseDocument(Base):
     """
     Template table for Documents, that works for all collection types.
@@ -88,12 +88,18 @@ class BaseDocument(Base):
     Use this field for auditing and diagnosing mixed-model corpora, not for runtime dispatch.
     """
 
+    content_hash = Column(String(64), nullable=True)
+    """
+    Hex sha256 of `content` + `embedding_config`, see `compute_content_hash()`.
+    Lets upserts reuse a stored embedding when a chunk is unchanged. NULL for rows that pre-date this column.
+    """
+
     # Audit fields
-    created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
+    created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
     updated_at = Column(
         DateTime(timezone=True),
-        default=lambda: datetime.now(timezone.utc),
-        onupdate=lambda: datetime.now(timezone.utc),
+        default=lambda: datetime.now(UTC),
+        onupdate=lambda: datetime.now(UTC),
     )
     is_deleted = Column(Boolean, default=False, index=True)
     """Entries can be logically marked for deletion before they are permanently deleted."""
@@ -131,7 +137,7 @@ class BaseDocument(Base):
 
     @classmethod
     def from_props(
-        cls: Type[T],
+        cls,
         corpus_id: UuidLiteral | str,
         chunk_index: int,
         content: str,
@@ -139,7 +145,7 @@ class BaseDocument(Base):
         embedding_config: dict[str, Any] | None = None,
         metadata: dict[str, Any] | None = None,
         optional_props: BaseDocumentOptionalProps | None = None,
-    ) -> T:
+    ) -> Self:
         """
         Create a BaseDocument instance from mandatory and optional properties.
 
@@ -167,6 +173,7 @@ class BaseDocument(Base):
             content=content,
             embedding=embedding,
             embedding_config=embedding_config,
+            content_hash=cls.compute_content_hash(content, embedding_config),
             title=optional_props.title,
             document_metadata=metadata or {},
             collection=optional_props.collection,
@@ -174,6 +181,15 @@ class BaseDocument(Base):
             language=optional_props.language,
         )
 
+    @classmethod
+    def compute_content_hash(cls, content: str, embedding_config: dict[str, Any] | None) -> str:
+        """
+        Hash of everything that determines a chunk's embedding vector. Metadata and other props are excluded.
+        Override to change what counts as "the same chunk".
+        """
+        payload = json.dumps([content, embedding_config], sort_keys=True, ensure_ascii=False)
+        return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
     @classmethod
     def get_embedding_index(cls, table_name: str) -> Index:
         """Override this method to customize the embedding index."""