pgvector-template 0.5.1__tar.gz → 0.6__tar.gz

{pgvector_template-0.5.1 → pgvector_template-0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-template
-Version: 0.5.1
+Version: 0.6
 Summary: Template library for flexible PGVector RAG implementations
 Author-email: DL <v49t9zpqd@mozmail.com>
 License: MIT
@@ -39,10 +39,10 @@ PGVector-Template provides a robust foundation for implementing vector-based doc
 ## Quick Start
 
 ```python
-from pgvector_template import DatabaseManager, DocumentDatabaseManager
+from pgvector_template import DocumentDatabaseManager
 from pgvector_template.core.document import BaseDocument
-from sqlalchemy import create_engine
-from uuid import uuid4
+from pgvector_template.models.search import SearchQuery
+from pgvector_template.service.document_service import DocumentService, DocumentServiceConfig
 
 
 # 1. Define your document model
@@ -50,29 +50,25 @@ class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 
 
-# 2. Set up database connection
-engine = create_engine("postgresql://user:pass@localhost/mydb")
-db_manager = DatabaseManager(engine)
-db_manager.create_tables([MyDocument])
+# 2. Set up the database: connects, enables pgvector, creates the schema and tables
+db = DocumentDatabaseManager(
+    database_url="postgresql://user:pass@localhost/mydb",
+    schema_suffix="my_kb",
+    document_classes=[MyDocument],
+)
+db.setup()
 
-# 3. Create document manager
-doc_manager = DocumentDatabaseManager(db_manager)
+# 3. Create a service. `my_embedding_provider` is your `BaseEmbeddingProvider` subclass
+config = DocumentServiceConfig(document_cls=MyDocument, embedding_provider=my_embedding_provider)
 
-# 4. Insert a document
-with db_manager.get_session() as session:
-    doc = MyDocument.from_props(
-        corpus_id=uuid4(),
-        chunk_index=0,
-        content="Your document content here",
-        embedding=[0.1, 0.2, 0.3, ...],  # Your embedding vector
-    )
-    doc_manager.insert_document(session, doc)
+with db.get_session() as session:
+    service = DocumentService(session, config)
 
-# 5. Search similar documents
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session, MyDocument, query_embedding=[0.1, 0.2, 0.3, ...], limit=5
-    )
+    # 4. Insert a corpus: it is split into chunks, embedded and stored
+    service.corpus_manager.insert_corpus("Your document content here", {}, corpus_id="doc-1")
+
+    # 5. Search
+    results = service.search_client.search(SearchQuery(text="your query", limit=5))
 ```
 
 ## Key Concepts
@@ -91,6 +87,7 @@ Example: A 50-page PDF (corpus) might be split into 200 documents (chunks), all
 - **Metadata Management**: JSON-based flexible metadata with GIN indexing
 - **Collection Support**: Organize documents into logical collections
 - **Chunk Management**: Handle long content by chunking into retrievable documents
+- **Smart Upserts**: Re-inserting a corpus only re-embeds chunks whose content changed, and skips the write entirely if nothing did
 - **Database Abstraction**: Clean SQLAlchemy-based database layer with schema creation API
 - **Type Safety**: Full Pydantic validation and type hints
 - **Production Ready**: Comprehensive testing and error handling
@@ -144,34 +141,14 @@ CREATE EXTENSION IF NOT EXISTS vector;
 3. **Set up your connection:**
 
 ```python
-from sqlalchemy import create_engine
 from pgvector_template import DatabaseManager
 
-# Option 1: Direct connection string
-engine = create_engine("postgresql://user:password@localhost:5432/mydb")
-db_manager = DatabaseManager(engine)
-
-# Option 2: From environment variable
-import os
-
-engine = create_engine(os.getenv("DATABASE_URL"))
-db_manager = DatabaseManager(engine)
+db_manager = DatabaseManager("postgresql://user:password@localhost:5432/mydb")
+db_manager.initialize()  # connects and enables the vector extension
 ```
 
-### Production Configuration
-
-```python
-from sqlalchemy import create_engine
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    "postgresql://user:password@localhost:5432/mydb",
-    poolclass=QueuePool,
-    pool_size=10,
-    max_overflow=20,
-    pool_pre_ping=True,
-)
-```
+`create_tables` also adds nullable model columns that are missing from existing tables (for example `content_hash`).
+Only the column name and type are applied: indexes, foreign keys, unique constraints and server defaults are not. This is a stopgap until proper migrations are added.
 
 ### Environment Variables
 
@@ -198,70 +175,67 @@ class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 ```
 
-### 2. Insert Documents
+### 2. Insert a Corpus
 
 ```python
 from pgvector_template.core.document import BaseDocumentOptionalProps
-from uuid import uuid4
 
-# Create document with metadata
-metadata = MyDocumentMetadata(source_type="pdf", author="John Doe")
 optional_props = BaseDocumentOptionalProps(
     title="Chapter 1: Introduction",
     collection="textbooks",
     language="en",
-    tags=["education", "intro"],
 )
 
-doc = MyDocument.from_props(
-    corpus_id=uuid4(),
-    chunk_index=0,
-    content="This is the document content...",
-    embedding=your_embedding_vector,  # list[float] with 1024 dimensions
-    metadata=metadata.to_dict(),
-    optional_props=optional_props,
-)
+with db.get_session() as session:
+    service = DocumentService(session, config)
+    result = service.corpus_manager.insert_corpus(
+        "This is the document content...",
+        {"source_type": "pdf", "author": "John Doe"},  # corpus metadata
+        optional_props,
+        corpus_id="intro-chapter",
+    )
+```
+
+To insert chunks you have already split and embedded, use `insert_documents()`.
+
+Calling `insert_corpus()` again with the same `corpus_id` replaces the corpus. Unchanged chunks reuse their stored vector, and the write is skipped if nothing changed:
 
-with db_manager.get_session() as session:
-    doc_manager.insert_document(session, doc)
+```python
+result = corpus_manager.insert_corpus(text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=12, reused=0, total=12)
+
+result = corpus_manager.insert_corpus(edited_text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=1, reused=11, total=12)
 ```
 
+Rows written before `content_hash` existed have no hash, so their first upsert re-embeds them.
+
 ### 3. Search Documents
 
 ```python
-# Basic similarity search
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session=session, document_cls=MyDocument, query_embedding=query_vector, limit=10
-    )
-
-# Search with filters
-from pgvector_template.models.search import MetadataFilter
+from pgvector_template.models.search import MetadataFilter, SearchQuery
 
-filters = [
-    MetadataFilter(key="source_type", value="pdf"),
-    MetadataFilter(key="author", value="John Doe"),
-]
+# Semantic search
+results = service.search_client.search(SearchQuery(text="how do I reset my password", limit=10))
 
-results = doc_manager.search_similar(
-    session=session,
-    document_cls=MyDocument,
-    query_embedding=query_vector,
-    limit=10,
-    metadata_filters=filters,
-    collection="textbooks",
+# Keyword search combined with a metadata filter
+results = service.search_client.search(
+    SearchQuery(
+        keywords=["password", "reset"],
+        metadata_filters=[MetadataFilter(field_name="source_type", condition="eq", value="pdf")],
+        limit=10,
+    )
 )
+
+for r in results:
+    print(r.score, r.document.content)
 ```
 
-### 4. Manage Collections
+### 4. Read a Corpus Back
 
 ```python
-# Get all documents in a collection
-with db_manager.get_session() as session:
-    docs = doc_manager.get_documents_by_collection(session, MyDocument, "textbooks")
-
-    # Get all chunks from a corpus
-    corpus_docs = doc_manager.get_documents_by_corpus_id(session, MyDocument, corpus_id)
+corpus = service.corpus_manager.get_full_corpus("intro-chapter")
+print(corpus.content, corpus.metadata, len(corpus.documents))
 ```
 
 ## Concept Reference
@@ -271,8 +245,11 @@ with db_manager.get_session() as session:
 - **`BaseDocument`**: Abstract SQLAlchemy model for documents with vector embeddings
 - **`BaseDocumentOptionalProps`**: Pydantic model for optional document properties
 - **`BaseDocumentMetadata`**: Base schema for structured document metadata
-- **`DatabaseManager`**: Database connection and session management
-- **`DocumentDatabaseManager`**: High-level document CRUD operations
+- **`BaseEmbeddingProvider`**: Interface to implement for your embedding model
+- **`BaseCorpusManager`**: Splits, embeds and upserts corpora (`insert_corpus`, `get_full_corpus`)
+- **`BaseSearchClient`**: Semantic, keyword and metadata search (`search`)
+- **`DocumentService`**: Bundles a corpus manager and a search client over one session
+- **`DatabaseManager`** / **`DocumentDatabaseManager`**: Connection, schema and table setup
 
 ## Testing
 
@@ -306,7 +283,8 @@ python -m unittest discover -s integ-tests
 
 ```bash
 pip install -e .[dev,test]
-black .  # Format code
+ruff check . && ruff format .
+ty check .
 ```
 
 ## License

{pgvector_template-0.5.1 → pgvector_template-0.6}/README.md

@@ -9,10 +9,10 @@ PGVector-Template provides a robust foundation for implementing vector-based doc
 ## Quick Start
 
 ```python
-from pgvector_template import DatabaseManager, DocumentDatabaseManager
+from pgvector_template import DocumentDatabaseManager
 from pgvector_template.core.document import BaseDocument
-from sqlalchemy import create_engine
-from uuid import uuid4
+from pgvector_template.models.search import SearchQuery
+from pgvector_template.service.document_service import DocumentService, DocumentServiceConfig
 
 
 # 1. Define your document model
@@ -20,29 +20,25 @@ class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 
 
-# 2. Set up database connection
-engine = create_engine("postgresql://user:pass@localhost/mydb")
-db_manager = DatabaseManager(engine)
-db_manager.create_tables([MyDocument])
+# 2. Set up the database: connects, enables pgvector, creates the schema and tables
+db = DocumentDatabaseManager(
+    database_url="postgresql://user:pass@localhost/mydb",
+    schema_suffix="my_kb",
+    document_classes=[MyDocument],
+)
+db.setup()
 
-# 3. Create document manager
-doc_manager = DocumentDatabaseManager(db_manager)
+# 3. Create a service. `my_embedding_provider` is your `BaseEmbeddingProvider` subclass
+config = DocumentServiceConfig(document_cls=MyDocument, embedding_provider=my_embedding_provider)
 
-# 4. Insert a document
-with db_manager.get_session() as session:
-    doc = MyDocument.from_props(
-        corpus_id=uuid4(),
-        chunk_index=0,
-        content="Your document content here",
-        embedding=[0.1, 0.2, 0.3, ...],  # Your embedding vector
-    )
-    doc_manager.insert_document(session, doc)
+with db.get_session() as session:
+    service = DocumentService(session, config)
 
-# 5. Search similar documents
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session, MyDocument, query_embedding=[0.1, 0.2, 0.3, ...], limit=5
-    )
+    # 4. Insert a corpus: it is split into chunks, embedded and stored
+    service.corpus_manager.insert_corpus("Your document content here", {}, corpus_id="doc-1")
+
+    # 5. Search
+    results = service.search_client.search(SearchQuery(text="your query", limit=5))
 ```
 
 ## Key Concepts
@@ -61,6 +57,7 @@ Example: A 50-page PDF (corpus) might be split into 200 documents (chunks), all
 - **Metadata Management**: JSON-based flexible metadata with GIN indexing
 - **Collection Support**: Organize documents into logical collections
 - **Chunk Management**: Handle long content by chunking into retrievable documents
+- **Smart Upserts**: Re-inserting a corpus only re-embeds chunks whose content changed, and skips the write entirely if nothing did
 - **Database Abstraction**: Clean SQLAlchemy-based database layer with schema creation API
 - **Type Safety**: Full Pydantic validation and type hints
 - **Production Ready**: Comprehensive testing and error handling
@@ -114,34 +111,14 @@ CREATE EXTENSION IF NOT EXISTS vector;
 3. **Set up your connection:**
 
 ```python
-from sqlalchemy import create_engine
 from pgvector_template import DatabaseManager
 
-# Option 1: Direct connection string
-engine = create_engine("postgresql://user:password@localhost:5432/mydb")
-db_manager = DatabaseManager(engine)
-
-# Option 2: From environment variable
-import os
-
-engine = create_engine(os.getenv("DATABASE_URL"))
-db_manager = DatabaseManager(engine)
+db_manager = DatabaseManager("postgresql://user:password@localhost:5432/mydb")
+db_manager.initialize()  # connects and enables the vector extension
 ```
 
-### Production Configuration
-
-```python
-from sqlalchemy import create_engine
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    "postgresql://user:password@localhost:5432/mydb",
-    poolclass=QueuePool,
-    pool_size=10,
-    max_overflow=20,
-    pool_pre_ping=True,
-)
-```
+`create_tables` also adds nullable model columns that are missing from existing tables (for example `content_hash`).
+Only the column name and type are applied: indexes, foreign keys, unique constraints and server defaults are not. This is a stopgap until proper migrations are added.
 
 ### Environment Variables
 
@@ -168,70 +145,67 @@ class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 ```
 
-### 2. Insert Documents
+### 2. Insert a Corpus
 
 ```python
 from pgvector_template.core.document import BaseDocumentOptionalProps
-from uuid import uuid4
 
-# Create document with metadata
-metadata = MyDocumentMetadata(source_type="pdf", author="John Doe")
 optional_props = BaseDocumentOptionalProps(
     title="Chapter 1: Introduction",
     collection="textbooks",
     language="en",
-    tags=["education", "intro"],
 )
 
-doc = MyDocument.from_props(
-    corpus_id=uuid4(),
-    chunk_index=0,
-    content="This is the document content...",
-    embedding=your_embedding_vector,  # list[float] with 1024 dimensions
-    metadata=metadata.to_dict(),
-    optional_props=optional_props,
-)
+with db.get_session() as session:
+    service = DocumentService(session, config)
+    result = service.corpus_manager.insert_corpus(
+        "This is the document content...",
+        {"source_type": "pdf", "author": "John Doe"},  # corpus metadata
+        optional_props,
+        corpus_id="intro-chapter",
+    )
+```
+
+To insert chunks you have already split and embedded, use `insert_documents()`.
+
+Calling `insert_corpus()` again with the same `corpus_id` replaces the corpus. Unchanged chunks reuse their stored vector, and the write is skipped if nothing changed:
 
-with db_manager.get_session() as session:
-    doc_manager.insert_document(session, doc)
+```python
+result = corpus_manager.insert_corpus(text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=12, reused=0, total=12)
+
+result = corpus_manager.insert_corpus(edited_text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=1, reused=11, total=12)
 ```
 
+Rows written before `content_hash` existed have no hash, so their first upsert re-embeds them.
+
 ### 3. Search Documents
 
 ```python
-# Basic similarity search
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session=session, document_cls=MyDocument, query_embedding=query_vector, limit=10
-    )
-
-# Search with filters
-from pgvector_template.models.search import MetadataFilter
+from pgvector_template.models.search import MetadataFilter, SearchQuery
 
-filters = [
-    MetadataFilter(key="source_type", value="pdf"),
-    MetadataFilter(key="author", value="John Doe"),
-]
+# Semantic search
+results = service.search_client.search(SearchQuery(text="how do I reset my password", limit=10))
 
-results = doc_manager.search_similar(
-    session=session,
-    document_cls=MyDocument,
-    query_embedding=query_vector,
-    limit=10,
-    metadata_filters=filters,
-    collection="textbooks",
+# Keyword search combined with a metadata filter
+results = service.search_client.search(
+    SearchQuery(
+        keywords=["password", "reset"],
+        metadata_filters=[MetadataFilter(field_name="source_type", condition="eq", value="pdf")],
+        limit=10,
+    )
 )
+
+for r in results:
+    print(r.score, r.document.content)
 ```
 
-### 4. Manage Collections
+### 4. Read a Corpus Back
 
 ```python
-# Get all documents in a collection
-with db_manager.get_session() as session:
-    docs = doc_manager.get_documents_by_collection(session, MyDocument, "textbooks")
-
-    # Get all chunks from a corpus
-    corpus_docs = doc_manager.get_documents_by_corpus_id(session, MyDocument, corpus_id)
+corpus = service.corpus_manager.get_full_corpus("intro-chapter")
+print(corpus.content, corpus.metadata, len(corpus.documents))
 ```
 
 ## Concept Reference
@@ -241,8 +215,11 @@ with db_manager.get_session() as session:
 - **`BaseDocument`**: Abstract SQLAlchemy model for documents with vector embeddings
 - **`BaseDocumentOptionalProps`**: Pydantic model for optional document properties
 - **`BaseDocumentMetadata`**: Base schema for structured document metadata
-- **`DatabaseManager`**: Database connection and session management
-- **`DocumentDatabaseManager`**: High-level document CRUD operations
+- **`BaseEmbeddingProvider`**: Interface to implement for your embedding model
+- **`BaseCorpusManager`**: Splits, embeds and upserts corpora (`insert_corpus`, `get_full_corpus`)
+- **`BaseSearchClient`**: Semantic, keyword and metadata search (`search`)
+- **`DocumentService`**: Bundles a corpus manager and a search client over one session
+- **`DatabaseManager`** / **`DocumentDatabaseManager`**: Connection, schema and table setup
 
 ## Testing
 
@@ -276,7 +253,8 @@ python -m unittest discover -s integ-tests
 
 ```bash
 pip install -e .[dev,test]
-black .  # Format code
+ruff check . && ruff format .
+ty check .
 ```
 
 ## License

{pgvector_template-0.5.1 → pgvector_template-0.6}/pgvector_template/core/document.py

@@ -1,3 +1,5 @@
+import hashlib
+import json
 from datetime import UTC, datetime
 from typing import Any, Self
 from uuid import UUID as UuidLiteral
@@ -86,6 +88,12 @@ class BaseDocument(Base):
     Use this field for auditing and diagnosing mixed-model corpora, not for runtime dispatch.
     """
 
+    content_hash = Column(String(64), nullable=True)
+    """
+    Hex sha256 of `content` + `embedding_config`, see `compute_content_hash()`.
+    Lets upserts reuse a stored embedding when a chunk is unchanged. NULL for rows that pre-date this column.
+    """
+
     # Audit fields
     created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
     updated_at = Column(
@@ -165,6 +173,7 @@ class BaseDocument(Base):
             content=content,
             embedding=embedding,
             embedding_config=embedding_config,
+            content_hash=cls.compute_content_hash(content, embedding_config),
             title=optional_props.title,
             document_metadata=metadata or {},
             collection=optional_props.collection,
@@ -172,6 +181,15 @@ class BaseDocument(Base):
             language=optional_props.language,
         )
 
+    @classmethod
+    def compute_content_hash(cls, content: str, embedding_config: dict[str, Any] | None) -> str:
+        """
+        Hash of everything that determines a chunk's embedding vector. Metadata and other props are excluded.
+        Override to change what counts as "the same chunk".
+        """
+        payload = json.dumps([content, embedding_config], sort_keys=True, ensure_ascii=False)
+        return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
     @classmethod
     def get_embedding_index(cls, table_name: str) -> Index:
         """Override this method to customize the embedding index."""

{pgvector_template-0.5.1 → pgvector_template-0.6}/pgvector_template/core/manager.py

@@ -1,11 +1,13 @@
+import hashlib
+import json
 from abc import ABC
 from dataclasses import dataclass
 from logging import getLogger
-from typing import Any
+from typing import Any, cast
 from uuid import UUID, uuid4
 
 from pydantic import BaseModel, Field
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, load_only
 
 from pgvector_template.core.document import (
     BaseDocument,
@@ -45,6 +47,22 @@ class Corpus:
     documents: list[BaseDocument]
 
 
+@dataclass(frozen=True)
+class UpsertResult:
+    """Outcome of `BaseCorpusManager.insert_corpus()`."""
+
+    corpus_hash: str
+    """Hex sha256 over the ordered chunk hashes. Computed in memory, not stored."""
+    skipped: bool
+    """True when the stored corpus already matched, so nothing was embedded or written."""
+    embedded: int
+    """Chunks sent to the embedding provider."""
+    reused: int
+    """Chunks whose stored vector was reused."""
+    total: int
+    """Number of chunks in the corpus."""
+
+
 class BaseCorpusManager(ABC):
     """
     Template class for `Corpus` & `Document` management operations.
@@ -121,31 +139,152 @@ class BaseCorpusManager(ABC):
         corpus_id: UUID | str | None = None,
         update_if_exists: bool = True,
         **kwargs,
-    ) -> int:
+    ) -> UpsertResult:
         """
         Insert a new `Corpus`, which will be split into 1-or-more `Document`s, depending on its length.
         Each `Document` chunk shall have its own embedding vector, but reference the parent corpus_id.
 
+        When `update_if_exists` is set and the corpus already exists, chunks whose content hash is
+        unchanged reuse their stored vector, and only the rest are embedded. If the stored rows already
+        match the new ones entirely, nothing is written.
+
         Args:
             content: The text content to be inserted as a corpus
             corpus_metadata: Dictionary of metadata associated with the corpus
             optional_props: Optional properties for the documents (title, collection, etc.)
 
         Returns:
-            int: The number of **documents** inserted for the provided corpus
+            UpsertResult: Whether the write was skipped, and how many chunks were embedded vs reused
         """
         corpus_id = self._generate_corpus_id(corpus_id)
         document_contents = self._split_corpus(content)
-        document_embeddings = self.embedding_provider.embed_batch(document_contents)
-        return self.insert_documents(
-            corpus_id,
-            document_contents,
-            document_embeddings,
-            corpus_metadata,
-            optional_props,
-            update_if_exists,
+        if not document_contents:
+            return UpsertResult(self._corpus_hash([]), False, 0, 0, 0)
+
+        total = len(document_contents)
+        # vectors are filled in by `_attach_embeddings`
+        documents = self._create_documents(
+            corpus_id, document_contents, [[]] * total, corpus_metadata, optional_props
+        )
+        chunk_hashes = [cast(str, d.content_hash) for d in documents]
+        corpus_hash = self._corpus_hash(chunk_hashes)
+
+        reusable: dict[str, Any] = {}
+        if update_if_exists and (existing := self._load_existing_chunks(corpus_id)):
+            if self._is_unchanged(corpus_id, existing, documents):
+                logger.info(f"Corpus {corpus_id} unchanged, skipping ({total} chunks)")
+                self.session.commit()  # release the read transaction
+                return UpsertResult(corpus_hash, True, 0, total, total)
+            reusable = self._load_reusable_vectors(corpus_id, set(chunk_hashes))
+
+        embedded = self._attach_embeddings(documents, document_contents, chunk_hashes, reusable)
+        self._replace_corpus(corpus_id, documents, delete_existing=update_if_exists)
+        return UpsertResult(corpus_hash, False, embedded, total - embedded, total)
+
+    def _attach_embeddings(
+        self,
+        documents: list[BaseDocument],
+        document_contents: list[str],
+        chunk_hashes: list[str],
+        reusable: dict[str, Any],
+    ) -> int:
+        """Set each document's vector from `reusable` or a fresh embedding. Returns the number embedded."""
+        misses = [i for i, h in enumerate(chunk_hashes) if h not in reusable]
+        miss_contents = [document_contents[i] for i in misses]
+        miss_embeddings: list[list[float]] = []
+        if misses:
+            # end the read transaction so the connection is not held idle during the remote call
+            self.session.commit()
+            miss_embeddings = self.embedding_provider.embed_batch(miss_contents)
+        self._validate_inputs(miss_contents, miss_embeddings)
+        fresh = dict(zip(misses, miss_embeddings))
+        for i, doc in enumerate(documents):
+            doc.embedding = fresh[i] if i in fresh else reusable[chunk_hashes[i]]  # ty: ignore[invalid-assignment]
+        return len(misses)
+
+    @staticmethod
+    def _corpus_hash(chunk_hashes: list[str]) -> str:
+        return hashlib.sha256(json.dumps(chunk_hashes).encode("utf-8")).hexdigest()
+
+    @staticmethod
+    def _fingerprint(doc: BaseDocument) -> tuple:
+        """Everything besides the vector that an upsert must keep in sync for a chunk."""
+        return (
+            doc.chunk_index,
+            doc.content_hash,
+            doc.document_metadata,
+            doc.title,
+            doc.collection,
+            doc.origin_url,
+            doc.language,
         )
 
+    def _load_existing_chunks(self, corpus_id: UUID | str) -> list[BaseDocument]:
+        """Stored chunks of a corpus ordered by `chunk_index`, without content or vectors."""
+        cls = self.config.document_cls
+        return (
+            self.session.query(cls)
+            .options(
+                load_only(
+                    cls.chunk_index,
+                    cls.content_hash,
+                    cls.document_metadata,
+                    cls.title,
+                    cls.collection,
+                    cls.origin_url,
+                    cls.language,
+                    cls.is_deleted,
+                )
+            )
+            .filter(cls.corpus_id == corpus_id)
+            .order_by(cls.chunk_index)
+            .all()
+        )
+
+    def _is_unchanged(
+        self, corpus_id: UUID | str, existing: list[BaseDocument], new: list[BaseDocument]
+    ) -> bool:
+        """True if the stored rows match the new ones and every stored row has a vector."""
+        if any(d.is_deleted for d in existing):
+            return False
+        if [self._fingerprint(d) for d in existing] != [self._fingerprint(d) for d in new]:
+            return False
+        return not self._has_missing_vectors(corpus_id)
+
+    def _has_missing_vectors(self, corpus_id: UUID | str) -> bool:
+        cls = self.config.document_cls
+        query = self.session.query(cls.id).filter(
+            cls.corpus_id == corpus_id, cls.embedding.is_(None)
+        )
+        return query.first() is not None
+
+    def _load_reusable_vectors(self, corpus_id: UUID | str, hashes: set[str]) -> dict[str, Any]:
+        """Map `content_hash -> embedding` for stored chunks of this corpus whose hash is in `hashes`."""
+        cls = self.config.document_cls
+        rows = (
+            self.session.query(cls.content_hash, cls.embedding)
+            .filter(
+                cls.corpus_id == corpus_id,
+                cls.content_hash.in_(hashes),
+                cls.embedding.is_not(None),
+            )
+            .all()
+        )
+        return dict(rows)
+
+    def _replace_corpus(
+        self, corpus_id: UUID | str, documents: list[BaseDocument], delete_existing: bool
+    ) -> None:
+        """Delete (optionally) and insert in a single transaction."""
+        try:
+            if delete_existing:
+                self._delete_existing_corpus(corpus_id)
+            self.session.add_all(documents)
+            self.session.commit()
+        except Exception:
+            self.session.rollback()
+            raise
+
     def insert_documents(
         self,
         corpus_id: UUID | str,
@@ -180,11 +319,7 @@ class BaseCorpusManager(ABC):
             corpus_id, document_contents, document_embeddings, corpus_metadata, optional_props
         )
 
-        if update_if_exists:
-            self._delete_existing_corpus(corpus_id)
-
-        self.session.add_all(documents_to_insert)
-        self.session.commit()
+        self._replace_corpus(corpus_id, documents_to_insert, delete_existing=update_if_exists)
         return len(documents_to_insert)
 
     def _validate_inputs(

{pgvector_template-0.5.1 → pgvector_template-0.6}/pgvector_template/db/connection.py

@@ -2,7 +2,7 @@ from collections.abc import Generator
 from contextlib import contextmanager
 from logging import getLogger
 
-from sqlalchemy import Engine, create_engine, text
+from sqlalchemy import Engine, create_engine, inspect, text
 from sqlalchemy.ext.declarative import DeclarativeMeta
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -46,8 +46,40 @@ class DatabaseManager:
         for table in base_class.metadata.tables.values():
             table.schema = schema_name
         base_class.metadata.create_all(self._require_engine(), checkfirst=True)
+        self._add_missing_columns(base_class, schema_name)
         self.logger.info(f"Created tables for schema: {schema_name}")
 
+    def _add_missing_columns(self, base_class: type[DeclarativeMeta], schema_name: str) -> None:
+        """
+        `create_all` never alters existing tables. Add nullable columns that are defined on the model but
+        missing from the database. Non-nullable columns can't be added safely to populated tables; they are skipped.
+        DDL is only issued for columns that are actually missing, so no table ownership is needed otherwise.
+        """
+        tables = list(base_class.metadata.tables.values())
+        if not tables:
+            return
+        engine = self._require_engine()
+        inspector = inspect(engine)
+        preparer = engine.dialect.identifier_preparer
+        for table in tables:
+            existing = {c["name"] for c in inspector.get_columns(table.name, schema=schema_name)}
+            for column in table.columns:
+                if column.name in existing:
+                    continue
+                if not column.nullable:
+                    self.logger.warning(
+                        f"Column {table.name}.{column.name} is missing and non-nullable; add it manually"
+                    )
+                    continue
+                ddl = (
+                    f"ALTER TABLE {preparer.quote_schema(schema_name)}.{preparer.quote(table.name)} "
+                    f"ADD COLUMN IF NOT EXISTS {preparer.quote(column.name)} "
+                    f"{column.type.compile(engine.dialect)}"
+                )
+                with engine.begin() as conn:
+                    conn.execute(text(ddl))
+                self.logger.info(f"Added column {schema_name}.{table.name}.{column.name}")
+
     def _ensure_pgvector_extension(self):
         """Ensure pgvector extension is available"""
         with self._require_engine().connect() as conn:

{pgvector_template-0.5.1 → pgvector_template-0.6}/pgvector_template.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-template
-Version: 0.5.1
+Version: 0.6
 Summary: Template library for flexible PGVector RAG implementations
 Author-email: DL <v49t9zpqd@mozmail.com>
 License: MIT
@@ -39,10 +39,10 @@ PGVector-Template provides a robust foundation for implementing vector-based doc
 ## Quick Start
 
 ```python
-from pgvector_template import DatabaseManager, DocumentDatabaseManager
+from pgvector_template import DocumentDatabaseManager
 from pgvector_template.core.document import BaseDocument
-from sqlalchemy import create_engine
-from uuid import uuid4
+from pgvector_template.models.search import SearchQuery
+from pgvector_template.service.document_service import DocumentService, DocumentServiceConfig
 
 
 # 1. Define your document model
@@ -50,29 +50,25 @@ class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 
 
-# 2. Set up database connection
-engine = create_engine("postgresql://user:pass@localhost/mydb")
-db_manager = DatabaseManager(engine)
-db_manager.create_tables([MyDocument])
+# 2. Set up the database: connects, enables pgvector, creates the schema and tables
+db = DocumentDatabaseManager(
+    database_url="postgresql://user:pass@localhost/mydb",
+    schema_suffix="my_kb",
+    document_classes=[MyDocument],
+)
+db.setup()
 
-# 3. Create document manager
-doc_manager = DocumentDatabaseManager(db_manager)
+# 3. Create a service. `my_embedding_provider` is your `BaseEmbeddingProvider` subclass
+config = DocumentServiceConfig(document_cls=MyDocument, embedding_provider=my_embedding_provider)
 
-# 4. Insert a document
-with db_manager.get_session() as session:
-    doc = MyDocument.from_props(
-        corpus_id=uuid4(),
-        chunk_index=0,
-        content="Your document content here",
-        embedding=[0.1, 0.2, 0.3, ...],  # Your embedding vector
-    )
-    doc_manager.insert_document(session, doc)
+with db.get_session() as session:
+    service = DocumentService(session, config)
 
-# 5. Search similar documents
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session, MyDocument, query_embedding=[0.1, 0.2, 0.3, ...], limit=5
-    )
+    # 4. Insert a corpus: it is split into chunks, embedded and stored
+    service.corpus_manager.insert_corpus("Your document content here", {}, corpus_id="doc-1")
+
+    # 5. Search
+    results = service.search_client.search(SearchQuery(text="your query", limit=5))
 ```
 
 ## Key Concepts
@@ -91,6 +87,7 @@ Example: A 50-page PDF (corpus) might be split into 200 documents (chunks), all
 - **Metadata Management**: JSON-based flexible metadata with GIN indexing
 - **Collection Support**: Organize documents into logical collections
 - **Chunk Management**: Handle long content by chunking into retrievable documents
+- **Smart Upserts**: Re-inserting a corpus only re-embeds chunks whose content changed, and skips the write entirely if nothing did
 - **Database Abstraction**: Clean SQLAlchemy-based database layer with schema creation API
 - **Type Safety**: Full Pydantic validation and type hints
 - **Production Ready**: Comprehensive testing and error handling
@@ -144,34 +141,14 @@ CREATE EXTENSION IF NOT EXISTS vector;
 3. **Set up your connection:**
 
 ```python
-from sqlalchemy import create_engine
 from pgvector_template import DatabaseManager
 
-# Option 1: Direct connection string
-engine = create_engine("postgresql://user:password@localhost:5432/mydb")
-db_manager = DatabaseManager(engine)
-
-# Option 2: From environment variable
-import os
-
-engine = create_engine(os.getenv("DATABASE_URL"))
-db_manager = DatabaseManager(engine)
+db_manager = DatabaseManager("postgresql://user:password@localhost:5432/mydb")
+db_manager.initialize()  # connects and enables the vector extension
 ```
 
-### Production Configuration
-
-```python
-from sqlalchemy import create_engine
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    "postgresql://user:password@localhost:5432/mydb",
-    poolclass=QueuePool,
-    pool_size=10,
-    max_overflow=20,
-    pool_pre_ping=True,
-)
-```
+`create_tables` also adds nullable model columns that are missing from existing tables (for example `content_hash`).
+Only the column name and type are applied: indexes, foreign keys, unique constraints and server defaults are not. This is a stopgap until proper migrations are added.
 
 ### Environment Variables
 
@@ -198,70 +175,67 @@ class MyDocument(BaseDocument):
     __tablename__ = "my_documents"
 ```
 
-### 2. Insert Documents
+### 2. Insert a Corpus
 
 ```python
 from pgvector_template.core.document import BaseDocumentOptionalProps
-from uuid import uuid4
 
-# Create document with metadata
-metadata = MyDocumentMetadata(source_type="pdf", author="John Doe")
 optional_props = BaseDocumentOptionalProps(
     title="Chapter 1: Introduction",
     collection="textbooks",
     language="en",
-    tags=["education", "intro"],
 )
 
-doc = MyDocument.from_props(
-    corpus_id=uuid4(),
-    chunk_index=0,
-    content="This is the document content...",
-    embedding=your_embedding_vector,  # list[float] with 1024 dimensions
-    metadata=metadata.to_dict(),
-    optional_props=optional_props,
-)
+with db.get_session() as session:
+    service = DocumentService(session, config)
+    result = service.corpus_manager.insert_corpus(
+        "This is the document content...",
+        {"source_type": "pdf", "author": "John Doe"},  # corpus metadata
+        optional_props,
+        corpus_id="intro-chapter",
+    )
+```
+
+To insert chunks you have already split and embedded, use `insert_documents()`.
+
+Calling `insert_corpus()` again with the same `corpus_id` replaces the corpus. Unchanged chunks reuse their stored vector, and the write is skipped if nothing changed:
 
-with db_manager.get_session() as session:
-    doc_manager.insert_document(session, doc)
+```python
+result = corpus_manager.insert_corpus(text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=12, reused=0, total=12)
+
+result = corpus_manager.insert_corpus(edited_text, {"source": "pdf"}, corpus_id="handbook")
+# UpsertResult(corpus_hash=..., skipped=False, embedded=1, reused=11, total=12)
 ```
 
+Rows written before `content_hash` existed have no hash, so their first upsert re-embeds them.
+
 ### 3. Search Documents
 
 ```python
-# Basic similarity search
-with db_manager.get_session() as session:
-    results = doc_manager.search_similar(
-        session=session, document_cls=MyDocument, query_embedding=query_vector, limit=10
-    )
-
-# Search with filters
-from pgvector_template.models.search import MetadataFilter
+from pgvector_template.models.search import MetadataFilter, SearchQuery
 
-filters = [
-    MetadataFilter(key="source_type", value="pdf"),
-    MetadataFilter(key="author", value="John Doe"),
-]
+# Semantic search
+results = service.search_client.search(SearchQuery(text="how do I reset my password", limit=10))
 
-results = doc_manager.search_similar(
-    session=session,
-    document_cls=MyDocument,
-    query_embedding=query_vector,
-    limit=10,
-    metadata_filters=filters,
-    collection="textbooks",
+# Keyword search combined with a metadata filter
+results = service.search_client.search(
+    SearchQuery(
+        keywords=["password", "reset"],
+        metadata_filters=[MetadataFilter(field_name="source_type", condition="eq", value="pdf")],
+        limit=10,
+    )
 )
+
+for r in results:
+    print(r.score, r.document.content)
 ```
 
-### 4. Manage Collections
+### 4. Read a Corpus Back
 
 ```python
-# Get all documents in a collection
-with db_manager.get_session() as session:
-    docs = doc_manager.get_documents_by_collection(session, MyDocument, "textbooks")
-
-    # Get all chunks from a corpus
-    corpus_docs = doc_manager.get_documents_by_corpus_id(session, MyDocument, corpus_id)
+corpus = service.corpus_manager.get_full_corpus("intro-chapter")
+print(corpus.content, corpus.metadata, len(corpus.documents))
 ```
 
 ## Concept Reference
@@ -271,8 +245,11 @@ with db_manager.get_session() as session:
 - **`BaseDocument`**: Abstract SQLAlchemy model for documents with vector embeddings
 - **`BaseDocumentOptionalProps`**: Pydantic model for optional document properties
 - **`BaseDocumentMetadata`**: Base schema for structured document metadata
-- **`DatabaseManager`**: Database connection and session management
-- **`DocumentDatabaseManager`**: High-level document CRUD operations
+- **`BaseEmbeddingProvider`**: Interface to implement for your embedding model
+- **`BaseCorpusManager`**: Splits, embeds and upserts corpora (`insert_corpus`, `get_full_corpus`)
+- **`BaseSearchClient`**: Semantic, keyword and metadata search (`search`)
+- **`DocumentService`**: Bundles a corpus manager and a search client over one session
+- **`DatabaseManager`** / **`DocumentDatabaseManager`**: Connection, schema and table setup
 
 ## Testing
 
@@ -306,7 +283,8 @@ python -m unittest discover -s integ-tests
 
 ```bash
 pip install -e .[dev,test]
-black .  # Format code
+ruff check . && ruff format .
+ty check .
 ```
 
 ## License

{pgvector_template-0.5.1 → pgvector_template-0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pgvector-template"
-version = "0.5.1"
+version = "0.6"
 description = "Template library for flexible PGVector RAG implementations"
 authors = [{ name="DL", email="v49t9zpqd@mozmail.com" }]
 license = { text = "MIT" }