hammad-python 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

hammad/data/collections/indexes/tantivy/index.py

@@ -0,0 +1,428 @@
+"""hammad.data.collections.indexes.tantivy.index"""
+
+from datetime import datetime, timezone, timedelta
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Type,
+    final
+)
+import uuid
+from pathlib import Path
+import json
+
+import tantivy
+
+from ....sql.types import (
+    DatabaseItemType,
+    DatabaseItemFilters,
+    DatabaseItem,
+)
+from ....sql.database import Database
+from . import utils
+from .settings import (
+    TantivyCollectionIndexSettings,
+    TantivyCollectionIndexQuerySettings
+)
+
+
+@final
+class TantivyCollectionIndex(Generic[DatabaseItemType]):
+    """A standalone (simplified) index that can be used as the
+    storage / search engine for a collection, that implements
+    fast indexing & querying capabilities using the
+    `tantivy` package.
+    
+    This collection index is built into the core dependencies
+    of the `hammad-python` package, and is the default index
+    used by the `Collection` class."""
+
+    def __init__(
+        self,
+        *,
+        name: str = "default",
+        schema: Optional[Type[DatabaseItemType]] = None,
+        ttl: Optional[int] = None,
+        path: Optional[Path | str] = None,
+        fast: bool = True,
+        settings: Optional[TantivyCollectionIndexSettings] = None,
+        query_settings: Optional[TantivyCollectionIndexQuerySettings] = None,
+    ) -> None:
+        """Initialize a new `TantivyCollectionIndex` with a given set
+        of parameters. 
+        
+        Args:
+            name: The name of the index.
+            schema: The schema of the items that can be stored
+                within this index.
+            ttl: The time to live for the items within this index.
+            path: The path to the directory where the index will be stored.
+                (If not provided, the collection will be built on memory. This is how to 
+                distinguish between different collection locations.)
+            fast: Whether to use fast schema building & indexing
+                from `tantivy`'s builtin implementation.
+            settings: Default settings to use for indexing & schema
+                building.
+            query_settings: Default settings to use for the query
+                engine.
+        """
+        self.name = name
+        self.schema = schema
+        self.ttl = ttl
+
+        if path is not None and not isinstance(path, Path):
+            path = Path(path)
+
+        self.path = path
+        """The file path to the collection index.
+
+        (You wouldnt know), but earlier versions of this package allowed
+        for implementing `databases` with file system paths. The new
+        structure of the package does not implement the `Database` class
+        anymore, and rather allows for creating custom extensions using
+        collections directly.
+        
+        Ex: `/database/collection.db | /database/collection.myextension`"""
+
+        if not settings:
+            settings = TantivyCollectionIndexSettings(
+                fast=fast,
+            )
+
+        if not query_settings:
+            query_settings = TantivyCollectionIndexQuerySettings()
+
+        self.settings = settings
+        """The default settings to use when indexing and schema building
+        for this index."""
+
+        self.query_settings = query_settings
+        """The default settings to use when querying this index."""
+
+        # Initialize SQL Database as storage backend
+        database_path = None
+        if self.path is not None:
+            database_path = self.path / f"{name}.db"
+        
+        self._database = Database[DatabaseItemType](
+            name=name,
+            schema=schema,
+            ttl=ttl,
+            path=database_path,
+            table_name=f"tantivy_{name}"
+        )
+
+        try:
+            self._tantivy_wrapper = utils.build_tantivy_index_from_settings(
+                settings=settings
+            )
+            self._index = self._tantivy_wrapper.index
+            self._schema = self._tantivy_wrapper.schema
+            self._writer = self._tantivy_wrapper.index_writer
+        except Exception as e:
+            raise utils.TantivyCollectionIndexError(
+                f"Failed to build tantivy index from settings: {e}"
+            ) from e
+
+    def add(
+        self,
+        item: DatabaseItemType,
+        *,
+        id: Optional[str] = None,
+        filters: Optional[DatabaseItemFilters] = None,
+        ttl: Optional[int] = None,
+    ) -> str:
+        """Add a new item to the index.
+        
+        Args:
+            item: The item to add to the index.
+            id: The id of the item.
+            filters: The filters to apply to the item.
+            ttl: The time to live for the item.
+            
+        Returns:
+            The ID of the added item.
+        """
+        # Add to SQL database first
+        item_id = self._database.add(
+            item=item,
+            id=id,
+            filters=filters,
+            ttl=ttl,
+        )
+        
+        # Add to tantivy index for search
+        self._add_to_tantivy_index(item_id, item, filters)
+        
+        return item_id
+
+    def _add_to_tantivy_index(
+        self,
+        item_id: str,
+        item: DatabaseItemType,
+        filters: Optional[DatabaseItemFilters] = None,
+    ) -> None:
+        """Add item to tantivy search index."""
+        doc = tantivy.Document()
+        
+        # Add ID field
+        doc.add_text("id", item_id)
+        
+        # Extract and add content for search
+        content = utils.extract_content_for_indexing(item)
+        doc.add_text("content", content)
+        
+        # Add title field if present
+        if isinstance(item, dict) and "title" in item:
+            doc.add_text("title", str(item["title"]))
+        
+        # Store the full data as JSON in tantivy
+        serialized_data = utils.serialize(item)
+        json_data = {"value": serialized_data}
+        doc.add_json("data", json.dumps(json_data))
+        
+        # Add filters as facets
+        if filters:
+            for key, value in filters.items():
+                facet_value = f"/{key}/{value}"
+                doc.add_facet("filters", tantivy.Facet.from_string(facet_value))
+        
+        # Add timestamps
+        now = datetime.now(timezone.utc)
+        doc.add_date("created_at", now)
+        
+        # Add score field if present
+        if isinstance(item, dict) and "score" in item and isinstance(item["score"], (int, float)):
+            doc.add_integer("score", int(item["score"]))
+        
+        # Add to index
+        self._writer.add_document(doc)
+        self._writer.commit()
+
+    def get(
+        self,
+        id: str,
+        *,
+        filters: Optional[DatabaseItemFilters] = None,
+    ) -> Optional[DatabaseItem[DatabaseItemType]]:
+        """Get an item by ID.
+        
+        Args:
+            id: The item ID.
+            filters: Optional filters to match.
+            
+        Returns:
+            The database item or None if not found.
+        """
+        return self._database.get(id, filters=filters)
+
+    def query(
+        self,
+        query: Optional[str] = None,
+        *,
+        filters: Optional[DatabaseItemFilters] = None,
+        limit: Optional[int] = None,
+        offset: int = 0,
+        fuzzy: bool = False,
+        fuzzy_distance: int = 2,
+        phrase: bool = False,
+        phrase_slop: int = 0,
+        boost_fields: Optional[Dict[str, float]] = None,
+        min_score: Optional[float] = None,
+        sort_by: Optional[str] = None,
+        ascending: bool = True,
+    ) -> List[DatabaseItem[DatabaseItemType]]:
+        """Query items using tantivy search.
+        
+        Args:
+            query: Search query string.
+            filters: Dictionary of filters to apply.
+            limit: Maximum number of results.
+            offset: Number of results to skip.
+            fuzzy: Enable fuzzy matching.
+            fuzzy_distance: Maximum edit distance for fuzzy matching.
+            phrase: Treat query as exact phrase match.
+            phrase_slop: Max words between phrase terms.
+            boost_fields: Field-specific score boosting.
+            min_score: Minimum relevance score threshold.
+            sort_by: Field to sort by.
+            ascending: Sort direction.
+            
+        Returns:
+            List of matching database items.
+        """
+        if not query:
+            # No search query - use database query directly
+            return self._database.query(
+                limit=limit,
+                offset=offset,
+                order_by=sort_by,
+                ascending=ascending,
+            )
+        
+        # Use tantivy for search
+        self._index.reload()
+        searcher = self._index.searcher()
+        
+        # Build tantivy query
+        query_parts = []
+        
+        # Add filter queries
+        if filters:
+            for key, value in filters.items():
+                facet_query = tantivy.Query.term_query(
+                    self._schema,
+                    "filters",
+                    tantivy.Facet.from_string(f"/{key}/{value}"),
+                )
+                query_parts.append((tantivy.Occur.Must, facet_query))
+        
+        # Add search query
+        if phrase:
+            words = query.split()
+            search_query = tantivy.Query.phrase_query(
+                self._schema, "content", words, slop=phrase_slop
+            )
+        elif fuzzy:
+            terms = query.split()
+            fuzzy_queries = []
+            for term in terms:
+                fuzzy_q = tantivy.Query.fuzzy_term_query(
+                    self._schema,
+                    "content", 
+                    term,
+                    distance=fuzzy_distance,
+                )
+                fuzzy_queries.append((tantivy.Occur.Should, fuzzy_q))
+            search_query = tantivy.Query.boolean_query(fuzzy_queries)
+        else:
+            # Use tantivy's query parser
+            if boost_fields:
+                search_query = self._index.parse_query(
+                    query,
+                    default_field_names=["content", "title"],
+                    field_boosts=boost_fields,
+                )
+            else:
+                search_query = self._index.parse_query(
+                    query, default_field_names=["content", "title"]
+                )
+        
+        query_parts.append((tantivy.Occur.Must, search_query))
+        
+        # Build final query
+        if query_parts:
+            final_query = tantivy.Query.boolean_query(query_parts)
+        else:
+            final_query = tantivy.Query.all_query()
+        
+        # Execute search
+        search_limit = limit or self.query_settings.limit
+        
+        # Perform search
+        search_result = searcher.search(
+            final_query,
+            limit=search_limit,
+            offset=offset,
+        )
+        
+        # Get IDs from search results and fetch from database
+        item_ids = []
+        for score, doc_address in search_result.hits:
+            if min_score and score < min_score:
+                continue
+                
+            doc = searcher.doc(doc_address)
+            item_id = doc.get_first("id")
+            if item_id:
+                item_ids.append(item_id)
+        
+        # Fetch items from database by IDs
+        results = []
+        for item_id in item_ids:
+            db_item = self._database.get(item_id, filters=filters)
+            if db_item:
+                results.append(db_item)
+                
+        return results
+
+    def delete(self, id: str) -> bool:
+        """Delete an item by ID.
+        
+        Args:
+            id: The item ID.
+            
+        Returns:
+            True if item was deleted, False if not found.
+        """
+        # Delete from database
+        deleted = self._database.delete(id)
+        
+        if deleted:
+            # Remove from tantivy index by reindexing without this item
+            # Note: Tantivy doesn't have efficient single-document deletion
+            # For now, we rely on the database as the source of truth
+            pass
+            
+        return deleted
+
+    def count(
+        self,
+        query: Optional[str] = None,
+        *,
+        filters: Optional[DatabaseItemFilters] = None,
+    ) -> int:
+        """Count items matching the query and filters.
+        
+        Args:
+            query: Search query string.
+            filters: Dictionary of filters to apply.
+            
+        Returns:
+            Number of matching items.
+        """
+        if not query:
+            # Simple count from database
+            from ....sql.types import QueryFilter, QueryCondition
+            
+            query_filter = None
+            if filters:
+                conditions = [
+                    QueryCondition(field="filters", operator="contains", value=json.dumps(filters))
+                ]
+                query_filter = QueryFilter(conditions=conditions)
+                
+            return self._database.count(query_filter)
+        else:
+            # Count via search results
+            results = self.query(query, filters=filters, limit=None)
+            return len(results)
+
+    def clear(self) -> int:
+        """Clear all items from the index.
+        
+        Returns:
+            Number of items deleted.
+        """
+        count = self._database.clear()
+        
+        # Clear tantivy index by rebuilding it
+        try:
+            self._tantivy_wrapper = utils.build_tantivy_index_from_settings(
+                settings=self.settings
+            )
+            self._index = self._tantivy_wrapper.index
+            self._schema = self._tantivy_wrapper.schema
+            self._writer = self._tantivy_wrapper.index_writer
+        except Exception:
+            pass
+            
+        return count
+
+    def __repr__(self) -> str:
+        """String representation of the index."""
+        location = str(self.path) if self.path else "memory"
+        return f"<TantivyCollectionIndex name='{self.name}' location='{location}'>"

hammad/data/collections/indexes/tantivy/settings.py

@@ -0,0 +1,51 @@
+"""hammad.data.collections.indexes.tantivy.settings"""
+
+from dataclasses import dataclass
+from typing import (
+    Any,
+    Dict,
+)
+
+__all__ = (
+    "TantivyCollectionIndexSettings",
+    "TantivyCollectionIndexQuerySettings"
+)
+
+
+@dataclass
+class TantivyCollectionIndexSettings:
+    """Object representation of user configurable settings
+    that can be used to configure a `TantivyCollectionIndex`."""
+
+    fast : bool = True
+    """Whether to use fast schema building & indexing from
+    `tantivy`'s builtin implementation."""
+
+    def get_tantivy_config(self) -> Dict[str, Any]:
+        """Returns a configuration dictionary used
+        to configure the tantivy index internally."""
+
+        return {
+            "text_fields" : {
+                "stored" : True, "fast" : self.fast
+            },
+            "numeric_fields" : {
+                "stored" : True, "indexed" : True, "fast" : self.fast
+            },
+            "date_fields" : {
+                "stored" : True, "indexed" : True, "fast" : self.fast
+            },
+            "json_fields" : {
+                "stored" : True
+            }
+        }
+
+
+@dataclass
+class TantivyCollectionIndexQuerySettings:
+    """Object representation of user configurable settings
+    that can be used to configure the query engine for a
+    `TantivyCollectionIndex`."""
+
+    limit : int = 10
+    """The maximum number of results to return."""

hammad/data/collections/indexes/tantivy/utils.py

@@ -0,0 +1,200 @@
+"""hammad.data.collections.indexes.tantivy.utils"""
+
+from dataclasses import (
+    dataclass,
+    is_dataclass,
+    asdict
+)
+from msgspec import json
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    final
+)
+
+import tantivy
+
+from .....cache import cached
+from .settings import (
+    TantivyCollectionIndexSettings,
+    TantivyCollectionIndexQuerySettings
+)
+
+
+__all__ = (
+    "TantivyCollectionIndexError",
+    "extract_content_for_indexing",
+)
+
+
+class TantivyCollectionIndexError(Exception):
+    """Exception raised when an error occurs in the `TantivyCollectionIndex`."""
+
+
+@dataclass
+class TantivyIndexWrapper:
+    """Wrapper over the `tantivy` index object."""
+
+    index : tantivy.Index
+    """The `tantivy` index object."""
+
+    schema : tantivy.Schema
+    """The `tantivy` schema object."""
+
+    index_writer : Any
+    """The `tantivy` index writer object."""
+
+
+@cached
+def match_filters_for_query(
+    stored_filters : Dict[str, Any] | None = None,
+    query_filters : Dict[str, Any] | None = None
+) -> bool:
+    """Checks if stored filters match query filters."""
+    if query_filters is None:
+        return True
+    if stored_filters is None:
+        return False
+    return all(stored_filters.get(k) == v for k, v in query_filters.items())
+
+
+@cached
+def serialize(
+    obj : Any
+) -> Any:
+    """Serializes an object to JSON."""
+    try:
+        return json.decode(json.encode(obj))
+    except Exception:
+        # Fallback to manual serialization if msgspec fails
+        if isinstance(obj, (str, int, float, bool, type(None))):
+            return obj
+        elif isinstance(obj, (list, tuple)):
+            return [serialize(item) for item in obj]
+        elif isinstance(obj, dict):
+            return {k: serialize(v) for k, v in obj.items()}
+        elif is_dataclass(obj):
+            return serialize(asdict(obj))
+        elif hasattr(obj, "__dict__"):
+            return serialize(obj.__dict__)
+        else:
+            return str(obj)
+
+
+@cached
+def build_tantivy_index_from_settings(
+    settings : TantivyCollectionIndexSettings
+) -> TantivyIndexWrapper:
+    """Builds a new `tantivy` index from the given settings."""
+    # Init schema for index
+    schema_builder = tantivy.SchemaBuilder()
+
+    # Add fields
+    # ID (stored and indexed)
+    schema_builder.add_text_field(
+        "id",
+        **settings.get_tantivy_config()["text_fields"]
+    )
+    # Content (stored and indexed) Contains entry content
+    schema_builder.add_text_field(
+        "content",
+        **{
+            **settings.get_tantivy_config()["text_fields"],
+            "tokenizer_name" : "default",
+            "index_option" : "position"
+        }
+    )
+    # Title (stored and indexed) Contains entry title
+    schema_builder.add_text_field(
+        "title",
+        **{
+            **settings.get_tantivy_config()["text_fields"],
+            "tokenizer_name" : "default",
+            "index_option" : "position"
+        }
+    )
+    # JSON (stored) Contains actual entry data
+    schema_builder.add_json_field(
+        "data",
+        **settings.get_tantivy_config()["json_fields"]
+    )
+
+    # Timestamps
+    schema_builder.add_date_field(
+        "created_at",
+        **settings.get_tantivy_config()["date_fields"]
+    )
+    schema_builder.add_date_field(
+        "expires_at",
+        **settings.get_tantivy_config()["date_fields"]
+    )
+
+    # Sorting / Scoring
+    schema_builder.add_integer_field(
+        "score",
+        **settings.get_tantivy_config()["numeric_fields"]
+    )
+
+    # Facet for Optional filters
+    schema_builder.add_facet_field("filters")
+
+    # Build the schema
+    schema = schema_builder.build()
+
+    # Create index in memory (no path means in-memory)
+    index = tantivy.Index(schema)
+
+    # Configure index writer with custom settings if provided
+    writer_config = {}
+    if "writer_heap_size" in settings.get_tantivy_config():
+        writer_config["heap_size"] = settings.get_tantivy_config()["writer_heap_size"]
+    if "writer_num_threads" in settings.get_tantivy_config():
+        writer_config["num_threads"] = settings.get_tantivy_config()["writer_num_threads"]
+
+    index_writer = index.writer(**writer_config)
+
+    # Configure index reader if settings provided
+    reader_config = settings.get_tantivy_config().get("reader_config", {})
+    if reader_config:
+        reload_policy = reader_config.get("reload_policy", "commit")
+        num_warmers = reader_config.get("num_warmers", 0)
+        index.config_reader(
+            reload_policy=reload_policy,
+            num_warmers=num_warmers
+        )
+
+    return TantivyIndexWrapper(
+        schema=schema,
+        index=index,
+        index_writer=index_writer
+    )
+
+
+@cached
+def extract_content_for_indexing(value: Any) -> str:
+    """Extract searchable text content from value for indexing."""
+    if isinstance(value, str):
+        return value
+    elif isinstance(value, dict):
+        # Concatenate all string values
+        content_parts = []
+        for v in value.values():
+            if isinstance(v, str):
+                content_parts.append(v)
+            elif isinstance(v, (list, dict)):
+                content_parts.append(json.encode(v).decode())
+            else:
+                content_parts.append(str(v))
+        return " ".join(content_parts)
+    elif isinstance(value, (list, tuple)):
+        content_parts = []
+        for item in value:
+            if isinstance(item, str):
+                content_parts.append(item)
+            else:
+                content_parts.append(str(item))
+        return " ".join(content_parts)
+    else:
+        return str(value)

hammad/data/configurations/__init__.py

@@ -1,11 +1,11 @@
-"""hammad.configuration
+"""hammad.data.configurations
 
 Contains the `Configuration` class and related functions for parsing configurations
 from various sources.
 """
 
 from typing import TYPE_CHECKING
-from ...performance.imports import create_getattr_importer
+from ..._internal import create_getattr_importer
 
 if TYPE_CHECKING:
     from .configuration import (

hammad/data/configurations/configuration.py

@@ -1,4 +1,4 @@
-"""hammad.data.types.files.configuration"""
+"""hammad.data.configurations.configuration"""
 
 import os
 import configparser
@@ -10,7 +10,7 @@ import msgspec
 import yaml
 
 from ..types.file import File, FileSource
-from ..models.base.fields import field
+from ..models.fields import field
 
 __all__ = (
     "Configuration",