postgres-fts-backend 0.0.1__py3-none-any.whl

postgres_fts_backend/__init__.py

@@ -0,0 +1,770 @@
+from __future__ import annotations
+
+import logging
+import re
+import warnings
+from typing import Any, NotRequired, TypedDict
+
+from django.apps import apps as django_apps
+from django.contrib.postgres.search import (
+    SearchHeadline,
+    SearchQuery,
+    SearchRank,
+    SearchVector,
+)
+from django.core.exceptions import FieldDoesNotExist
+from django.db import DatabaseError, connection, models
+from django.db.models import Count, F, FloatField, Q, Value
+from django.db.models.functions import Trunc
+from django.utils.encoding import force_str
+from haystack import connections
+from haystack.backends import (
+    BaseEngine,
+    BaseSearchBackend,
+    BaseSearchQuery,
+    SearchNode,
+    log_query,
+)
+from haystack.constants import DJANGO_CT, DJANGO_ID
+from haystack.indexes import SearchIndex
+from haystack.models import SearchResult
+from haystack.utils import get_model_ct
+
+from postgres_fts_backend.models import (
+    AlignedUnionQuerySet,
+    IndexQuerySet,
+    generate_index_models,
+    get_index_model,
+)
+
+
+class FacetResults(TypedDict, total=False):
+    fields: dict[str, list[tuple[Any, int]]]
+    dates: dict[str, list[tuple[Any, int]]]
+    queries: dict[str, int]
+
+
+class SearchResponse(TypedDict):
+    results: list[SearchResult]
+    hits: int
+    spelling_suggestion: str | None
+    facets: NotRequired[FacetResults]
+
+
+class _CtInfo(TypedDict):
+    model: type[models.Model]
+    field_names: list[str]
+
+
+default_app_config = "postgres_fts_backend.apps.PostgresFTSConfig"
+
+log = logging.getLogger("haystack")
+
+
+def _table_name(model: type[models.Model]) -> str:
+    return f"haystack_index_{model._meta.app_label}_{model._meta.model_name}"
+
+
+def validate_all_schemas() -> None:
+    """Validate all index tables at startup. Called from AppConfig.ready()."""
+    try:
+        ui = connections["default"].get_unified_index()
+        existing_tables = connection.introspection.table_names()
+    except Exception:
+        warnings.warn(
+            "Could not connect to database to validate index schemas. "
+            "Run 'manage.py build_postgres_schema' then "
+            "'manage.py migrate postgres_fts_backend' once the database is available."
+        )
+        return
+
+    for model, index in ui.get_indexes().items():
+        table = _table_name(model)
+
+        if table not in existing_tables:
+            warnings.warn(
+                f"Table '{table}' does not exist. Run 'manage.py build_postgres_schema' "
+                "then 'manage.py migrate postgres_fts_backend'."
+            )
+            continue
+
+        expected_columns = {"id", "django_id", "django_ct", "search_vector"}
+        for field_name in index.fields:
+            if field_name not in ("django_ct", "django_id"):
+                expected_columns.add(field_name)
+
+        with connection.cursor() as cursor:
+            db_columns = {
+                info.name
+                for info in connection.introspection.get_table_description(
+                    cursor, table
+                )
+            }
+
+        missing = expected_columns - db_columns
+        if missing:
+            warnings.warn(
+                "Index table '{}' schema is out of date (missing columns: {}). "
+                "Run 'manage.py build_postgres_schema' then "
+                "'manage.py migrate postgres_fts_backend'.".format(
+                    table, ", ".join(sorted(missing))
+                )
+            )
+
+
+def _field_names(index: SearchIndex) -> list[str]:
+    return [name for name in index.fields if name not in ("django_ct", "django_id")]
+
+
+def _resolve_field_name(field_name: str) -> str:
+    if field_name.endswith("_exact"):
+        return field_name[:-6]
+    return field_name
+
+
+def _parse_narrow_query(query_string: str) -> tuple[str, str]:
+    match = re.match(r'^(\w+):"(.+)"$', query_string)
+    if not match:
+        raise ValueError(f"Cannot parse narrow query: '{query_string}'")
+    return match.group(1), match.group(2)
+
+
+class IndexSearch:
+    def __init__(
+        self,
+        qs: IndexQuerySet,
+        index: SearchIndex,
+        search_config: str,
+        has_rank: bool = False,
+        search_text: str | None = None,
+    ) -> None:
+        self.qs = qs
+        self.index = index
+        self.search_config: str = search_config
+        self.has_rank: bool = has_rank
+        self.search_text: str | None = search_text
+        self.score_field: str = "rank"
+        self.highlight_field: str | None = None
+
+    @classmethod
+    def from_query_string(
+        cls,
+        index_model: type[models.Model],
+        index: SearchIndex,
+        search_config: str,
+        query_string: str,
+    ) -> IndexSearch:
+        if query_string == "*":
+            qs = index_model.objects.all().annotate(  # type: ignore[attr-defined]
+                rank=Value(0, output_field=FloatField())
+            )
+            return cls(qs, index, search_config)
+
+        if ":" in query_string and not query_string.startswith('"'):
+            field, _, value = query_string.partition(":")
+            content_field = index.get_content_field()
+            if field == content_field:
+                return cls(
+                    index_model.objects.search(value, config=search_config),  # type: ignore[attr-defined]
+                    index,
+                    search_config,
+                    has_rank=True,
+                    search_text=value,
+                )
+            qs = index_model.objects.filter(**{field: value}).annotate(  # type: ignore[attr-defined]
+                rank=Value(0, output_field=FloatField())
+            )
+            return cls(qs, index, search_config)
+
+        return cls(
+            index_model.objects.search(query_string, config=search_config),  # type: ignore[attr-defined]
+            index,
+            search_config,
+            has_rank=True,
+            search_text=query_string,
+        )
+
+    @classmethod
+    def from_orm_query(
+        cls,
+        index_model: type[models.Model],
+        index: SearchIndex,
+        search_config: str,
+        orm_query: Q,
+    ) -> IndexSearch:
+        content_search_text = orm_query.content_search_text  # type: ignore[attr-defined]
+        qs = index_model.objects.filter(orm_query)  # type: ignore[attr-defined]
+        if content_search_text:
+            qs = qs.ranked(content_search_text, config=search_config)
+        else:
+            qs = qs.annotate(rank=Value(0, output_field=FloatField()))
+        return cls(
+            qs,
+            index,
+            search_config,
+            has_rank=bool(content_search_text),
+            search_text=content_search_text,
+        )
+
+    def narrow(self, narrow_queries: list[str]) -> None:
+        for nq in narrow_queries:
+            field, value = _parse_narrow_query(nq)
+            col = _resolve_field_name(field)
+            try:
+                self.qs.model._meta.get_field(col)
+            except FieldDoesNotExist:
+                self.qs = self.qs.none()
+                return
+            self.qs = self.qs.filter(**{col: value})
+
+    def highlight(self) -> None:
+        if self.search_text is None:
+            return
+        content_field = self.index.get_content_field()
+        sq = SearchQuery(
+            self.search_text, search_type="websearch", config=self.search_config
+        )
+        self.qs = self.qs.annotate(
+            headline=SearchHeadline(content_field, sq, config=self.search_config)
+        )
+        self.highlight_field = content_field
+
+    def boost(self, boost_dict: dict[str, float]) -> None:
+        if not self.has_rank or not boost_dict:
+            return
+        annotations: dict[str, Any] = {}
+        combined: Any = F("rank")
+        for i, (term, weight) in enumerate(boost_dict.items()):
+            alias = f"_boost_{i}"
+            bq = SearchQuery(term, search_type="websearch", config=self.search_config)
+            annotations[alias] = SearchRank(
+                "search_vector", bq, cover_density=True, normalization=32
+            )
+            combined = combined * (1.0 + F(alias) * weight)
+        annotations["_boosted_rank"] = combined
+        self.qs = self.qs.annotate(**annotations)
+        self.score_field = "_boosted_rank"
+
+    def count(self) -> int:
+        return self.qs.count()
+
+    def facets(
+        self,
+        facets: list[str] | None = None,
+        date_facets: dict[str, Any] | None = None,
+        query_facets: list[tuple[str, str]] | None = None,
+    ) -> FacetResults:
+        result: FacetResults = {}
+
+        if facets:
+            result["fields"] = {}
+            for field_name in facets:
+                col = _resolve_field_name(field_name)
+                facet_qs = (
+                    self.qs.values(col)
+                    .annotate(count=Count("id"))
+                    .order_by("-count", col)
+                )
+                result["fields"][field_name] = [
+                    (row[col], row["count"]) for row in facet_qs
+                ]
+
+        if date_facets:
+            result["dates"] = {}
+            for field_name, facet_opts in date_facets.items():
+                col = _resolve_field_name(field_name)
+                gap_by = facet_opts["gap_by"]
+                start_date = facet_opts["start_date"]
+                end_date = facet_opts["end_date"]
+                facet_qs = (
+                    self.qs.filter(
+                        **{
+                            f"{col}__gte": start_date,
+                            f"{col}__lt": end_date,
+                        }
+                    )
+                    .annotate(bucket=Trunc(col, gap_by))
+                    .values("bucket")
+                    .annotate(count=Count("id"))
+                    .order_by("bucket")
+                )
+                result["dates"][field_name] = [
+                    (row["bucket"], row["count"]) for row in facet_qs
+                ]
+
+        if query_facets:
+            result["queries"] = {}
+            for field_name, value in query_facets:
+                col = _resolve_field_name(field_name)
+                count = self.qs.filter(**{col: value}).count()
+                result["queries"][f"{field_name}_{value}"] = count
+
+        return result
+
+    def results(
+        self,
+        sort_by: list[str] | None = None,
+        start_offset: int = 0,
+        end_offset: int | None = None,
+        result_class: type = SearchResult,
+    ) -> list[SearchResult]:
+        qs = self.qs
+
+        # Ordering
+        if sort_by:
+            qs = qs.order_by(*sort_by)
+        elif self.has_rank:
+            qs = qs.order_by(f"-{self.score_field}")
+
+        # Pagination
+        if end_offset is not None:
+            qs = qs[start_offset:end_offset]
+        elif start_offset:
+            qs = qs[start_offset:]
+
+        # Materialize
+        model = self.index.get_model()
+        field_names = _field_names(self.index)
+        app_label = model._meta.app_label
+        model_name = model._meta.model_name
+
+        results = []
+        for obj in qs:
+            stored_fields = {fn: getattr(obj, fn) for fn in field_names}
+            if self.highlight_field:
+                headline = getattr(obj, "headline", None)
+                if headline:
+                    stored_fields["highlighted"] = {self.highlight_field: [headline]}
+            rank = getattr(obj, self.score_field, None)
+            score = float(rank) if self.has_rank and rank is not None else 0
+            results.append(
+                result_class(
+                    app_label, model_name, obj.django_id, score, **stored_fields
+                )
+            )
+        return results
+
+
+class MultiIndexSearch:
+    """Wraps multiple IndexSearch instances for cross-model search."""
+
+    def __init__(self, searches: list[tuple[IndexSearch, type[models.Model]]]) -> None:
+        self.searches = searches
+
+    def count(self) -> int:
+        return sum(s.count() for s, _model in self.searches)
+
+    def facets(
+        self,
+        facets: list[str] | None = None,
+        date_facets: dict[str, Any] | None = None,
+        query_facets: list[tuple[str, str]] | None = None,
+    ) -> FacetResults:
+        merged: FacetResults = {}
+
+        if facets:
+            merged["fields"] = {}
+            for field_name in facets:
+                col = _resolve_field_name(field_name)
+                combined_counts: dict[Any, int] = {}
+                for s, model in self.searches:
+                    index_model = get_index_model(model)
+                    try:
+                        index_model._meta.get_field(col)
+                    except FieldDoesNotExist:
+                        continue
+                    sub = s.facets(facets=[field_name])
+                    for value, count in sub.get("fields", {}).get(field_name, []):
+                        combined_counts[value] = combined_counts.get(value, 0) + count
+                merged["fields"][field_name] = sorted(
+                    combined_counts.items(), key=lambda x: (-x[1], x[0])
+                )
+
+        if date_facets:
+            merged["dates"] = {}
+            for field_name, facet_opts in date_facets.items():
+                col = _resolve_field_name(field_name)
+                combined_buckets: dict[Any, int] = {}
+                for s, model in self.searches:
+                    index_model = get_index_model(model)
+                    try:
+                        index_model._meta.get_field(col)
+                    except FieldDoesNotExist:
+                        continue
+                    sub = s.facets(date_facets={field_name: facet_opts})
+                    for bucket, count in sub.get("dates", {}).get(field_name, []):
+                        combined_buckets[bucket] = (
+                            combined_buckets.get(bucket, 0) + count
+                        )
+                merged["dates"][field_name] = sorted(combined_buckets.items())
+
+        if query_facets:
+            merged["queries"] = {}
+            for field_name, value in query_facets:
+                col = _resolve_field_name(field_name)
+                total = 0
+                for s, model in self.searches:
+                    index_model = get_index_model(model)
+                    try:
+                        index_model._meta.get_field(col)
+                    except FieldDoesNotExist:
+                        continue
+                    sub = s.facets(query_facets=[(field_name, value)])
+                    key = f"{field_name}_{value}"
+                    total += sub.get("queries", {}).get(key, 0)
+                merged["queries"][f"{field_name}_{value}"] = total
+
+        return merged
+
+    def results(
+        self,
+        sort_by: list[str] | None = None,
+        start_offset: int = 0,
+        end_offset: int | None = None,
+        result_class: type = SearchResult,
+    ) -> list[SearchResult]:
+        # These are uniform across all searches (same kwargs applied to each)
+        first_search = self.searches[0][0]
+        score_field = first_search.score_field
+        has_rank = first_search.has_rank
+        highlight_field = first_search.highlight_field
+
+        # Build the aligned union
+        union_qs: IndexQuerySet | AlignedUnionQuerySet = first_search.qs
+        for s, model in self.searches[1:]:
+            union_qs = union_qs.aligned_union(s.qs)
+
+        # Per-model lookup: field_names and model identity vary across indexes
+        ct_map: dict[str, _CtInfo] = {
+            get_model_ct(model): {
+                "model": model,
+                "field_names": _field_names(s.index),
+            }
+            for s, model in self.searches
+        }
+
+        # Ordering — always include tiebreakers for stable pagination
+        if sort_by:
+            ordered_qs = union_qs.order_by(*sort_by, "django_ct", "django_id")
+        else:
+            ordered_qs = union_qs.order_by("-rank", "django_ct", "django_id")
+
+        # Pagination
+        if end_offset is not None:
+            ordered_qs = ordered_qs[start_offset:end_offset]
+        elif start_offset:
+            ordered_qs = ordered_qs[start_offset:]
+
+        # Materialize
+        results = []
+        for row in ordered_qs:
+            info = ct_map[row["django_ct"]]
+            model = info["model"]
+
+            stored_fields = {fn: row.get(fn) for fn in info["field_names"]}
+
+            if highlight_field and "headline" in row and row["headline"]:
+                stored_fields["highlighted"] = {highlight_field: [row["headline"]]}
+
+            rank = row.get(score_field)
+            score = float(rank) if has_rank and rank is not None else 0
+
+            results.append(
+                result_class(
+                    model._meta.app_label,
+                    model._meta.model_name,
+                    row["django_id"],
+                    score,
+                    **stored_fields,
+                )
+            )
+        return results
+
+
+class PostgresFTSSearchBackend(BaseSearchBackend):
+    def __init__(self, connection_alias, **connection_options):
+        super().__init__(connection_alias, **connection_options)
+        self.search_config = connection_options.get("SEARCH_CONFIG", "english")
+
+    def build_schema(self, fields):
+        return generate_index_models()
+
+    def update(self, index, iterable, commit=True):
+        try:
+            model = index.get_model()
+            index_model = get_index_model(model)
+            field_names = _field_names(index)
+            content_field = index.get_content_field()
+
+            rows = []
+            for obj in iterable:
+                prepared = index.full_prepare(obj)
+                defaults = {fn: prepared.get(fn) for fn in field_names}
+                rows.append(
+                    index_model(
+                        django_ct=prepared[DJANGO_CT],
+                        django_id=prepared[DJANGO_ID],
+                        **defaults,
+                    )
+                )
+
+            if rows:
+                index_model.objects.bulk_create(
+                    rows,
+                    update_conflicts=True,
+                    unique_fields=["django_ct", "django_id"],
+                    update_fields=field_names,
+                )
+                index_model.objects.filter(
+                    django_ct=get_model_ct(model),
+                    django_id__in=[r.django_id for r in rows],
+                ).update(
+                    search_vector=SearchVector(content_field, config=self.search_config)
+                )
+        except DatabaseError:
+            if not self.silently_fail:
+                raise
+            log.exception("Failed to update index for %s", index)
+
+    def remove(self, obj_or_string, commit=True):
+        try:
+            if isinstance(obj_or_string, str):
+                # String format: "app_label.model_name.pk"
+                parts = obj_or_string.split(".", 2)
+                if len(parts) != 3:
+                    raise ValueError(
+                        "String identifier must be 'app_label.model_name.pk', "
+                        f"got '{obj_or_string}'"
+                    )
+                model = django_apps.get_model(parts[0], parts[1])
+                django_ct = get_model_ct(model)
+                django_id = parts[2]
+            else:
+                model = type(obj_or_string)
+                django_ct = get_model_ct(model)
+                django_id = force_str(obj_or_string.pk)
+
+            index_model = get_index_model(model)
+            index_model.objects.filter(
+                django_ct=django_ct, django_id=django_id
+            ).delete()
+        except DatabaseError:
+            if not self.silently_fail:
+                raise
+            log.exception("Failed to remove document '%s'", obj_or_string)
+
+    def clear(self, models=None, commit=True):
+        try:
+            if models is None:
+                ui = connections["default"].get_unified_index()
+                models = ui.get_indexes().keys()
+
+            for model in models:
+                index_model = get_index_model(model)
+                index_model.objects.all().delete()
+        except DatabaseError:
+            if not self.silently_fail:
+                raise
+            log.exception("Failed to clear index")
+
+    @log_query
+    def search(self, query_string: str, **kwargs: Any) -> SearchResponse:
+        if not query_string or not query_string.strip():
+            return {"results": [], "hits": 0, "spelling_suggestion": None}
+
+        try:
+            ui = connections["default"].get_unified_index()
+
+            requested_models = kwargs.get("models")
+            if requested_models:
+                model_index_pairs = [(m, ui.get_index(m)) for m in requested_models]
+            else:
+                model_index_pairs = list(ui.get_indexes().items())
+
+            orm_query = kwargs.pop("orm_query", None)
+
+            result_class = kwargs.get("result_class", SearchResult)
+            sort_by = kwargs.get("sort_by")
+            start_offset = int(kwargs.get("start_offset", 0))
+            end_offset = (
+                int(kwargs["end_offset"])
+                if kwargs.get("end_offset") is not None
+                else None
+            )
+
+            # Build IndexSearch per model
+            searches = []
+            for model, index in model_index_pairs:
+                index_model = get_index_model(model)
+                if orm_query is not None:
+                    s = IndexSearch.from_orm_query(
+                        index_model, index, self.search_config, orm_query
+                    )
+                else:
+                    s = IndexSearch.from_query_string(
+                        index_model, index, self.search_config, query_string
+                    )
+
+                if narrow_queries := kwargs.get("narrow_queries"):
+                    s.narrow(narrow_queries)
+                if boost := kwargs.get("boost"):
+                    s.boost(boost)
+                if kwargs.get("highlight"):
+                    s.highlight()
+
+                searches.append((s, model))
+
+            if len(searches) == 1:
+                # Single-model path (no behavior change)
+                s, model = searches[0]
+                total_count = s.count()
+                facets = s.facets(
+                    facets=kwargs.get("facets"),
+                    date_facets=kwargs.get("date_facets"),
+                    query_facets=kwargs.get("query_facets"),
+                )
+                results = s.results(sort_by, start_offset, end_offset, result_class)
+            else:
+                # Multi-model path
+                multi = MultiIndexSearch(searches)
+                total_count = multi.count()
+                facets = multi.facets(
+                    facets=kwargs.get("facets"),
+                    date_facets=kwargs.get("date_facets"),
+                    query_facets=kwargs.get("query_facets"),
+                )
+                results = multi.results(sort_by, start_offset, end_offset, result_class)
+
+            response: SearchResponse = {
+                "results": results,
+                "hits": total_count,
+                "spelling_suggestion": None,
+            }
+            if facets:
+                response["facets"] = facets
+            return response
+        except DatabaseError:
+            if not self.silently_fail:
+                raise
+            log.exception("Failed to search with query '%s'", query_string)
+            return {"results": [], "hits": 0, "spelling_suggestion": None}
+
+    def prep_value(self, value: Any) -> Any:
+        return value
+
+    def more_like_this(
+        self,
+        model_instance: models.Model,
+        additional_query_string: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        raise NotImplementedError(
+            "postgres_fts_backend does not support more_like_this. "
+            "PostgreSQL has no native document similarity feature."
+        )
+
+
+class ORMSearchNode(SearchNode):
+    def as_orm_query(self, query_fragment_callback):
+        result = []
+        for child in self.children:
+            if hasattr(child, "as_orm_query"):
+                result.append(child.as_orm_query(query_fragment_callback))
+            else:
+                expression, value = child
+                field, filter_type = self.split_expression(expression)
+                result.append(query_fragment_callback(field, filter_type, value))
+
+        query = Q()
+        if self.connector == self.AND:
+            for subquery in result:
+                query &= subquery
+        elif self.connector == self.OR:
+            for subquery in result:
+                query |= subquery
+
+        if query and self.negated:
+            query = ~query
+
+        return query
+
+
+class ORMSearchQuery(BaseSearchQuery):
+
+    def __init__(self, using="default"):
+        super().__init__(using=using)
+        self.query_filter = ORMSearchNode()
+        self.content_search_text = None
+
+    def clean(self, query_fragment):
+        return query_fragment
+
+    def build_query_fragment(self, field, filter_type, value):
+        if hasattr(value, "prepare"):
+            value = value.prepare(self)
+
+        if filter_type == "content":
+            if field == "content":
+                self.content_search_text = value
+                return Q(
+                    search_vector=SearchQuery(
+                        value,
+                        search_type="websearch",
+                        config=self.backend.search_config,
+                    )
+                )
+            return Q(**{f"{field}__trigram_similar": value})
+
+        if filter_type == "fuzzy":
+            return Q(**{f"{field}__trigram_similar": value})
+
+        if filter_type == "in":
+            value = list(value)
+            if not value:
+                return Q(pk__in=[])
+        elif filter_type == "range":
+            value = (value[0], value[1])
+
+        # contains/startswith/endswith → case-insensitive Django lookups
+        lookup = {
+            "contains": "icontains",
+            "startswith": "istartswith",
+            "endswith": "iendswith",
+        }.get(filter_type, filter_type)
+
+        return Q(**{f"{field}__{lookup}": value})
+
+    def build_query(self):
+        final_query = self.query_filter.as_orm_query(self.build_query_fragment)
+        if not final_query:
+            return Q()
+        return final_query
+
+    def matching_all_fragment(self):
+        return Q()
+
+    def run(self, spelling_query=None, **kwargs):
+        final_query = self.build_query()
+        search_kwargs = self.build_params(spelling_query=spelling_query)
+
+        if kwargs:
+            search_kwargs.update(kwargs)
+
+        final_query.content_search_text = self.content_search_text
+        search_kwargs["orm_query"] = final_query
+
+        results = self.backend.search("*", **search_kwargs)
+        self._results = results.get("results", [])
+        self._hit_count = results.get("hits", 0)
+        self._facet_counts = self.post_process_facets(results)
+        self._spelling_suggestion = results.get("spelling_suggestion", None)
+
+    def get_count(self):
+        if self._hit_count is None:
+            self.run()
+        return self._hit_count
+
+
+class PostgresFTSEngine(BaseEngine):
+    backend = PostgresFTSSearchBackend
+    query = ORMSearchQuery

postgres_fts_backend/apps.py

@@ -0,0 +1,11 @@
+from django.apps import AppConfig
+
+
+class PostgresFTSConfig(AppConfig):
+    name = "postgres_fts_backend"
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        from postgres_fts_backend import validate_all_schemas  # noqa: PLC0415
+
+        validate_all_schemas()

postgres_fts_backend/management/commands/build_postgres_schema.py

@@ -0,0 +1,84 @@
+import os
+import sys
+
+from django.conf import settings
+from django.contrib.postgres.operations import TrigramExtension
+from django.core.management.base import BaseCommand
+from django.db.migrations.autodetector import MigrationAutodetector
+from django.db.migrations.loader import MigrationLoader
+from django.db.migrations.state import ModelState, ProjectState
+from django.db.migrations.writer import MigrationWriter
+
+from postgres_fts_backend.models import generate_index_models
+
+APP_LABEL = "postgres_fts_backend"
+
+
+class Command(BaseCommand):
+    help = (
+        "Generate Django migrations for haystack search index tables. "
+        "Run this after changing SearchIndex definitions, then run "
+        "'manage.py migrate postgres_fts_backend'."
+    )
+
+    def handle(self, *args, **options):
+        index_models = generate_index_models()
+
+        if not index_models:
+            self.stdout.write("No search indexes found.")
+            return
+
+        # Build the target state from the dynamic models
+        new_state = ProjectState()
+        for model_cls in index_models.values():
+            model_state = ModelState.from_model(model_cls)
+            new_state.add_model(model_state)
+
+        # Load existing migrations to get the current state
+        loader = MigrationLoader(None, ignore_no_migrations=True)
+        old_state = loader.project_state()
+
+        # Detect changes
+        autodetector = MigrationAutodetector(old_state, new_state)
+        changes = autodetector.changes(graph=loader.graph)
+
+        if not changes.get(APP_LABEL):
+            self.stdout.write("No changes detected.")
+            return
+
+        # Determine output directory
+        migrations_module = getattr(settings, "MIGRATION_MODULES", {}).get(APP_LABEL)
+        if migrations_module:
+            migrations_dir = os.path.join(*migrations_module.split("."))
+            # Make it relative to the project base if not absolute
+            if not os.path.isabs(migrations_dir):
+                # Find the root by looking at the first component on sys.path
+                # that contains the module
+                for path in sys.path:
+                    candidate = os.path.join(path, migrations_dir)
+                    if os.path.isdir(os.path.dirname(candidate)) or path == "":
+                        migrations_dir = candidate
+                        break
+        else:
+            # Default to the package's own migrations dir
+            migrations_dir = os.path.join(
+                os.path.dirname(os.path.dirname(__file__)),
+                "migrations",
+            )
+
+        os.makedirs(migrations_dir, exist_ok=True)
+
+        # Ensure __init__.py exists
+        init_path = os.path.join(migrations_dir, "__init__.py")
+        if not os.path.exists(init_path):
+            with open(init_path, "w") as f:
+                f.write("")
+
+        for migration in changes[APP_LABEL]:
+            if getattr(migration, "initial", False):
+                migration.operations.insert(0, TrigramExtension())
+            writer = MigrationWriter(migration)
+            migration_path = os.path.join(migrations_dir, f"{migration.name}.py")
+            with open(migration_path, "w") as f:
+                f.write(writer.as_string())
+            self.stdout.write(f"Created {migration_path}")

postgres_fts_backend/models.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from typing import Any
+
+from django.contrib.postgres.indexes import GinIndex, OpClass
+from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVectorField
+from django.db import models
+from django.db.models import Value
+from haystack import connections as haystack_connections
+from haystack import indexes as haystack_indexes
+
+
+class IndexQuerySet(models.QuerySet):
+    def search(self, search_text: str, config: str = "english") -> IndexQuerySet:
+        """Full-text filter + rank annotation in one call."""
+        sq = SearchQuery(search_text, search_type="websearch", config=config)
+        return self.filter(search_vector=sq).annotate(
+            rank=SearchRank("search_vector", sq, cover_density=True, normalization=32)
+        )
+
+    def ranked(self, search_text: str, config: str = "english") -> IndexQuerySet:
+        """Add rank annotation only (when filter is applied separately)."""
+        sq = SearchQuery(search_text, search_type="websearch", config=config)
+        return self.annotate(
+            rank=SearchRank("search_vector", sq, cover_density=True, normalization=32)
+        )
+
+    def aligned_union(self, other: IndexQuerySet) -> AlignedUnionQuerySet:
+        """Start a chainable aligned union with another queryset.
+
+        Returns an AlignedUnionQuerySet that can be further chained:
+            qs1.aligned_union(qs2).aligned_union(qs3)
+        """
+        return AlignedUnionQuerySet([self, other])
+
+
+class AlignedUnionQuerySet:
+    """Accumulates querysets and builds an aligned union lazily.
+
+    Introspects model fields and annotations to build a superset of columns.
+    Missing columns are filled with Value(None, output_field=...).
+    """
+
+    SKIP_FIELDS: set[str] = {"id", "search_vector"}
+
+    def __init__(self, querysets: list[IndexQuerySet]) -> None:
+        self._querysets = list(querysets)
+        self._built: models.QuerySet | None = None
+
+    def aligned_union(self, other: IndexQuerySet) -> AlignedUnionQuerySet:
+        return AlignedUnionQuerySet(self._querysets + [other])
+
+    def _build(self) -> models.QuerySet:
+        if self._built is not None:
+            return self._built
+
+        all_columns = {}
+        per_qs_columns = []
+
+        for qs in self._querysets:
+            qs_cols = set()
+            for f in qs.model._meta.get_fields():
+                if f.name in self.SKIP_FIELDS:
+                    continue
+                if hasattr(f, "column"):
+                    qs_cols.add(f.name)
+                    if f.name not in all_columns:
+                        all_columns[f.name] = f
+            if hasattr(qs, "query") and hasattr(qs.query, "annotations"):
+                for name, annotation in qs.query.annotations.items():
+                    qs_cols.add(name)
+                    if name not in all_columns:
+                        all_columns[name] = annotation.output_field
+            per_qs_columns.append(qs_cols)
+
+        sorted_cols = sorted(all_columns.keys())
+
+        aligned = []
+        for qs, qs_cols in zip(self._querysets, per_qs_columns):
+            missing = set(sorted_cols) - qs_cols
+            if missing:
+                annotations = {}
+                for col_name in missing:
+                    field_meta = all_columns[col_name]
+                    if hasattr(field_meta, "column"):
+                        output_field = field_meta.__class__(null=True)
+                    else:
+                        output_field = field_meta.__class__()
+                    annotations[col_name] = Value(None, output_field=output_field)
+                qs = qs.annotate(**annotations)
+            aligned.append(qs.values(*sorted_cols))
+
+        self._built = aligned[0].union(*aligned[1:], all=True)
+        return self._built
+
+    def order_by(self, *args: str) -> models.QuerySet:
+        return self._build().order_by(*args)
+
+    def count(self) -> int:
+        return self._build().count()
+
+    def __iter__(self) -> Iterator[dict[str, Any]]:
+        return iter(self._build())
+
+    def __getitem__(self, key: int | slice) -> Any:
+        return self._build()[key]
+
+
+FIELD_MAP: dict[type, Callable[[], models.Field]] = {
+    haystack_indexes.CharField: lambda: models.TextField(null=True),
+    haystack_indexes.EdgeNgramField: lambda: models.TextField(null=True),
+    haystack_indexes.NgramField: lambda: models.TextField(null=True),
+    haystack_indexes.DateTimeField: lambda: models.DateTimeField(null=True),
+    haystack_indexes.DateField: lambda: models.DateField(null=True),
+    haystack_indexes.IntegerField: lambda: models.IntegerField(null=True),
+    haystack_indexes.FloatField: lambda: models.FloatField(null=True),
+    haystack_indexes.BooleanField: lambda: models.BooleanField(null=True),
+}
+
+
+def _django_field_for(haystack_field: haystack_indexes.SearchField) -> models.Field:
+    for haystack_cls, factory in FIELD_MAP.items():
+        if isinstance(haystack_field, haystack_cls):
+            return factory()
+    return models.TextField(null=True)
+
+
+# Django models for search index tables are created dynamically from Haystack
+# SearchIndex definitions. Unlike normal Django models (defined statically in
+# models.py), these must be built at runtime because the set of fields depends
+# on the user's SearchIndex classes, which can change independently of
+# migrations. The database schema is managed separately via the
+# build_postgres_schema management command, so the runtime model and the
+# database can be out of sync until the user regenerates and applies migrations.
+# validate_all_schemas() (called at startup) checks for this.
+
+_index_models_cache: dict[str, type[models.Model]] = {}
+
+
+def _build_index_model(
+    source_model: type[models.Model], search_index: haystack_indexes.SearchIndex
+) -> type[models.Model]:
+    app_label: str = source_model._meta.app_label
+    model_name: str = source_model._meta.model_name  # type: ignore[assignment]
+
+    class_name = f"HaystackIndex_{app_label.capitalize()}_{model_name.capitalize()}"
+
+    if class_name in _index_models_cache:
+        return _index_models_cache[class_name]
+
+    attrs = {
+        "__module__": "postgres_fts_backend.models",
+        "django_id": models.CharField(max_length=255),
+        "django_ct": models.CharField(max_length=255),
+        "search_vector": SearchVectorField(null=True),
+        "objects": IndexQuerySet.as_manager(),
+    }
+
+    for field_name, field_obj in search_index.fields.items():
+        if field_name in ("django_ct", "django_id"):
+            continue
+        attrs[field_name] = _django_field_for(field_obj)
+
+    table_name = f"haystack_index_{app_label}_{model_name}"
+
+    db_indexes = [
+        GinIndex(
+            fields=["search_vector"],
+            name=f"{table_name}_sv_gin",
+        )
+    ]
+
+    for field_name, field_obj in search_index.fields.items():
+        if isinstance(
+            field_obj, (haystack_indexes.EdgeNgramField, haystack_indexes.NgramField)
+        ):
+            db_indexes.append(
+                GinIndex(
+                    OpClass(models.F(field_name), name="gin_trgm_ops"),
+                    name=f"{table_name}_{field_name}_trgm",
+                )
+            )
+
+    meta = type(
+        "Meta",
+        (),
+        {
+            "app_label": "postgres_fts_backend",
+            "db_table": table_name,
+            "unique_together": [("django_ct", "django_id")],
+            "indexes": db_indexes,
+        },
+    )
+    attrs["Meta"] = meta
+
+    model_cls = type(class_name, (models.Model,), attrs)
+    _index_models_cache[class_name] = model_cls
+    return model_cls
+
+
+def get_index_model(source_model: type[models.Model]) -> type[models.Model]:
+    ui = haystack_connections["default"].get_unified_index()
+    search_index = ui.get_index(source_model)
+    return _build_index_model(source_model, search_index)
+
+
+def generate_index_models() -> dict[type[models.Model], type[models.Model]]:
+    ui = haystack_connections["default"].get_unified_index()
+    return {
+        source_model: _build_index_model(source_model, search_index)
+        for source_model, search_index in ui.get_indexes().items()
+    }

postgres_fts_backend-0.0.1.dist-info/METADATA

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: postgres-fts-backend
+Version: 0.0.1
+Summary: A PostgreSQL Full Text Seach Backend for Haystack
+Author-email: Forest Gregg <fgregg@datamade.us>
+Project-URL: Repository, https://github.com/fgregg/postgres-fts-backend
+Project-URL: Issues, https://github.com/fgregg/postgres-fts-backend/issues
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: django>=5.0
+Requires-Dist: django-haystack>=2.8.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-django; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: django-stubs; extra == "dev"
+Dynamic: license-file
+
+# postgres-fts-backend
+
+A [Django Haystack](https://django-haystack.readthedocs.io/) backend that uses
+PostgreSQL's built-in full-text search. No external search service required.
+
+## Requirements
+
+- Python >= 3.12
+- Django >= 5.0
+- django-haystack >= 2.8.0
+- PostgreSQL
+
+## Installation
+
+```bash
+pip install postgres-fts-backend
+```
+
+Add to `INSTALLED_APPS`:
+
+```python
+INSTALLED_APPS = [
+    "django.contrib.postgres",
+    "haystack",
+    "postgres_fts_backend",
+    # ...
+]
+```
+
+Set a migration module so the generated search index migrations live in
+your project rather than inside the installed package:
+
+```python
+MIGRATION_MODULES = {
+    "postgres_fts_backend": "myapp.search_migrations",
+}
+```
+
+Configure Haystack:
+
+```python
+HAYSTACK_CONNECTIONS = {
+    "default": {
+        "ENGINE": "postgres_fts_backend.PostgresFTSEngine",
+    },
+}
+```
+
+To use a search configuration other than `"english"`:
+
+```python
+HAYSTACK_CONNECTIONS = {
+    "default": {
+        "ENGINE": "postgres_fts_backend.PostgresFTSEngine",
+        "SEARCH_CONFIG": "spanish",
+    },
+}
+```
+
+## Other Peculiarities of this backend
+
+### Build indexes through models and migrations
+
+```bash
+python manage.py build_postgres_schema
+python manage.py migrate postgres_fts_backend
+```
+
+Run these two commands again whenever you change a `SearchIndex` definition.
+
+### Fuzzy search
+
+Fuzzy queries use PostgreSQL's trigram similarity matching (`pg_trgm`):
+
+```python
+results = SearchQuerySet().filter(author__fuzzy="Janee")
+```
+
+The similarity threshold is controlled by PostgreSQL's
+`pg_trgm.similarity_threshold` setting (default 0.3). To adjust it:
+
+```sql
+ALTER DATABASE mydb SET pg_trgm.similarity_threshold = 0.5;
+```
+
+### `more_like_this` not implemented
+PostgreSQL FTS doesn't provide any facilities for this. It could be done, but I just need to think more about it.
+
+### `spelling_suggestions` are not supported

postgres_fts_backend-0.0.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+postgres_fts_backend/__init__.py,sha256=PeVca49tR0wFo87SEYdGc5lt4-b5JTV3-IKHCux5q0w,27151
+postgres_fts_backend/apps.py,sha256=Yc9zEu63CorJLVh1HkuLjGqTThn0BBqIW3S2D72n3JA,293
+postgres_fts_backend/models.py,sha256=U_Gjur12y_ck3U4wNB5qeRFG46AtneHgsH-2VR-lNB8,8175
+postgres_fts_backend/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+postgres_fts_backend/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+postgres_fts_backend/management/commands/build_postgres_schema.py,sha256=-y5pLZ1Eq2cgOUjL07b5li8tI_iNKHQSO4QYwKtvv8w,3296
+postgres_fts_backend/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+postgres_fts_backend-0.0.1.dist-info/licenses/LICENSE,sha256=tLxqA1coRYSQ1uk2S_CuxAb-5Pxr3uJAVx3_BbWWTOQ,1084
+postgres_fts_backend-0.0.1.dist-info/METADATA,sha256=bBQEQb_wHZlQq0QtaMXq5Cvr0dJApGBEkSIMahbARM0,2688
+postgres_fts_backend-0.0.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+postgres_fts_backend-0.0.1.dist-info/top_level.txt,sha256=o2Bn_Q4mThOCsDP4O-3P2HR3L03fuZnB6hwDF-ywmGQ,21
+postgres_fts_backend-0.0.1.dist-info/RECORD,,

postgres_fts_backend-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

postgres_fts_backend-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Forest Gregg, 2024 Datamade
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

postgres_fts_backend-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+postgres_fts_backend