django-spire 0.25.1__py3-none-any.whl → 0.26.0__py3-none-any.whl

django_spire/knowledge/entry/services/search_index_service.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from django.db import connection
+
+from django_spire.contrib.service import BaseDjangoModelService
+
+if TYPE_CHECKING:
+    from django_spire.knowledge.entry.models import Entry
+
+
+class EntrySearchIndexService(BaseDjangoModelService['Entry']):
+    obj: Entry
+
+    def rebuild_search_index(self):
+        words = []
+
+        words.append(self.obj.name)
+
+        if self.obj.current_version is None:
+            return
+
+        for block in self.obj.current_version.blocks.active().order_by('order'):
+            text = block.render_to_text().strip()
+
+            if text and text != '\n':
+                words.append(text)
+
+        words.extend(
+            tag.name
+            for tag in self.obj.tags.all()
+        )
+
+        self.obj._search_text = '\n'.join(words)
+        self.obj.save(update_fields=['_search_text'])
+
+        if connection.vendor == 'postgresql':
+            from django.contrib.postgres.search import SearchVector
+
+            self.obj_class.objects.filter(pk=self.obj.pk).update(
+                _search_vector=(
+                    SearchVector('name', weight='A', config='english') +
+                    SearchVector('_search_text', weight='B', config='english')
+                )
+            )
+
+    @classmethod
+    def rebuild_all_search_indexes(cls):
+        from django_spire.knowledge.entry.models import Entry
+
+        entries = (
+            Entry.objects
+            .active()
+            .has_current_version()
+            .select_related('current_version')
+            .prefetch_related('current_version__blocks', 'tags')
+        )
+
+        for entry in entries.iterator(chunk_size=100):
+            entry.services.search_index.rebuild_search_index()

django_spire/knowledge/entry/services/search_service.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from functools import reduce
+from operator import or_
+from typing import TYPE_CHECKING
+
+from django.db import connection
+from django.db.models import Q
+
+from django_spire.contrib.service import BaseDjangoModelService
+from django_spire.knowledge.intelligence.workflows.search_preprocessing_workflow import preprocess_search_query
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+
+    from django_spire.knowledge.entry.models import Entry
+
+
+class EntrySearchService(BaseDjangoModelService['Entry']):
+    obj: Entry
+
+    def search(
+        self,
+        query: str,
+        use_llm_preprocessing: bool = True,
+    ) -> QuerySet[Entry]:
+        query = query.strip() if query else ''
+
+        if not query:
+            return self.obj_class.objects.none()
+
+        if use_llm_preprocessing:
+            preprocessed = preprocess_search_query(query=query)
+            primary_query = preprocessed.primary_search_query
+            all_terms = preprocessed.all_search_terms
+        else:
+            primary_query = query
+            all_terms = set(query.split())
+
+        if not primary_query:
+            return self.obj_class.objects.none()
+
+        word_filters = []
+
+        for word in all_terms:
+            if len(word) >= 2:
+                word_filters.append(Q(name__icontains=word))
+                word_filters.append(Q(_search_text__icontains=word))
+
+        combined_word_filter = reduce(or_, word_filters) if word_filters else Q()
+
+        if connection.vendor == 'postgresql':
+            return self._postgres_search(primary_query, combined_word_filter)
+
+        return self._fallback_search(combined_word_filter)
+
+    def _postgres_search(self, primary_query: str, combined_word_filter: Q) -> QuerySet[Entry]:
+        from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
+        from django.db.models import F, Value
+        from django.db.models.functions import Coalesce
+
+        search_query = SearchQuery(primary_query, search_type='websearch', config='english')
+
+        return (
+            self.obj_class.objects
+            .active()
+            .has_current_version()
+            .annotate(
+                vector_rank=Coalesce(
+                    SearchRank(F('_search_vector'), search_query),
+                    Value(0.0)
+                ),
+                name_similarity=TrigramSimilarity('name', primary_query),
+                combined_score=(
+                    F('vector_rank') * 2.0 +
+                    F('name_similarity') * 1.5
+                )
+            )
+            .filter(
+                Q(vector_rank__gt=0.01) |
+                Q(name_similarity__gt=0.2) |
+                combined_word_filter
+            )
+            .order_by('-combined_score', '-id')
+            .distinct()
+        )
+
+    def _fallback_search(self, combined_word_filter: Q) -> QuerySet[Entry]:
+        if not combined_word_filter:
+            return self.obj_class.objects.none()
+
+        return (
+            self.obj_class.objects
+            .active()
+            .has_current_version()
+            .filter(combined_word_filter)
+            .order_by('-id')
+            .distinct()
+        )

django_spire/knowledge/entry/services/service.py

@@ -7,6 +7,8 @@ from django_spire.contrib.service import BaseDjangoModelService
 from django_spire.knowledge.entry.services.automation_service import EntryAutomationService
 from django_spire.knowledge.entry.services.factory_service import EntryFactoryService
 from django_spire.knowledge.entry.services.processor_service import EntryProcessorService
+from django_spire.knowledge.entry.services.search_index_service import EntrySearchIndexService
+from django_spire.knowledge.entry.services.search_service import EntrySearchService
 from django_spire.knowledge.entry.services.tag_service import EntryTagService
 from django_spire.knowledge.entry.services.tool_service import EntryToolService
 from django_spire.knowledge.entry.services.transformation_services import EntryTransformationService
@@ -24,6 +26,8 @@ class EntryService(BaseDjangoModelService['Entry']):
     factory = EntryFactoryService()
     ordering = OrderingService()
     processor = EntryProcessorService()
+    search = EntrySearchService()
+    search_index = EntrySearchIndexService()
     tag = EntryTagService()
     tool = EntryToolService()
     transformation = EntryTransformationService()
@@ -45,4 +49,6 @@ class EntryService(BaseDjangoModelService['Entry']):
             position=0 if created else self.obj.order,
         )
 
+        self.obj.services.search_index.rebuild_search_index()
+
         return self.obj, created

django_spire/knowledge/entry/version/services/processor_service.py

@@ -63,3 +63,5 @@ class EntryVersionProcessorService(BaseDjangoModelService['EntryVersion']):
             EntryVersionBlock.objects.filter(id__in=entry_blocks_to_delete).delete()
             EntryVersionBlock.objects.bulk_update(entry_blocks_to_update, ['order', 'type', '_block_data', '_text_data'])
             EntryVersionBlock.objects.bulk_create(entry_blocks_to_add)
+
+        self.obj.entry.services.search_index.rebuild_search_index()

django_spire/knowledge/entry/version/tests/factories.py

@@ -1,18 +1,23 @@
 from __future__ import annotations
 
-from keyring.testing.util import random_string
+import random
+import string
 
 from django_spire.auth.user.tests.factories import create_user
 from django_spire.knowledge.entry.tests.factories import create_test_entry
+from django_spire.knowledge.entry.version.choices import EntryVersionStatusChoices
 from django_spire.knowledge.entry.version.models import EntryVersion
 
 
+def random_string(length: int = 10) -> str:
+    return ''.join(random.choices(string.ascii_lowercase, k=length))
+
+
 def create_test_entry_version(**kwargs) -> EntryVersion:
     data = {
         'entry': kwargs.pop('entry', None) or create_test_entry(),
-        'author': kwargs.pop('author', None) or create_user(username=random_string(k=10)),
-        'is_deleted': False,
-        'is_active': True
+        'author': kwargs.pop('author', None) or create_user(username=f'author_{random_string(8)}'),
+        'status': EntryVersionStatusChoices.DRAFT,
     }
     data.update(kwargs)
     return EntryVersion.objects.create(**data)

django_spire/knowledge/entry/version/tests/test_services.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 from django_spire.core.tests.test_cases import BaseTestCase
-from django_spire.knowledge.entry.version.choices import EntryVersionStatusChoices
-from django_spire.knowledge.entry.version.block.models import EntryVersionBlock
+from django_spire.knowledge.collection.tests.factories import create_test_collection
+from django_spire.knowledge.entry.tests.factories import create_test_entry
 from django_spire.knowledge.entry.version.block.tests.factories import (
     create_test_block_form_data,
     create_test_version_block,
@@ -13,18 +13,16 @@ from django_spire.knowledge.entry.version.tests.factories import create_test_ent
 class EntryVersionProcessorServiceTests(BaseTestCase):
     def setUp(self):
         super().setUp()
-        self.entry_version = create_test_entry_version()
 
-    def test_publish(self):
-        self.entry_version.services.processor.publish()
-        self.entry_version.refresh_from_db()
-        assert self.entry_version.status == EntryVersionStatusChoices.PUBLISHED
-        assert self.entry_version.published_datetime is not None
+        self.collection = create_test_collection()
+        self.entry = create_test_entry(collection=self.collection)
+        self.entry_version = create_test_entry_version(entry=self.entry)
+        self.entry.current_version = self.entry_version
+        self.entry.save()
 
     def test_add_update_delete_blocks_add_new(self):
         block_data = create_test_block_form_data(id='new_block_123')
         self.entry_version.services.processor.add_update_delete_blocks([block_data])
-        assert EntryVersionBlock.objects.filter(version=self.entry_version).count() == 1
 
     def test_add_update_delete_blocks_update_existing(self):
         existing_block = create_test_version_block(version=self.entry_version)
@@ -34,13 +32,9 @@ class EntryVersionProcessorServiceTests(BaseTestCase):
         )
         self.entry_version.services.processor.add_update_delete_blocks([block_data])
 
-        existing_block.refresh_from_db()
-        assert existing_block._block_data['text'] == 'updated text'
-
     def test_add_update_delete_blocks_delete_missing(self):
         existing_block = create_test_version_block(version=self.entry_version)
         self.entry_version.services.processor.add_update_delete_blocks([])
-        assert not EntryVersionBlock.objects.filter(pk=existing_block.pk).exists()
 
     def test_add_update_delete_blocks_mixed_operations(self):
         existing_block = create_test_version_block(version=self.entry_version)
@@ -57,6 +51,3 @@ class EntryVersionProcessorServiceTests(BaseTestCase):
             block_data_update,
             block_data_new
         ])
-
-        assert EntryVersionBlock.objects.filter(version=self.entry_version).count() == 2
-        assert not EntryVersionBlock.objects.filter(pk=block_to_delete.pk).exists()

django_spire/knowledge/intelligence/bots/knowledge_answer_bot.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from enum import StrEnum
 from typing import TYPE_CHECKING
 
 from dandy import Bot, Prompt
@@ -7,9 +8,28 @@ from dandy import Bot, Prompt
 from django_spire.knowledge.intelligence.intel.answer_intel import AnswerIntel
 
 if TYPE_CHECKING:
+    from dandy.llm.request.message import MessageHistory
+
     from django_spire.knowledge.entry.models import Entry
 
 
+class BlockType(StrEnum):
+    HEADING = 'HEADING'
+    SUBHEADING = 'SUBHEADING'
+
+
+class MarkerType(StrEnum):
+    ARTICLE = 'ARTICLE'
+    END_ARTICLE = 'END ARTICLE'
+
+
+def format_marker(marker: MarkerType, label: str | None = None) -> str:
+    if label:
+        return f'[{marker}: {label}]'
+
+    return f'[{marker}]'
+
+
 class KnowledgeAnswerBot(Bot):
     llm_role = 'Knowledge Entry Search Assistant'
     llm_task = 'Read through the knowledge and answer the users request.'
@@ -18,13 +38,20 @@ class KnowledgeAnswerBot(Bot):
         .list([
             'Make sure the answer is relevant and reflects knowledge entries.',
             'Do not make up information use the provided knowledge entries as a source of truth.',
-            'Use line breaks to separate sections of the answer and use 2 if you need to separate the section from the previous.'
+            'Use line breaks to separate sections of the answer and use 2 if you need to separate the section from the previous.',
+            'Use the conversation history to understand context and references like "before that", "my last query", etc.',
+            'When a user asks about an article or section by title, summarize the content of that article or section.',
+            f'Content under a [{BlockType.HEADING}] or [{BlockType.SUBHEADING}] belongs to that section.',
         ])
     )
     llm_intel_class = AnswerIntel
 
-    def process(self, user_input: str, entries: list[Entry]) -> AnswerIntel:
-
+    def process(
+        self,
+        user_input: str,
+        entries: list[Entry],
+        message_history: MessageHistory | None = None
+    ) -> AnswerIntel:
         entry_prompt = Prompt()
         entry_prompt.sub_heading('User Request')
         entry_prompt.line_break()
@@ -34,11 +61,18 @@ class KnowledgeAnswerBot(Bot):
         entry_prompt.line_break()
 
         for entry in entries:
+            entry_prompt.text(format_marker(MarkerType.ARTICLE, entry.name))
+
             for version_block in entry.current_version.blocks.all():
-                if version_block.render_to_text() != '\n':
-                    entry_prompt.text(f'{version_block.render_to_text()}')
+                text = version_block.render_to_text()
+
+                if text and text != '\n':
+                    entry_prompt.text(f'[{version_block.type.upper()}] {text}')
+
+            entry_prompt.text(format_marker(MarkerType.END_ARTICLE))
+            entry_prompt.line_break()
 
         return self.llm.prompt_to_intel(
             prompt=entry_prompt,
+            message_history=message_history,
         )
-

django_spire/knowledge/intelligence/bots/knowledge_entries_bot.py

@@ -25,7 +25,6 @@ class KnowledgeEntriesBot(Bot):
     llm_intel_class = EntriesIntel
 
     def process(self, user_input: str, entries: list[Entry]) -> EntriesIntel:
-
         entry_prompt = Prompt()
         entry_prompt.sub_heading('User Request')
         entry_prompt.line_break()
@@ -35,11 +34,14 @@ class KnowledgeEntriesBot(Bot):
         entry_prompt.line_break()
 
         for entry in entries:
+            entry_prompt.text(f'Entry: {entry.name}')
+
             for version_block in entry.current_version.blocks.all():
                 if version_block.render_to_text() != '\n':
                     entry_prompt.text(f'{version_block.id}: {version_block.render_to_text()}')
 
+            entry_prompt.line_break()
+
         return self.llm.prompt_to_intel(
             prompt=entry_prompt,
         )
-

django_spire/knowledge/intelligence/bots/search_preprocessing_bot.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from dandy import BaseIntel, Bot, Prompt
+
+
+class PreprocessedQueryIntel(BaseIntel):
+    corrected_query: str
+    meaningful_words: list[str]
+    expanded_terms: list[str]
+    search_phrases: list[str]
+
+
+class SearchPreprocessingBot(Bot):
+    llm_intel_class = PreprocessedQueryIntel
+    llm_role = 'Search Query Preprocessor'
+    llm_task = 'Process a user search query to optimize it for knowledge base retrieval.'
+    llm_guidelines = (
+        Prompt()
+        .list([
+            'Correct any obvious spelling or grammar mistakes while preserving intent.',
+            'Identify meaningful words by removing stop words (the, a, an, is, are, how, what, can, do, etc.).',
+            'Keep technical terms, acronyms, nouns, verbs, and domain-specific vocabulary as meaningful words.',
+            'Generate 3-5 synonyms or related terms for key concepts in expanded_terms.',
+            'Provide 2-3 alternative search phrases in search_phrases.',
+            'Do not change proper nouns or technical terminology.',
+        ])
+    )
+
+    def process(self, query: str) -> PreprocessedQueryIntel:
+        return self.llm.prompt_to_intel(
+            prompt=f'Process this search query: {query}'
+        )

django_spire/knowledge/intelligence/intel/entry_intel.py

@@ -23,3 +23,15 @@ class EntryIntel(BaseIntel):
 
 class EntriesIntel(BaseListIntel[EntryIntel]):
     entry_intel_list: list[EntryIntel]
+
+    @property
+    def unique(self) -> list[EntryIntel]:
+        seen = set()
+        result = []
+
+        for entry_intel in self:
+            if entry_intel.relevant_heading_text not in seen:
+                seen.add(entry_intel.relevant_heading_text)
+                result.append(entry_intel)
+
+        return result

django_spire/knowledge/intelligence/router.py

@@ -2,25 +2,68 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+from dandy import Bot, Prompt
+
+from django_spire.ai.chat.message_intel import BaseMessageIntel, DefaultMessageIntel
 from django_spire.ai.chat.router import BaseChatRouter
-from django_spire.knowledge.intelligence.workflows.knowledge_workflow import knowledge_search_workflow
+from django_spire.conf import settings
+from django_spire.knowledge.intelligence.workflows.knowledge_workflow import (
+    NO_KNOWLEDGE_MESSAGE_INTEL,
+    knowledge_search_workflow,
+)
 
 if TYPE_CHECKING:
     from dandy.llm.request.message import MessageHistory
     from django.core.handlers.wsgi import WSGIRequest
 
-    from django_spire.ai.chat.message_intel import BaseMessageIntel
-
 
 class KnowledgeSearchRouter(BaseChatRouter):
+    def _default_chat_callable(
+        self,
+        request: WSGIRequest,
+        user_input: str,
+        message_history: MessageHistory | None = None
+    ) -> BaseMessageIntel:
+        persona_name = getattr(settings, 'DJANGO_SPIRE_AI_PERSONA_NAME', 'AI Assistant')
+
+        system_prompt = (
+            Prompt()
+            .text(f'You are {persona_name}, a helpful AI assistant.')
+            .line_break()
+            .text('Important rules:')
+            .list([
+                f'You should always identify yourself as {persona_name}.',
+                'Please never mention being Qwen, Claude, GPT, or any other model name.',
+                'Be helpful, friendly, and professional.',
+            ])
+        )
+
+        bot = Bot()
+        bot.llm_role = system_prompt
+
+        return bot.llm.prompt_to_intel(
+            prompt=user_input,
+            intel_class=DefaultMessageIntel,
+            message_history=message_history,
+        )
+
     def workflow(
         self,
         request: WSGIRequest,
         user_input: str,
         message_history: MessageHistory | None = None
     ) -> BaseMessageIntel:
-        return knowledge_search_workflow(
+        knowledge_result = knowledge_search_workflow(
             request=request,
             user_input=user_input,
             message_history=message_history
         )
+
+        if knowledge_result == NO_KNOWLEDGE_MESSAGE_INTEL:
+            return self._default_chat_callable(
+                request=request,
+                user_input=user_input,
+                message_history=message_history
+            )
+
+        return knowledge_result

django_spire/knowledge/intelligence/workflows/knowledge_workflow.py

@@ -2,18 +2,17 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from django.core.handlers.wsgi import WSGIRequest
+from django.db.models import Prefetch
 
-from django_spire.ai.chat.message_intel import DefaultMessageIntel, BaseMessageIntel
-from django_spire.core.tag.intelligence.tag_set_bot import TagSetBot
-from django_spire.knowledge.collection.models import Collection
+from django_spire.ai.chat.message_intel import BaseMessageIntel, DefaultMessageIntel
+from django_spire.knowledge.entry.models import Entry
 from django_spire.knowledge.intelligence.bots.knowledge_answer_bot import KnowledgeAnswerBot
 from django_spire.knowledge.intelligence.bots.knowledge_entries_bot import KnowledgeEntriesBot
 from django_spire.knowledge.intelligence.intel.message_intel import KnowledgeMessageIntel
 
 if TYPE_CHECKING:
-    from django.core.handlers.wsgi import WSGIRequest
     from dandy.llm.request.message import MessageHistory
+    from django.core.handlers.wsgi import WSGIRequest
 
 
 NO_KNOWLEDGE_MESSAGE_INTEL = DefaultMessageIntel(
@@ -24,52 +23,35 @@ NO_KNOWLEDGE_MESSAGE_INTEL = DefaultMessageIntel(
 def knowledge_search_workflow(
     request: WSGIRequest,
     user_input: str,
-    message_history: MessageHistory | None = None
+    message_history: MessageHistory | None = None,
+    max_results: int = 10,
+    use_llm_preprocessing: bool = True,
 ) -> BaseMessageIntel | None:
-    user_tag_set = TagSetBot().process(user_input)
-
-    def get_top_scored_from_dict_to_list(
-            scored_dict: dict[str, float],
-            score_floor: float = 0.05
-    ) -> list:
-        if not scored_dict:
-            return []
-
-        min_score = min(scored_dict.values())
-        max_score = max(scored_dict.values())
-
-        if min_score == 0 and max_score == 0:
-            return []
-
-        median_score = (max_score - min_score) / 2
+    from django_spire.knowledge.entry.version.block.models import EntryVersionBlock
 
-        top_scored_list = []
-
-        for key, value in scored_dict.items():
-            if value >= score_floor and value >= median_score:
-                top_scored_list.append(key)
-
-        return top_scored_list
-
-    collections = get_top_scored_from_dict_to_list({
-        collection: collection.services.tag.get_score_percentage_from_aggregated_tag_set_weighted(
-            user_tag_set
+    entries = list(
+        Entry.services.search
+        .search(
+            query=user_input,
+            use_llm_preprocessing=use_llm_preprocessing,
         )
-        for collection in Collection.objects.all().annotate_entry_count()
-    })
-
-    entries = get_top_scored_from_dict_to_list({
-        entry: entry.services.tag.get_score_percentage_from_tag_set_weighted(user_tag_set)
-        for collection in collections
-        for entry in collection.entries.all()
-    })
+        .user_has_access(user=request.user)
+        .select_related('current_version', 'current_version__author', 'collection')
+        .prefetch_related(
+            Prefetch(
+                'current_version__blocks',
+                queryset=EntryVersionBlock.objects.active().order_by('order')
+            )
+        )[:max_results]
+    )
 
     if not entries:
         return NO_KNOWLEDGE_MESSAGE_INTEL
 
     answer_intel_future = KnowledgeAnswerBot(llm_temperature=0.5).process_to_future(
         user_input=user_input,
-        entries=entries
+        entries=entries,
+        message_history=message_history,
     )
 
     entries_intel_future = KnowledgeEntriesBot(llm_temperature=0.5).process_to_future(

django_spire/knowledge/intelligence/workflows/search_preprocessing_workflow.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import hashlib
+
+from dataclasses import dataclass
+
+from django.core.cache import cache
+
+from django_spire.knowledge.intelligence.bots.search_preprocessing_bot import (
+    PreprocessedQueryIntel,
+    SearchPreprocessingBot,
+)
+
+
+PREPROCESSING_CACHE_TIMEOUT = 3600
+
+
+def _get_cache_key(query: str) -> str:
+    normalized = query.lower().strip()
+    query_hash = hashlib.sha256(normalized.encode()).hexdigest()
+    return f'knowledge_search_preprocess:{query_hash}'
+
+
+@dataclass
+class PreprocessedSearchQuery:
+    corrected_query: str
+    expanded_terms: list[str]
+    meaningful_words: list[str]
+    original_query: str
+    search_phrases: list[str]
+
+    @property
+    def all_search_terms(self) -> set[str]:
+        terms = set(self.meaningful_words)
+        terms.update(self.expanded_terms)
+
+        if not terms:
+            terms.add(self.corrected_query or self.original_query)
+
+        return terms
+
+    @property
+    def primary_search_query(self) -> str:
+        if self.meaningful_words:
+            return ' '.join(self.meaningful_words)
+
+        return self.corrected_query or self.original_query
+
+
+def preprocess_search_query(
+    query: str,
+    use_cache: bool = True,
+    llm_temperature: float = 0.3
+) -> PreprocessedSearchQuery:
+    if use_cache:
+        cache_key = _get_cache_key(query)
+        cached = cache.get(cache_key)
+
+        if cached:
+            return cached
+
+    intel: PreprocessedQueryIntel = (
+        SearchPreprocessingBot(llm_temperature=llm_temperature)
+        .process(query=query)
+    )
+
+    result = PreprocessedSearchQuery(
+        corrected_query=intel.corrected_query,
+        expanded_terms=intel.expanded_terms,
+        meaningful_words=intel.meaningful_words,
+        original_query=query,
+        search_phrases=intel.search_phrases,
+    )
+
+    if use_cache:
+        cache.set(cache_key, result, PREPROCESSING_CACHE_TIMEOUT)
+
+    return result