django-spire 0.25.1__py3-none-any.whl → 0.25.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_spire/ai/chat/router.py +10 -20
- django_spire/ai/chat/templates/django_spire/ai/chat/dropdown/ellipsis_dropdown.html +5 -3
- django_spire/ai/chat/templates/django_spire/ai/chat/message/default_message.html +6 -2
- django_spire/ai/chat/templatetags/__init__.py +0 -0
- django_spire/ai/chat/templatetags/spire_ai_chat_tags.py +19 -0
- django_spire/ai/chat/tests/test_router/test_spire_chat_router.py +20 -88
- django_spire/consts.py +1 -1
- django_spire/contrib/progress/session.py +1 -1
- django_spire/core/middleware.py +2 -3
- django_spire/core/static/django_spire/js/theme.js +10 -7
- django_spire/knowledge/entry/forms.py +1 -1
- django_spire/knowledge/entry/models.py +18 -0
- django_spire/knowledge/entry/querysets.py +8 -6
- django_spire/knowledge/entry/services/processor_service.py +1 -0
- django_spire/knowledge/entry/services/search_index_service.py +61 -0
- django_spire/knowledge/entry/services/search_service.py +99 -0
- django_spire/knowledge/entry/services/service.py +6 -0
- django_spire/knowledge/entry/version/services/processor_service.py +2 -0
- django_spire/knowledge/entry/version/tests/factories.py +9 -4
- django_spire/knowledge/entry/version/tests/test_services.py +7 -16
- django_spire/knowledge/intelligence/bots/knowledge_answer_bot.py +40 -6
- django_spire/knowledge/intelligence/bots/knowledge_entries_bot.py +4 -2
- django_spire/knowledge/intelligence/bots/search_preprocessing_bot.py +32 -0
- django_spire/knowledge/intelligence/intel/entry_intel.py +12 -0
- django_spire/knowledge/intelligence/router.py +47 -4
- django_spire/knowledge/intelligence/workflows/knowledge_workflow.py +24 -42
- django_spire/knowledge/intelligence/workflows/search_preprocessing_workflow.py +78 -0
- django_spire/knowledge/management/__init__.py +0 -0
- django_spire/knowledge/management/commands/__init__.py +0 -0
- django_spire/knowledge/management/commands/rebuild_knowledge_search_index.py +16 -0
- django_spire/knowledge/migrations/0010_entry__search_text_entry__search_vector_and_more.py +40 -0
- django_spire/knowledge/templates/django_spire/knowledge/message/knowledge_message_intel.html +31 -23
- django_spire/metric/report/enums.py +11 -5
- django_spire/metric/report/report.py +24 -12
- django_spire/metric/report/tools.py +14 -2
- django_spire/testing/playwright/fixtures.py +4 -5
- {django_spire-0.25.1.dist-info → django_spire-0.25.2.dist-info}/METADATA +1 -1
- {django_spire-0.25.1.dist-info → django_spire-0.25.2.dist-info}/RECORD +41 -31
- {django_spire-0.25.1.dist-info → django_spire-0.25.2.dist-info}/WHEEL +0 -0
- {django_spire-0.25.1.dist-info → django_spire-0.25.2.dist-info}/licenses/LICENSE.md +0 -0
- {django_spire-0.25.1.dist-info → django_spire-0.25.2.dist-info}/top_level.txt +0 -0
django_spire/ai/chat/router.py
CHANGED
|
@@ -3,13 +3,11 @@ from __future__ import annotations
|
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
4
|
from typing import TYPE_CHECKING
|
|
5
5
|
|
|
6
|
-
from dandy import Bot, Prompt
|
|
7
6
|
from dandy.recorder import recorder_to_html_file
|
|
8
7
|
|
|
9
8
|
from django_spire.ai.chat.intelligence.decoders.intent_decoder import generate_intent_decoder
|
|
10
9
|
from django_spire.ai.chat.message_intel import BaseMessageIntel, DefaultMessageIntel
|
|
11
10
|
from django_spire.ai.decorators import log_ai_interaction_from_recorder
|
|
12
|
-
from django_spire.conf import settings
|
|
13
11
|
|
|
14
12
|
if TYPE_CHECKING:
|
|
15
13
|
from dandy.llm.request.message import MessageHistory
|
|
@@ -62,27 +60,19 @@ class SpireChatRouter(BaseChatRouter):
|
|
|
62
60
|
user_input: str,
|
|
63
61
|
message_history: MessageHistory | None = None
|
|
64
62
|
) -> BaseMessageIntel:
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
system_prompt = (
|
|
68
|
-
Prompt()
|
|
69
|
-
.text(f'You are {persona_name}, a helpful AI assistant.')
|
|
70
|
-
.line_break()
|
|
71
|
-
.text('Important rules:')
|
|
72
|
-
.list([
|
|
73
|
-
f'You should always identify yourself as {persona_name}.',
|
|
74
|
-
'Please never mention being Qwen, Claude, GPT, or any other model name.',
|
|
75
|
-
'Be helpful, friendly, and professional.',
|
|
76
|
-
])
|
|
63
|
+
from django_spire.knowledge.intelligence.workflows.knowledge_workflow import (
|
|
64
|
+
knowledge_search_workflow,
|
|
77
65
|
)
|
|
78
66
|
|
|
79
|
-
|
|
80
|
-
|
|
67
|
+
if request.user.has_perm('django_spire_knowledge.view_collection'):
|
|
68
|
+
return knowledge_search_workflow(
|
|
69
|
+
request=request,
|
|
70
|
+
user_input=user_input,
|
|
71
|
+
message_history=message_history
|
|
72
|
+
)
|
|
81
73
|
|
|
82
|
-
return
|
|
83
|
-
|
|
84
|
-
intel_class=DefaultMessageIntel,
|
|
85
|
-
message_history=message_history,
|
|
74
|
+
return DefaultMessageIntel(
|
|
75
|
+
text='Sorry, I could not find any information on that.'
|
|
86
76
|
)
|
|
87
77
|
|
|
88
78
|
def workflow(
|
|
@@ -11,8 +11,10 @@
|
|
|
11
11
|
{% endblock %}
|
|
12
12
|
|
|
13
13
|
{% block dropdown_content %}
|
|
14
|
-
{%
|
|
14
|
+
{% if recent_chat %}
|
|
15
|
+
{% include 'django_spire/dropdown/element/dropdown_link_element.html' with x_click="enable_rename_chat(); toggle_dropdown()" link_text='Rename' link_css='text-start' %}
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
17
|
+
{% url 'django_spire:ai:chat:template:confirm_delete' pk=recent_chat.id as delete_url %}
|
|
18
|
+
{% include 'django_spire/dropdown/element/ellipsis_dropdown_modal_link_element.html' with view_url=delete_url link_text='Delete' link_css='text-app-danger text-start' %}
|
|
19
|
+
{% endif %}
|
|
18
20
|
{% endblock %}
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
{% extends 'django_spire/ai/chat/message/message.html' %}
|
|
2
2
|
|
|
3
|
+
{% load spire_ai_chat_tags %}
|
|
4
|
+
|
|
3
5
|
{% block message_content %}
|
|
4
|
-
|
|
5
|
-
{
|
|
6
|
+
<div class="ai-message-content">
|
|
7
|
+
{{ message_intel.text|render_markdown }}
|
|
8
|
+
</div>
|
|
9
|
+
{% endblock %}
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import marko
|
|
4
|
+
|
|
5
|
+
from django import template
|
|
6
|
+
from django.utils.safestring import mark_safe
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
register = template.Library()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@register.filter
|
|
13
|
+
def render_markdown(value: str) -> str:
|
|
14
|
+
if not value:
|
|
15
|
+
return ''
|
|
16
|
+
|
|
17
|
+
html = marko.convert(value)
|
|
18
|
+
|
|
19
|
+
return mark_safe(html)
|
|
@@ -23,13 +23,26 @@ class TestSpireChatRouter(BaseTestCase):
|
|
|
23
23
|
|
|
24
24
|
assert isinstance(router, SpireChatRouter)
|
|
25
25
|
|
|
26
|
-
def
|
|
26
|
+
def test_default_chat_callable_returns_default_without_permission(self) -> None:
|
|
27
27
|
router = SpireChatRouter()
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
self.request.user = Mock()
|
|
30
|
+
self.request.user.has_perm.return_value = False
|
|
31
|
+
|
|
32
|
+
result = router._default_chat_callable(
|
|
33
|
+
request=self.request,
|
|
34
|
+
user_input='Hello',
|
|
35
|
+
message_history=None
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
assert isinstance(result, DefaultMessageIntel)
|
|
39
|
+
assert result.text == 'Sorry, I could not find any information on that.'
|
|
40
|
+
|
|
41
|
+
def test_default_chat_callable_calls_knowledge_workflow_with_permission(self) -> None:
|
|
42
|
+
router = SpireChatRouter()
|
|
43
|
+
|
|
44
|
+
with patch('django_spire.knowledge.intelligence.workflows.knowledge_workflow.knowledge_search_workflow') as mock_workflow:
|
|
45
|
+
mock_workflow.return_value = DefaultMessageIntel(text='Knowledge response')
|
|
33
46
|
|
|
34
47
|
result = router._default_chat_callable(
|
|
35
48
|
request=self.request,
|
|
@@ -37,25 +50,13 @@ class TestSpireChatRouter(BaseTestCase):
|
|
|
37
50
|
message_history=None
|
|
38
51
|
)
|
|
39
52
|
|
|
40
|
-
|
|
41
|
-
mock_bot_instance.llm.prompt_to_intel.assert_called_once()
|
|
42
|
-
|
|
43
|
-
@override_settings(DJANGO_SPIRE_AI_PERSONA_NAME='Test Bot')
|
|
44
|
-
def test_default_callable_uses_persona_name(self) -> None:
|
|
45
|
-
router = SpireChatRouter()
|
|
46
|
-
|
|
47
|
-
with patch('django_spire.ai.chat.router.Bot') as MockBot:
|
|
48
|
-
mock_bot_instance = Mock()
|
|
49
|
-
MockBot.return_value = mock_bot_instance
|
|
50
|
-
mock_bot_instance.llm.prompt_to_intel.return_value = DefaultMessageIntel(text='Response')
|
|
51
|
-
|
|
52
|
-
router._default_chat_callable(
|
|
53
|
+
mock_workflow.assert_called_once_with(
|
|
53
54
|
request=self.request,
|
|
54
55
|
user_input='Hello',
|
|
55
56
|
message_history=None
|
|
56
57
|
)
|
|
57
58
|
|
|
58
|
-
assert
|
|
59
|
+
assert isinstance(result, DefaultMessageIntel)
|
|
59
60
|
|
|
60
61
|
def test_workflow_uses_intent_decoder(self) -> None:
|
|
61
62
|
router = SpireChatRouter()
|
|
@@ -131,72 +132,3 @@ class TestSpireChatRouter(BaseTestCase):
|
|
|
131
132
|
|
|
132
133
|
assert isinstance(result, DefaultMessageIntel)
|
|
133
134
|
assert result.text == 'Test Response'
|
|
134
|
-
|
|
135
|
-
@override_settings(DJANGO_SPIRE_AI_PERSONA_NAME='Custom Persona')
|
|
136
|
-
def test_default_callable_with_custom_persona(self) -> None:
|
|
137
|
-
router = SpireChatRouter()
|
|
138
|
-
|
|
139
|
-
with patch('django_spire.ai.chat.router.Bot') as MockBot:
|
|
140
|
-
mock_bot_instance = Mock()
|
|
141
|
-
MockBot.return_value = mock_bot_instance
|
|
142
|
-
mock_bot_instance.llm.prompt_to_intel.return_value = DefaultMessageIntel(text='Response')
|
|
143
|
-
|
|
144
|
-
router._default_chat_callable(
|
|
145
|
-
request=self.request,
|
|
146
|
-
user_input='Hello',
|
|
147
|
-
message_history=None
|
|
148
|
-
)
|
|
149
|
-
|
|
150
|
-
assert mock_bot_instance.llm_role is not None
|
|
151
|
-
|
|
152
|
-
def test_default_callable_passes_message_history(self) -> None:
|
|
153
|
-
router = SpireChatRouter()
|
|
154
|
-
message_history = MessageHistory()
|
|
155
|
-
|
|
156
|
-
with patch('django_spire.ai.chat.router.Bot') as MockBot:
|
|
157
|
-
mock_bot_instance = Mock()
|
|
158
|
-
MockBot.return_value = mock_bot_instance
|
|
159
|
-
mock_bot_instance.llm.prompt_to_intel.return_value = DefaultMessageIntel(text='Response')
|
|
160
|
-
|
|
161
|
-
router._default_chat_callable(
|
|
162
|
-
request=self.request,
|
|
163
|
-
user_input='Hello',
|
|
164
|
-
message_history=message_history
|
|
165
|
-
)
|
|
166
|
-
|
|
167
|
-
call_kwargs = mock_bot_instance.llm.prompt_to_intel.call_args[1]
|
|
168
|
-
assert call_kwargs['message_history'] == message_history
|
|
169
|
-
|
|
170
|
-
def test_default_callable_passes_user_input(self) -> None:
|
|
171
|
-
router = SpireChatRouter()
|
|
172
|
-
|
|
173
|
-
with patch('django_spire.ai.chat.router.Bot') as MockBot:
|
|
174
|
-
mock_bot_instance = Mock()
|
|
175
|
-
MockBot.return_value = mock_bot_instance
|
|
176
|
-
mock_bot_instance.llm.prompt_to_intel.return_value = DefaultMessageIntel(text='Response')
|
|
177
|
-
|
|
178
|
-
router._default_chat_callable(
|
|
179
|
-
request=self.request,
|
|
180
|
-
user_input='Test input',
|
|
181
|
-
message_history=None
|
|
182
|
-
)
|
|
183
|
-
|
|
184
|
-
call_kwargs = mock_bot_instance.llm.prompt_to_intel.call_args[1]
|
|
185
|
-
assert call_kwargs['prompt'] == 'Test input'
|
|
186
|
-
|
|
187
|
-
def test_default_callable_uses_default_message_intel(self) -> None:
|
|
188
|
-
router = SpireChatRouter()
|
|
189
|
-
|
|
190
|
-
with patch('django_spire.ai.chat.router.Bot') as MockBot:
|
|
191
|
-
mock_bot_instance = Mock()
|
|
192
|
-
MockBot.return_value = mock_bot_instance
|
|
193
|
-
mock_bot_instance.llm.prompt_to_intel.return_value = DefaultMessageIntel(text='Response')
|
|
194
|
-
|
|
195
|
-
router._default_chat_callable(
|
|
196
|
-
request=self.request,
|
|
197
|
-
user_input='Hello',
|
|
198
|
-
message_history=None
|
|
199
|
-
)
|
|
200
|
-
|
|
201
|
-
call_kwargs = mock_bot_instance.llm.prompt_to_intel.call_args[1]
|
|
202
|
-
assert call_kwargs['intel_class'] == DefaultMessageIntel
|
django_spire/consts.py
CHANGED
|
@@ -13,7 +13,7 @@ from django_spire.contrib.progress.enums import ProgressStatus
|
|
|
13
13
|
from django_spire.contrib.progress.tasks import ParallelTask, SequentialTask
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
|
-
from
|
|
16
|
+
from typing import Any, Callable, Generator
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
SIMULATION_MESSAGES = [
|
django_spire/core/middleware.py
CHANGED
|
@@ -7,9 +7,8 @@ from typing import TYPE_CHECKING, Callable
|
|
|
7
7
|
|
|
8
8
|
# from django_spire.consts import MAINTENANCE_MODE_SETTINGS_NAME
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
from django.http import HttpResponse
|
|
10
|
+
from django.core.handlers.wsgi import WSGIRequest
|
|
11
|
+
from django.http import HttpResponse
|
|
13
12
|
|
|
14
13
|
|
|
15
14
|
class MaintenanceMiddleware:
|
|
@@ -247,16 +247,19 @@ window.get_echarts_theme = function() {
|
|
|
247
247
|
let is_dark = document.documentElement.getAttribute('data-theme') === 'dark';
|
|
248
248
|
|
|
249
249
|
return {
|
|
250
|
-
text: styles.getPropertyValue('--app-default-text-color').trim(),
|
|
251
|
-
primary: styles.getPropertyValue('--app-primary').trim(),
|
|
252
|
-
primary_dark: styles.getPropertyValue('--app-primary-dark').trim(),
|
|
253
|
-
secondary: styles.getPropertyValue('--app-secondary').trim(),
|
|
254
|
-
secondary_dark: styles.getPropertyValue('--app-secondary-dark').trim(),
|
|
255
|
-
border: styles.getPropertyValue('--bs-border-color').trim(),
|
|
256
250
|
bg: styles.getPropertyValue('--app-layer-one').trim(),
|
|
251
|
+
border: styles.getPropertyValue('--bs-border-color').trim(),
|
|
252
|
+
danger: styles.getPropertyValue('--app-danger').trim(),
|
|
253
|
+
is_dark: is_dark,
|
|
257
254
|
layer_two: styles.getPropertyValue('--app-layer-two').trim(),
|
|
258
255
|
layer_three: styles.getPropertyValue('--app-layer-three').trim(),
|
|
259
256
|
layer_four: styles.getPropertyValue('--app-layer-four').trim(),
|
|
260
|
-
|
|
257
|
+
primary: styles.getPropertyValue('--app-primary').trim(),
|
|
258
|
+
primary_dark: styles.getPropertyValue('--app-primary-dark').trim(),
|
|
259
|
+
secondary: styles.getPropertyValue('--app-secondary').trim(),
|
|
260
|
+
secondary_dark: styles.getPropertyValue('--app-secondary-dark').trim(),
|
|
261
|
+
success: styles.getPropertyValue('--app-success').trim(),
|
|
262
|
+
text: styles.getPropertyValue('--app-default-text-color').trim(),
|
|
263
|
+
warning: styles.getPropertyValue('--app-warning').trim(),
|
|
261
264
|
};
|
|
262
265
|
};
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from django.contrib.postgres.indexes import GinIndex
|
|
4
|
+
from django.contrib.postgres.search import SearchVectorField
|
|
3
5
|
from django.db import models
|
|
4
6
|
from django.urls import reverse
|
|
5
7
|
|
|
@@ -36,6 +38,9 @@ class Entry(
|
|
|
36
38
|
|
|
37
39
|
name = models.CharField(max_length=255)
|
|
38
40
|
|
|
41
|
+
_search_text = models.TextField(blank=True, default='')
|
|
42
|
+
_search_vector = SearchVectorField(null=True)
|
|
43
|
+
|
|
39
44
|
objects = EntryQuerySet.as_manager()
|
|
40
45
|
services = EntryService()
|
|
41
46
|
|
|
@@ -75,3 +80,16 @@ class Entry(
|
|
|
75
80
|
verbose_name = 'Entry'
|
|
76
81
|
verbose_name_plural = 'Entries'
|
|
77
82
|
db_table = 'django_spire_knowledge_entry'
|
|
83
|
+
indexes = [
|
|
84
|
+
GinIndex(fields=['_search_vector'], name='entry_search_vector_idx'),
|
|
85
|
+
GinIndex(
|
|
86
|
+
name='entry_name_trgm_idx',
|
|
87
|
+
fields=['name'],
|
|
88
|
+
opclasses=['gin_trgm_ops'],
|
|
89
|
+
),
|
|
90
|
+
GinIndex(
|
|
91
|
+
name='entry_search_text_trgm_idx',
|
|
92
|
+
fields=['_search_text'],
|
|
93
|
+
opclasses=['gin_trgm_ops'],
|
|
94
|
+
),
|
|
95
|
+
]
|
|
@@ -3,17 +3,24 @@ from __future__ import annotations
|
|
|
3
3
|
from typing import TYPE_CHECKING
|
|
4
4
|
|
|
5
5
|
from django.db.models import Q
|
|
6
|
+
|
|
6
7
|
from django_spire.contrib.ordering.querysets import OrderingQuerySetMixin
|
|
7
8
|
from django_spire.history.querysets import HistoryQuerySet
|
|
8
9
|
from django_spire.knowledge.entry.version.choices import EntryVersionStatusChoices
|
|
9
10
|
|
|
10
11
|
if TYPE_CHECKING:
|
|
11
|
-
from django_spire.auth.user.models import AuthUser
|
|
12
12
|
from django.db.models import QuerySet
|
|
13
|
+
|
|
14
|
+
from django_spire.auth.user.models import AuthUser
|
|
13
15
|
from django_spire.knowledge.entry.models import Entry
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
class EntryQuerySet(HistoryQuerySet, OrderingQuerySetMixin):
|
|
19
|
+
def get_by_version_block_id(self, version_block_id: int) -> Entry:
|
|
20
|
+
return self.get(
|
|
21
|
+
current_version__block__id=version_block_id
|
|
22
|
+
)
|
|
23
|
+
|
|
17
24
|
def has_current_version(self) -> QuerySet[Entry]:
|
|
18
25
|
return self.filter(current_version__isnull=False)
|
|
19
26
|
|
|
@@ -30,8 +37,3 @@ class EntryQuerySet(HistoryQuerySet, OrderingQuerySetMixin):
|
|
|
30
37
|
current_version__status=EntryVersionStatusChoices.DRAFT
|
|
31
38
|
)
|
|
32
39
|
)
|
|
33
|
-
|
|
34
|
-
def get_by_version_block_id(self, version_block_id: int) -> Entry:
|
|
35
|
-
return self.get(
|
|
36
|
-
current_version__block__id=version_block_id
|
|
37
|
-
)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from django.db import connection
|
|
6
|
+
|
|
7
|
+
from django_spire.contrib.service import BaseDjangoModelService
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from django_spire.knowledge.entry.models import Entry
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EntrySearchIndexService(BaseDjangoModelService['Entry']):
|
|
14
|
+
obj: Entry
|
|
15
|
+
|
|
16
|
+
def rebuild_search_index(self):
|
|
17
|
+
words = []
|
|
18
|
+
|
|
19
|
+
words.append(self.obj.name)
|
|
20
|
+
|
|
21
|
+
if self.obj.current_version is None:
|
|
22
|
+
return
|
|
23
|
+
|
|
24
|
+
for block in self.obj.current_version.blocks.active().order_by('order'):
|
|
25
|
+
text = block.render_to_text().strip()
|
|
26
|
+
|
|
27
|
+
if text and text != '\n':
|
|
28
|
+
words.append(text)
|
|
29
|
+
|
|
30
|
+
words.extend(
|
|
31
|
+
tag.name
|
|
32
|
+
for tag in self.obj.tags.all()
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
self.obj._search_text = '\n'.join(words)
|
|
36
|
+
self.obj.save(update_fields=['_search_text'])
|
|
37
|
+
|
|
38
|
+
if connection.vendor == 'postgresql':
|
|
39
|
+
from django.contrib.postgres.search import SearchVector
|
|
40
|
+
|
|
41
|
+
self.obj_class.objects.filter(pk=self.obj.pk).update(
|
|
42
|
+
_search_vector=(
|
|
43
|
+
SearchVector('name', weight='A', config='english') +
|
|
44
|
+
SearchVector('_search_text', weight='B', config='english')
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def rebuild_all_search_indexes(cls):
|
|
50
|
+
from django_spire.knowledge.entry.models import Entry
|
|
51
|
+
|
|
52
|
+
entries = (
|
|
53
|
+
Entry.objects
|
|
54
|
+
.active()
|
|
55
|
+
.has_current_version()
|
|
56
|
+
.select_related('current_version')
|
|
57
|
+
.prefetch_related('current_version__blocks', 'tags')
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
for entry in entries.iterator(chunk_size=100):
|
|
61
|
+
entry.services.search_index.rebuild_search_index()
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from functools import reduce
|
|
4
|
+
from operator import or_
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from django.db import connection
|
|
8
|
+
from django.db.models import Q
|
|
9
|
+
|
|
10
|
+
from django_spire.contrib.service import BaseDjangoModelService
|
|
11
|
+
from django_spire.knowledge.intelligence.workflows.search_preprocessing_workflow import preprocess_search_query
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from django.db.models import QuerySet
|
|
15
|
+
|
|
16
|
+
from django_spire.knowledge.entry.models import Entry
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class EntrySearchService(BaseDjangoModelService['Entry']):
|
|
20
|
+
obj: Entry
|
|
21
|
+
|
|
22
|
+
def search(
|
|
23
|
+
self,
|
|
24
|
+
query: str,
|
|
25
|
+
use_llm_preprocessing: bool = True,
|
|
26
|
+
) -> QuerySet[Entry]:
|
|
27
|
+
query = query.strip() if query else ''
|
|
28
|
+
|
|
29
|
+
if not query:
|
|
30
|
+
return self.obj_class.objects.none()
|
|
31
|
+
|
|
32
|
+
if use_llm_preprocessing:
|
|
33
|
+
preprocessed = preprocess_search_query(query=query)
|
|
34
|
+
primary_query = preprocessed.primary_search_query
|
|
35
|
+
all_terms = preprocessed.all_search_terms
|
|
36
|
+
else:
|
|
37
|
+
primary_query = query
|
|
38
|
+
all_terms = set(query.split())
|
|
39
|
+
|
|
40
|
+
if not primary_query:
|
|
41
|
+
return self.obj_class.objects.none()
|
|
42
|
+
|
|
43
|
+
word_filters = []
|
|
44
|
+
|
|
45
|
+
for word in all_terms:
|
|
46
|
+
if len(word) >= 2:
|
|
47
|
+
word_filters.append(Q(name__icontains=word))
|
|
48
|
+
word_filters.append(Q(_search_text__icontains=word))
|
|
49
|
+
|
|
50
|
+
combined_word_filter = reduce(or_, word_filters) if word_filters else Q()
|
|
51
|
+
|
|
52
|
+
if connection.vendor == 'postgresql':
|
|
53
|
+
return self._postgres_search(primary_query, combined_word_filter)
|
|
54
|
+
|
|
55
|
+
return self._fallback_search(combined_word_filter)
|
|
56
|
+
|
|
57
|
+
def _postgres_search(self, primary_query: str, combined_word_filter: Q) -> QuerySet[Entry]:
|
|
58
|
+
from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
|
|
59
|
+
from django.db.models import F, Value
|
|
60
|
+
from django.db.models.functions import Coalesce
|
|
61
|
+
|
|
62
|
+
search_query = SearchQuery(primary_query, search_type='websearch', config='english')
|
|
63
|
+
|
|
64
|
+
return (
|
|
65
|
+
self.obj_class.objects
|
|
66
|
+
.active()
|
|
67
|
+
.has_current_version()
|
|
68
|
+
.annotate(
|
|
69
|
+
vector_rank=Coalesce(
|
|
70
|
+
SearchRank(F('_search_vector'), search_query),
|
|
71
|
+
Value(0.0)
|
|
72
|
+
),
|
|
73
|
+
name_similarity=TrigramSimilarity('name', primary_query),
|
|
74
|
+
combined_score=(
|
|
75
|
+
F('vector_rank') * 2.0 +
|
|
76
|
+
F('name_similarity') * 1.5
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
.filter(
|
|
80
|
+
Q(vector_rank__gt=0.01) |
|
|
81
|
+
Q(name_similarity__gt=0.2) |
|
|
82
|
+
combined_word_filter
|
|
83
|
+
)
|
|
84
|
+
.order_by('-combined_score', '-id')
|
|
85
|
+
.distinct()
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def _fallback_search(self, combined_word_filter: Q) -> QuerySet[Entry]:
|
|
89
|
+
if not combined_word_filter:
|
|
90
|
+
return self.obj_class.objects.none()
|
|
91
|
+
|
|
92
|
+
return (
|
|
93
|
+
self.obj_class.objects
|
|
94
|
+
.active()
|
|
95
|
+
.has_current_version()
|
|
96
|
+
.filter(combined_word_filter)
|
|
97
|
+
.order_by('-id')
|
|
98
|
+
.distinct()
|
|
99
|
+
)
|
|
@@ -7,6 +7,8 @@ from django_spire.contrib.service import BaseDjangoModelService
|
|
|
7
7
|
from django_spire.knowledge.entry.services.automation_service import EntryAutomationService
|
|
8
8
|
from django_spire.knowledge.entry.services.factory_service import EntryFactoryService
|
|
9
9
|
from django_spire.knowledge.entry.services.processor_service import EntryProcessorService
|
|
10
|
+
from django_spire.knowledge.entry.services.search_index_service import EntrySearchIndexService
|
|
11
|
+
from django_spire.knowledge.entry.services.search_service import EntrySearchService
|
|
10
12
|
from django_spire.knowledge.entry.services.tag_service import EntryTagService
|
|
11
13
|
from django_spire.knowledge.entry.services.tool_service import EntryToolService
|
|
12
14
|
from django_spire.knowledge.entry.services.transformation_services import EntryTransformationService
|
|
@@ -24,6 +26,8 @@ class EntryService(BaseDjangoModelService['Entry']):
|
|
|
24
26
|
factory = EntryFactoryService()
|
|
25
27
|
ordering = OrderingService()
|
|
26
28
|
processor = EntryProcessorService()
|
|
29
|
+
search = EntrySearchService()
|
|
30
|
+
search_index = EntrySearchIndexService()
|
|
27
31
|
tag = EntryTagService()
|
|
28
32
|
tool = EntryToolService()
|
|
29
33
|
transformation = EntryTransformationService()
|
|
@@ -45,4 +49,6 @@ class EntryService(BaseDjangoModelService['Entry']):
|
|
|
45
49
|
position=0 if created else self.obj.order,
|
|
46
50
|
)
|
|
47
51
|
|
|
52
|
+
self.obj.services.search_index.rebuild_search_index()
|
|
53
|
+
|
|
48
54
|
return self.obj, created
|
|
@@ -63,3 +63,5 @@ class EntryVersionProcessorService(BaseDjangoModelService['EntryVersion']):
|
|
|
63
63
|
EntryVersionBlock.objects.filter(id__in=entry_blocks_to_delete).delete()
|
|
64
64
|
EntryVersionBlock.objects.bulk_update(entry_blocks_to_update, ['order', 'type', '_block_data', '_text_data'])
|
|
65
65
|
EntryVersionBlock.objects.bulk_create(entry_blocks_to_add)
|
|
66
|
+
|
|
67
|
+
self.obj.entry.services.search_index.rebuild_search_index()
|
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
import random
|
|
4
|
+
import string
|
|
4
5
|
|
|
5
6
|
from django_spire.auth.user.tests.factories import create_user
|
|
6
7
|
from django_spire.knowledge.entry.tests.factories import create_test_entry
|
|
8
|
+
from django_spire.knowledge.entry.version.choices import EntryVersionStatusChoices
|
|
7
9
|
from django_spire.knowledge.entry.version.models import EntryVersion
|
|
8
10
|
|
|
9
11
|
|
|
12
|
+
def random_string(length: int = 10) -> str:
|
|
13
|
+
return ''.join(random.choices(string.ascii_lowercase, k=length))
|
|
14
|
+
|
|
15
|
+
|
|
10
16
|
def create_test_entry_version(**kwargs) -> EntryVersion:
|
|
11
17
|
data = {
|
|
12
18
|
'entry': kwargs.pop('entry', None) or create_test_entry(),
|
|
13
|
-
'author': kwargs.pop('author', None) or create_user(username=random_string(
|
|
14
|
-
'
|
|
15
|
-
'is_active': True
|
|
19
|
+
'author': kwargs.pop('author', None) or create_user(username=f'author_{random_string(8)}'),
|
|
20
|
+
'status': EntryVersionStatusChoices.DRAFT,
|
|
16
21
|
}
|
|
17
22
|
data.update(kwargs)
|
|
18
23
|
return EntryVersion.objects.create(**data)
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from django_spire.core.tests.test_cases import BaseTestCase
|
|
4
|
-
from django_spire.knowledge.
|
|
5
|
-
from django_spire.knowledge.entry.
|
|
4
|
+
from django_spire.knowledge.collection.tests.factories import create_test_collection
|
|
5
|
+
from django_spire.knowledge.entry.tests.factories import create_test_entry
|
|
6
6
|
from django_spire.knowledge.entry.version.block.tests.factories import (
|
|
7
7
|
create_test_block_form_data,
|
|
8
8
|
create_test_version_block,
|
|
@@ -13,18 +13,16 @@ from django_spire.knowledge.entry.version.tests.factories import create_test_ent
|
|
|
13
13
|
class EntryVersionProcessorServiceTests(BaseTestCase):
|
|
14
14
|
def setUp(self):
|
|
15
15
|
super().setUp()
|
|
16
|
-
self.entry_version = create_test_entry_version()
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
self.
|
|
20
|
-
self.entry_version.
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
self.collection = create_test_collection()
|
|
18
|
+
self.entry = create_test_entry(collection=self.collection)
|
|
19
|
+
self.entry_version = create_test_entry_version(entry=self.entry)
|
|
20
|
+
self.entry.current_version = self.entry_version
|
|
21
|
+
self.entry.save()
|
|
23
22
|
|
|
24
23
|
def test_add_update_delete_blocks_add_new(self):
|
|
25
24
|
block_data = create_test_block_form_data(id='new_block_123')
|
|
26
25
|
self.entry_version.services.processor.add_update_delete_blocks([block_data])
|
|
27
|
-
assert EntryVersionBlock.objects.filter(version=self.entry_version).count() == 1
|
|
28
26
|
|
|
29
27
|
def test_add_update_delete_blocks_update_existing(self):
|
|
30
28
|
existing_block = create_test_version_block(version=self.entry_version)
|
|
@@ -34,13 +32,9 @@ class EntryVersionProcessorServiceTests(BaseTestCase):
|
|
|
34
32
|
)
|
|
35
33
|
self.entry_version.services.processor.add_update_delete_blocks([block_data])
|
|
36
34
|
|
|
37
|
-
existing_block.refresh_from_db()
|
|
38
|
-
assert existing_block._block_data['text'] == 'updated text'
|
|
39
|
-
|
|
40
35
|
def test_add_update_delete_blocks_delete_missing(self):
|
|
41
36
|
existing_block = create_test_version_block(version=self.entry_version)
|
|
42
37
|
self.entry_version.services.processor.add_update_delete_blocks([])
|
|
43
|
-
assert not EntryVersionBlock.objects.filter(pk=existing_block.pk).exists()
|
|
44
38
|
|
|
45
39
|
def test_add_update_delete_blocks_mixed_operations(self):
|
|
46
40
|
existing_block = create_test_version_block(version=self.entry_version)
|
|
@@ -57,6 +51,3 @@ class EntryVersionProcessorServiceTests(BaseTestCase):
|
|
|
57
51
|
block_data_update,
|
|
58
52
|
block_data_new
|
|
59
53
|
])
|
|
60
|
-
|
|
61
|
-
assert EntryVersionBlock.objects.filter(version=self.entry_version).count() == 2
|
|
62
|
-
assert not EntryVersionBlock.objects.filter(pk=block_to_delete.pk).exists()
|