iatoolkit 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of iatoolkit might be problematic. Click here for more details.
- iatoolkit/base_company.py +3 -3
- iatoolkit/cli_commands.py +5 -0
- iatoolkit/company_registry.py +7 -8
- iatoolkit/iatoolkit.py +24 -16
- {iatoolkit-0.4.1.dist-info → iatoolkit-0.4.4.dist-info}/METADATA +1 -1
- iatoolkit-0.4.4.dist-info/RECORD +31 -0
- services/__init__.py +3 -3
- services/api_service.py +3 -3
- services/benchmark_service.py +3 -3
- services/dispatcher_service.py +6 -28
- services/document_service.py +4 -4
- services/excel_service.py +3 -3
- services/file_processor_service.py +3 -3
- services/history_service.py +3 -3
- services/jwt_service.py +3 -3
- services/load_documents_service.py +13 -96
- services/mail_service.py +3 -3
- services/profile_service.py +3 -3
- services/prompt_manager_service.py +3 -3
- services/query_service.py +4 -4
- services/search_service.py +19 -3
- services/sql_service.py +20 -3
- services/tasks_service.py +3 -3
- services/user_feedback_service.py +3 -3
- services/user_session_context_service.py +3 -3
- iatoolkit/system_prompts/arquitectura.prompt +0 -32
- iatoolkit-0.4.1.dist-info/RECORD +0 -32
- {iatoolkit-0.4.1.dist-info → iatoolkit-0.4.4.dist-info}/WHEEL +0 -0
- {iatoolkit-0.4.1.dist-info → iatoolkit-0.4.4.dist-info}/top_level.txt +0 -0
iatoolkit/base_company.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
# companies/base_company.py
|
|
7
7
|
from abc import ABC, abstractmethod
|
iatoolkit/cli_commands.py
CHANGED
iatoolkit/company_registry.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
3
5
|
|
|
4
6
|
from typing import Dict, Type, Any
|
|
5
7
|
from .base_company import BaseCompany
|
|
@@ -19,19 +21,17 @@ class CompanyRegistry:
|
|
|
19
21
|
|
|
20
22
|
def instantiate_companies(self, injector) -> Dict[str, BaseCompany]:
|
|
21
23
|
"""
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
Returns:
|
|
25
|
-
Dict con instancias de empresas {name: instance}
|
|
24
|
+
intantiate all registered companies using the toolkit injector
|
|
26
25
|
"""
|
|
27
|
-
|
|
28
26
|
for company_key, company_class in self._company_classes.items():
|
|
29
27
|
if company_key not in self._company_instances:
|
|
30
28
|
try:
|
|
31
29
|
# use de injector to create the instance
|
|
32
30
|
company_instance = injector.get(company_class)
|
|
31
|
+
|
|
32
|
+
# save the created instance in the registry
|
|
33
33
|
self._company_instances[company_key] = company_instance
|
|
34
|
-
logging.info(f"company '{company_key}'
|
|
34
|
+
logging.info(f"company '{company_key}' instantiated")
|
|
35
35
|
|
|
36
36
|
except Exception as e:
|
|
37
37
|
logging.error(f"Error instanciando empresa {company_key}: {e}")
|
|
@@ -48,7 +48,6 @@ class CompanyRegistry:
|
|
|
48
48
|
return self._company_classes.copy()
|
|
49
49
|
|
|
50
50
|
def clear(self) -> None:
|
|
51
|
-
"""Limpia el registro (útil para tests)"""
|
|
52
51
|
self._company_classes.clear()
|
|
53
52
|
self._company_instances.clear()
|
|
54
53
|
|
iatoolkit/iatoolkit.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
4
5
|
|
|
5
6
|
from flask import Flask, url_for, current_app
|
|
6
7
|
from flask_session import Session
|
|
@@ -90,7 +91,10 @@ class IAToolkit:
|
|
|
90
91
|
# and other integrations, as views are handled manually.
|
|
91
92
|
FlaskInjector(app=self.app, injector=self._injector)
|
|
92
93
|
|
|
93
|
-
# Step 6:
|
|
94
|
+
# Step 6: initialize dispatcher and registered compaies
|
|
95
|
+
self._init_dispatcher_and_company_instances()
|
|
96
|
+
|
|
97
|
+
# Step 7: Finalize setup within the application context
|
|
94
98
|
self._setup_redis_sessions()
|
|
95
99
|
self._setup_cors()
|
|
96
100
|
self._setup_additional_services()
|
|
@@ -107,7 +111,7 @@ class IAToolkit:
|
|
|
107
111
|
return self.app
|
|
108
112
|
|
|
109
113
|
def _get_config_value(self, key: str, default=None):
|
|
110
|
-
|
|
114
|
+
# get a value from the config dict or the environment variable
|
|
111
115
|
return self.config.get(key, os.getenv(key, default))
|
|
112
116
|
|
|
113
117
|
def _setup_logging(self):
|
|
@@ -231,7 +235,7 @@ class IAToolkit:
|
|
|
231
235
|
"""⚙️ Configures all system dependencies."""
|
|
232
236
|
try:
|
|
233
237
|
# Core dependencies
|
|
234
|
-
binder.bind(Flask, to=self.app
|
|
238
|
+
binder.bind(Flask, to=self.app)
|
|
235
239
|
binder.bind(DatabaseManager, to=self.db_manager, scope=singleton)
|
|
236
240
|
|
|
237
241
|
# Bind all application components by calling the specific methods
|
|
@@ -285,7 +289,7 @@ class IAToolkit:
|
|
|
285
289
|
binder.bind(LoadDocumentsService, to=LoadDocumentsService)
|
|
286
290
|
binder.bind(ProfileService, to=ProfileService)
|
|
287
291
|
binder.bind(JWTService, to=JWTService)
|
|
288
|
-
binder.bind(Dispatcher, to=Dispatcher
|
|
292
|
+
binder.bind(Dispatcher, to=Dispatcher)
|
|
289
293
|
|
|
290
294
|
def _bind_infrastructure(self, binder: Binder):
|
|
291
295
|
from infra.llm_client import llmClient
|
|
@@ -317,9 +321,19 @@ class IAToolkit:
|
|
|
317
321
|
def _setup_additional_services(self):
|
|
318
322
|
Bcrypt(self.app)
|
|
319
323
|
|
|
324
|
+
def _init_dispatcher_and_company_instances(self):
|
|
325
|
+
from iatoolkit.company_registry import get_company_registry
|
|
326
|
+
from services.dispatcher_service import Dispatcher
|
|
327
|
+
|
|
328
|
+
# instantiate all the registered companies
|
|
329
|
+
get_company_registry().instantiate_companies(self._injector)
|
|
330
|
+
|
|
331
|
+
# use the dispatcher to start the execution of every company
|
|
332
|
+
dispatcher = self._injector.get(Dispatcher)
|
|
333
|
+
dispatcher.start_execution()
|
|
334
|
+
|
|
320
335
|
def _setup_cli_commands(self):
|
|
321
336
|
from iatoolkit.cli_commands import register_core_commands
|
|
322
|
-
from services.dispatcher_service import Dispatcher
|
|
323
337
|
from iatoolkit.company_registry import get_company_registry
|
|
324
338
|
|
|
325
339
|
# 1. Register core commands
|
|
@@ -328,16 +342,10 @@ class IAToolkit:
|
|
|
328
342
|
|
|
329
343
|
# 2. Register company-specific commands
|
|
330
344
|
try:
|
|
331
|
-
# Get the dispatcher, which holds the company instances
|
|
332
|
-
dispatcher = self.get_injector().get(Dispatcher)
|
|
333
|
-
registry = get_company_registry()
|
|
334
|
-
|
|
335
345
|
# Iterate through the registered company names
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
company_instance.register_cli_commands(self.app)
|
|
340
|
-
logging.info(f"✅ Comandos CLI para la compañía '{company_name}' registrados.")
|
|
346
|
+
all_company_instances = get_company_registry().get_all_company_instances()
|
|
347
|
+
for company_name, company_instance in all_company_instances.items():
|
|
348
|
+
company_instance.register_cli_commands(self.app)
|
|
341
349
|
|
|
342
350
|
except Exception as e:
|
|
343
351
|
logging.error(f"❌ Error durante el registro de comandos de compañías: {e}")
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
iatoolkit/__init__.py,sha256=GkFxAQHKPifz4Kd8M73Rc8TWRVIxjxkl1N0nsPvb_sU,1743
|
|
2
|
+
iatoolkit/base_company.py,sha256=LyTnNqxfKKks0eu2DaisWFyMF11PetWNy5QiZe1nYRA,1938
|
|
3
|
+
iatoolkit/cli_commands.py,sha256=XApFY4TrLjyyEt8L1ckY-Ro7AWFed8_tHh-aVsim3ww,3011
|
|
4
|
+
iatoolkit/company_registry.py,sha256=tduqt3oV8iDX_IB1eA7KIgvIxE4edTcy-3qZIXh3Lzw,2549
|
|
5
|
+
iatoolkit/iatoolkit.py,sha256=WgBm34nAeVBCpfDDZD13Pos9xIoRx5viVwxjyYCXAmY,15790
|
|
6
|
+
iatoolkit/system_prompts/format_styles.prompt,sha256=MSMe1qvR3cF_0IbFshn8R0z6Wx6VCHQq1p37rpu5wwk,3576
|
|
7
|
+
iatoolkit/system_prompts/query_main.prompt,sha256=Eu5VOQzUygJ45Ct1WKYGbi0JMltgI6FQIZWlGmN1bdk,3214
|
|
8
|
+
iatoolkit/system_prompts/sql_rules.prompt,sha256=y4nURVnb9AyFwt-lrbMNBHHtZlhk6kC9grYoOhRnrJo,59174
|
|
9
|
+
services/__init__.py,sha256=5JqK9sZ6jBuK83zDQokUhxQ0wuJJJ9DXB8pYCLkX7X4,102
|
|
10
|
+
services/api_service.py,sha256=7FxO9_g-gxuEmYIl5PhgJb7lHVgXJ1btHdH9fDrdhvY,2840
|
|
11
|
+
services/benchmark_service.py,sha256=_ruKh9YzrTLtR0ZKrRNxqJQW0HdbwWuFz1gfLzJ9owA,5850
|
|
12
|
+
services/dispatcher_service.py,sha256=pc_ksXM96c876LJ38c2kCLGIZlKRQcLahC66UruZN9c,14520
|
|
13
|
+
services/document_service.py,sha256=r9EPjkhkjbAdfdZ0LCmUcvQbmGZR7BWgOsBiaeSmuoU,5890
|
|
14
|
+
services/excel_service.py,sha256=ATPaeAvkLwQAkPZ3AKIUpO73RVyRg0D8c6i37_mcql0,3559
|
|
15
|
+
services/file_processor_service.py,sha256=98yWYF7nIq1nm7nh6IzMmTKaOMTIeqCFWYwXVtV-ZJI,4102
|
|
16
|
+
services/history_service.py,sha256=j0QCqcIIyw7DBy3GrZrEZNk0I4m-uuRoG5g0Z2RCcOE,1586
|
|
17
|
+
services/jwt_service.py,sha256=YoZ9h7_o9xBko-arNQv4MbcwnxoSWVNj4VbZmMo_QGY,3908
|
|
18
|
+
services/load_documents_service.py,sha256=XQ_Ck1pvj6TfJRIC1cNrWRbLmC0u7E0VJkNWC6L2gI4,7027
|
|
19
|
+
services/mail_service.py,sha256=_67pctxZO46DHnWBip51ayuYtWd4bEoS1kg29ACO7_I,2162
|
|
20
|
+
services/profile_service.py,sha256=vVRGNVJqByM-Drl17hmcDRnb-usQKJpiwlb733FfHXs,17851
|
|
21
|
+
services/prompt_manager_service.py,sha256=wz5hhthLKt_tsj7WhsbuerXfpTCJhY4aIfjvCXREjs4,7690
|
|
22
|
+
services/query_service.py,sha256=yITLcopc1stgzTkFdrBuF84k_XYsZyoKGue1GfXsb-U,15614
|
|
23
|
+
services/search_service.py,sha256=bB3FWFxJi1iYsOdBxyu9tzIO406nQxcyeQzEowpmpjY,1803
|
|
24
|
+
services/sql_service.py,sha256=s84K1ADlvMtum949wgMh8jsmqlOUeL-m_SWfAM4Wsv4,2141
|
|
25
|
+
services/tasks_service.py,sha256=1DdbERlAxIkCpGEylnHDKC-KAsXRJugbaRSzRbPfL58,6790
|
|
26
|
+
services/user_feedback_service.py,sha256=_LeNBYz4hHFapXfYTQVfkkD34gE8j2UeKnyOZ8H0nWo,2442
|
|
27
|
+
services/user_session_context_service.py,sha256=GluNSgqP6W_hFke4oslSnfGnU_b-ph28BHH6jf3EIm0,3797
|
|
28
|
+
iatoolkit-0.4.4.dist-info/METADATA,sha256=h3whrSrA15ia8BjgxLrhVSH9EqX24u9IFkbDgfgoWxc,9300
|
|
29
|
+
iatoolkit-0.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
30
|
+
iatoolkit-0.4.4.dist-info/top_level.txt,sha256=dqlBbmgo9okD9d_WMR9uYzdup7Rxgj26yFF85jRGeu4,19
|
|
31
|
+
iatoolkit-0.4.4.dist-info/RECORD,,
|
services/__init__.py
CHANGED
services/api_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from infra.call_service import CallServiceClient
|
|
7
7
|
from injector import inject
|
services/benchmark_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
import time
|
services/dispatcher_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from common.exceptions import IAToolkitException
|
|
7
7
|
from services.prompt_manager_service import PromptService
|
|
@@ -39,12 +39,6 @@ class Dispatcher:
|
|
|
39
39
|
self._company_registry = None
|
|
40
40
|
self._company_instances = None
|
|
41
41
|
|
|
42
|
-
# load into the dispatcher the configured companies
|
|
43
|
-
self.initialize_companies()
|
|
44
|
-
|
|
45
|
-
# run the statrtup logic for all companies
|
|
46
|
-
self.start_execution()
|
|
47
|
-
|
|
48
42
|
self.tool_handlers = {
|
|
49
43
|
"iat_generate_excel": self.excel_service.excel_generator,
|
|
50
44
|
"iat_send_email": self.mail_service.send_mail,
|
|
@@ -66,25 +60,9 @@ class Dispatcher:
|
|
|
66
60
|
self._company_instances = self.company_registry.get_all_company_instances()
|
|
67
61
|
return self._company_instances
|
|
68
62
|
|
|
69
|
-
def initialize_companies(self):
|
|
70
|
-
from iatoolkit import current_iatoolkit
|
|
71
|
-
"""
|
|
72
|
-
Initializes and instantiates all registered company classes.
|
|
73
|
-
This method should be called *after* the main injector is fully configured
|
|
74
|
-
and the company registry is populated.
|
|
75
|
-
"""
|
|
76
|
-
if self.company_registry.get_all_company_instances(): # Check if already instantiated
|
|
77
|
-
return
|
|
78
|
-
|
|
79
|
-
# ✅ NOW it is safe to get the injector and instantiate companies.
|
|
80
|
-
injector = current_iatoolkit().get_injector()
|
|
81
|
-
self.company_registry.instantiate_companies(injector)
|
|
82
|
-
|
|
83
|
-
|
|
84
63
|
def start_execution(self):
|
|
85
64
|
"""Runs the startup logic for all registered companies."""
|
|
86
65
|
for company_name, company_instance in self.company_instances.items():
|
|
87
|
-
logging.info(f'Starting execution for company: {company_name}')
|
|
88
66
|
company_instance.start_execution()
|
|
89
67
|
|
|
90
68
|
return True
|
|
@@ -109,7 +87,7 @@ class Dispatcher:
|
|
|
109
87
|
prompt_name=prompt['name'],
|
|
110
88
|
description=prompt['description'],
|
|
111
89
|
order=1,
|
|
112
|
-
is_system_prompt=True
|
|
90
|
+
is_system_prompt=True,
|
|
113
91
|
)
|
|
114
92
|
i += 1
|
|
115
93
|
|
|
@@ -117,7 +95,7 @@ class Dispatcher:
|
|
|
117
95
|
for company in self.company_instances.values():
|
|
118
96
|
company.register_company()
|
|
119
97
|
|
|
120
|
-
def dispatch(self, company_name: str, action: str, **kwargs) ->
|
|
98
|
+
def dispatch(self, company_name: str, action: str, **kwargs) -> dict:
|
|
121
99
|
company_key = company_name.lower()
|
|
122
100
|
|
|
123
101
|
if company_key not in self.company_instances:
|
|
@@ -219,7 +197,7 @@ class Dispatcher:
|
|
|
219
197
|
"""
|
|
220
198
|
Asegura que los datos del usuario siempre tengan una estructura consistente.
|
|
221
199
|
"""
|
|
222
|
-
#
|
|
200
|
+
# default values
|
|
223
201
|
normalized_user = {
|
|
224
202
|
"id": raw_data.get("id", 0),
|
|
225
203
|
"user_email": raw_data.get("email", ""),
|
services/document_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from docx import Document
|
|
7
7
|
import fitz # PyMuPDF
|
|
@@ -16,7 +16,7 @@ class DocumentService:
|
|
|
16
16
|
@inject
|
|
17
17
|
def __init__(self):
|
|
18
18
|
# max number of pages to load
|
|
19
|
-
self.max_doc_pages = int(os.getenv("MAX_DOC_PAGES", "
|
|
19
|
+
self.max_doc_pages = int(os.getenv("MAX_DOC_PAGES", "200"))
|
|
20
20
|
|
|
21
21
|
def file_to_txt(self, filename, file_content):
|
|
22
22
|
try:
|
services/excel_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from common.util import Utility
|
|
7
7
|
import pandas as pd
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from infra.connectors.file_connector import FileConnector
|
|
7
7
|
import logging
|
services/history_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from injector import inject
|
|
7
7
|
from repositories.llm_query_repo import LLMQueryRepo
|
services/jwt_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
import jwt
|
|
7
7
|
import time
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from repositories.vs_repo import VSRepo
|
|
7
7
|
from repositories.document_repo import DocumentRepo
|
|
@@ -52,103 +52,16 @@ class LoadDocumentsService:
|
|
|
52
52
|
separators=["\n\n", "\n", "."]
|
|
53
53
|
)
|
|
54
54
|
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
def load_company_files(self,
|
|
56
|
+
company: Company,
|
|
57
|
+
connector_config: Dict,
|
|
58
|
+
predefined_metadata: Dict = None,
|
|
59
|
+
filters: Dict = None):
|
|
57
60
|
"""
|
|
58
|
-
|
|
59
|
-
It can load all document types or a specific one if provided.
|
|
60
|
-
|
|
61
|
-
Args:
|
|
62
|
-
doc_type (str, optional): A specific document type to load.
|
|
63
|
-
If None, all configured types are loaded.
|
|
64
|
-
|
|
65
|
-
Returns:
|
|
66
|
-
Dict: A dictionary with a summary message.
|
|
67
|
-
"""
|
|
68
|
-
# doc_type: an optional document_type for loading
|
|
69
|
-
files_loaded = 0
|
|
70
|
-
companies = self.profile_repo.get_companies()
|
|
71
|
-
|
|
72
|
-
for company in companies:
|
|
73
|
-
load_config = company.parameters.get('load', {})
|
|
74
|
-
if not load_config:
|
|
75
|
-
continue
|
|
76
|
-
|
|
77
|
-
print(f"Cargando datos de ** {company.short_name} **")
|
|
78
|
-
|
|
79
|
-
# Si hay configuraciones de tipos de documento específicos
|
|
80
|
-
doc_types_config = load_config.get('document_types', {})
|
|
81
|
-
|
|
82
|
-
if doc_types_config and len(doc_types_config) > 0:
|
|
83
|
-
# Si se especificó un tipo de documento, cargar solo ese tipo
|
|
84
|
-
if doc_type and doc_type in doc_types_config:
|
|
85
|
-
files_loaded += self._load_document_type(company, doc_type, doc_types_config[doc_type])
|
|
86
|
-
# Si no se especificó, cargar todos los tipos configurados
|
|
87
|
-
elif not doc_type:
|
|
88
|
-
for type_name, type_config in doc_types_config.items():
|
|
89
|
-
files_loaded += self._load_document_type(company, type_name, type_config)
|
|
90
|
-
else:
|
|
91
|
-
# Comportamiento anterior: usar la configuración general
|
|
92
|
-
connector = load_config.get('connector', {})
|
|
93
|
-
if not connector:
|
|
94
|
-
raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
|
|
95
|
-
f"Falta configurar conector en empresa {company.short_name}")
|
|
96
|
-
|
|
97
|
-
files_loaded += self.load_data_source(company=company,
|
|
98
|
-
connector_config=connector)
|
|
99
|
-
|
|
100
|
-
return {'message': f'{files_loaded} files processed'}
|
|
101
|
-
|
|
102
|
-
def load_company_files(self, company: Company,
|
|
103
|
-
connector: dict,
|
|
104
|
-
predefined_metadata: Dict = None,
|
|
105
|
-
filters: Dict = None):
|
|
106
|
-
"""
|
|
107
|
-
Loads all files for a specific company using a given connector.
|
|
61
|
+
Loads all the company files from a connector
|
|
108
62
|
|
|
109
63
|
Args:
|
|
110
64
|
company (Company): The company to load files for.
|
|
111
|
-
connector (dict): The connector configuration.
|
|
112
|
-
|
|
113
|
-
Returns:
|
|
114
|
-
Dict: A dictionary with a summary message.
|
|
115
|
-
"""
|
|
116
|
-
if not connector:
|
|
117
|
-
raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
|
|
118
|
-
f"Falta configurar conector")
|
|
119
|
-
|
|
120
|
-
files_loaded = self.load_data_source(
|
|
121
|
-
company=company,
|
|
122
|
-
connector_config=connector,
|
|
123
|
-
predefined_metadata=predefined_metadata,
|
|
124
|
-
filters=filters)
|
|
125
|
-
|
|
126
|
-
return {'message': f'{files_loaded} files processed'}
|
|
127
|
-
|
|
128
|
-
def _load_document_type(self, company: Company, doc_type_name: str, type_config: Dict) -> int:
|
|
129
|
-
# load specific document_types for a company
|
|
130
|
-
connector = type_config.get('connector')
|
|
131
|
-
if not connector:
|
|
132
|
-
logging.warning(f"Falta configurar conector para tipo {doc_type_name} en empresa {company.short_name}")
|
|
133
|
-
raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
|
|
134
|
-
f"Falta configurar conector para tipo {doc_type_name} en empresa {company.short_name}")
|
|
135
|
-
|
|
136
|
-
# get the metadata for this connector
|
|
137
|
-
predefined_metadata = type_config.get('metadata', {})
|
|
138
|
-
|
|
139
|
-
# config specific filters
|
|
140
|
-
filters = type_config.get('filters', {"filename_contains": ".pdf"})
|
|
141
|
-
|
|
142
|
-
return self.load_data_source(company=company,
|
|
143
|
-
connector_config=connector,
|
|
144
|
-
predefined_metadata=predefined_metadata,
|
|
145
|
-
filters=filters)
|
|
146
|
-
|
|
147
|
-
def load_data_source(self, company: Company, connector_config: Dict, predefined_metadata: Dict = None, filters: Dict = None):
|
|
148
|
-
"""
|
|
149
|
-
Loads files from a data source using a connector and a FileProcessor.
|
|
150
|
-
|
|
151
|
-
Args:
|
|
152
65
|
connector_config (Dict): The configuration for the file connector.
|
|
153
66
|
predefined_metadata (Dict, optional): Metadata to be added to all documents from this source.
|
|
154
67
|
filters (Dict, optional): Filters to apply to the files.
|
|
@@ -156,6 +69,10 @@ class LoadDocumentsService:
|
|
|
156
69
|
Returns:
|
|
157
70
|
int: The number of processed files.
|
|
158
71
|
"""
|
|
72
|
+
if not connector_config:
|
|
73
|
+
raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
|
|
74
|
+
f"Falta configurar conector")
|
|
75
|
+
|
|
159
76
|
try:
|
|
160
77
|
if not filters:
|
|
161
78
|
filters = {"filename_contains": ".pdf"}
|
services/mail_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from infra.mail_app import MailApp
|
|
7
7
|
from injector import inject
|
services/profile_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from injector import inject
|
|
7
7
|
from repositories.profile_repo import ProfileRepo
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from injector import inject
|
|
7
7
|
from repositories.llm_query_repo import LLMQueryRepo
|
services/query_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from infra.llm_client import llmClient
|
|
7
7
|
from repositories.document_repo import DocumentRepo
|
|
@@ -48,7 +48,7 @@ class QueryService:
|
|
|
48
48
|
self.session_context = session_context
|
|
49
49
|
self.llm_client = llm_client
|
|
50
50
|
|
|
51
|
-
#
|
|
51
|
+
# get the model from the environment variable
|
|
52
52
|
self.model = os.getenv("LLM_MODEL", "")
|
|
53
53
|
if not self.model:
|
|
54
54
|
raise IAToolkitException(IAToolkitException.ErrorType.API_KEY,
|
services/search_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from repositories.vs_repo import VSRepo
|
|
7
7
|
from repositories.document_repo import DocumentRepo
|
|
@@ -18,6 +18,22 @@ class SearchService:
|
|
|
18
18
|
self.doc_repo = doc_repo
|
|
19
19
|
|
|
20
20
|
def search(self, company_id: int, query: str, metadata_filter: dict = None) -> str:
|
|
21
|
+
"""
|
|
22
|
+
Performs a semantic search for a given query within a company's documents.
|
|
23
|
+
|
|
24
|
+
This method queries the vector store for relevant documents based on the
|
|
25
|
+
provided query text. It then constructs a formatted string containing the
|
|
26
|
+
content of the retrieved documents, which can be used as context for an LLM.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
company_id: The ID of the company to search within.
|
|
30
|
+
query: The text query to search for.
|
|
31
|
+
metadata_filter: An optional dictionary to filter documents by their metadata.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
A string containing the concatenated content of the found documents,
|
|
35
|
+
formatted to be used as a context.
|
|
36
|
+
"""
|
|
21
37
|
document_list = self.vs_repo.query(company_id=company_id,
|
|
22
38
|
query_text=query,
|
|
23
39
|
metadata_filter=metadata_filter)
|
services/sql_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from repositories.database_manager import DatabaseManager
|
|
7
7
|
from common.util import Utility
|
|
@@ -17,6 +17,23 @@ class SqlService:
|
|
|
17
17
|
self.util = util
|
|
18
18
|
|
|
19
19
|
def exec_sql(self, db_manager: DatabaseManager, sql_statement: str) -> str:
|
|
20
|
+
"""
|
|
21
|
+
Executes a raw SQL statement and returns the result as a JSON string.
|
|
22
|
+
|
|
23
|
+
This method takes a DatabaseManager instance and a SQL query, executes it
|
|
24
|
+
against the database, and fetches all results. The results are converted
|
|
25
|
+
into a list of dictionaries, where each dictionary represents a row.
|
|
26
|
+
This list is then serialized to a JSON string.
|
|
27
|
+
If an exception occurs during execution, the transaction is rolled back,
|
|
28
|
+
and a custom IAToolkitException is raised.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
db_manager: The DatabaseManager instance to get the database session from.
|
|
32
|
+
sql_statement: The raw SQL statement to be executed.
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
A JSON string representing the list of rows returned by the query.
|
|
36
|
+
"""
|
|
20
37
|
try:
|
|
21
38
|
# here the SQL is executed
|
|
22
39
|
result = db_manager.get_session().execute(text(sql_statement))
|
services/tasks_service.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from injector import inject
|
|
7
7
|
from repositories.models import Task, TaskStatus
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from repositories.models import UserFeedback
|
|
7
7
|
from injector import inject
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) 2024 Fernando Libedinsky
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# Product: IAToolkit
|
|
3
|
+
#
|
|
4
|
+
# IAToolkit is open source software.
|
|
5
5
|
|
|
6
6
|
from infra.redis_session_manager import RedisSessionManager
|
|
7
7
|
from typing import List, Dict, Optional
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
Mi app es un chatbot multiempresa, que se conecta
|
|
2
|
-
con datos empresariales por sql y llamadas a endpoints.
|
|
3
|
-
|
|
4
|
-
Es principalmente un backend, pero tambien puede actuar como frontend.
|
|
5
|
-
Cuando actua como backend, los usuarios se identifican como external_user_id
|
|
6
|
-
y el front lo informa en la vista external_login_view.
|
|
7
|
-
|
|
8
|
-
Cuando actua como front y back,
|
|
9
|
-
existe un acceso que simula un usuario externo a través de
|
|
10
|
-
public_chat_view.
|
|
11
|
-
|
|
12
|
-
El otro mecanismo de acceso es con el sistema de usuarios
|
|
13
|
-
integrado a mi app, con tablas propias: user, company, user_company
|
|
14
|
-
y el acceso se hace a través de profile_service.
|
|
15
|
-
|
|
16
|
-
Estos accesos sirven para autentificar las consultas.
|
|
17
|
-
Cada vez que hay un acceso en estos puntos, se llama a query_service
|
|
18
|
-
para que construya el contexto de la empresa y se lo envie al
|
|
19
|
-
llm.
|
|
20
|
-
|
|
21
|
-
Se guarda en session de redis, el response.id generado
|
|
22
|
-
para linkearlo con las consultas futuras (responses api de openai).
|
|
23
|
-
|
|
24
|
-
Por otra parte, las queries ingresan a través de llm_query_view,
|
|
25
|
-
el que autentifica al solicitante y la dirige a query_service.
|
|
26
|
-
|
|
27
|
-
Query_service gestiona los prompts que pueden ser solicitados
|
|
28
|
-
y llama a invoke en openai_client, quien se encarga
|
|
29
|
-
de interactuar con el llm.
|
|
30
|
-
|
|
31
|
-
Una vez terminada la interacción, se graba un registro en la
|
|
32
|
-
tabla llm_queries.
|
iatoolkit-0.4.1.dist-info/RECORD
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
iatoolkit/__init__.py,sha256=GkFxAQHKPifz4Kd8M73Rc8TWRVIxjxkl1N0nsPvb_sU,1743
|
|
2
|
-
iatoolkit/base_company.py,sha256=FlB-HFYH8FoTl4nbtsYgfKjkdZtizJbKwXqaosxmRqc,2009
|
|
3
|
-
iatoolkit/cli_commands.py,sha256=CyaabHA3HdKd-eIqrJD8IQFT7Tqn_BEdi4jb1utisMo,2909
|
|
4
|
-
iatoolkit/company_registry.py,sha256=KOUzJHLYgzMAV6BxkTiDPlN_ME4fktp7yRzKLlXZ5-w,2597
|
|
5
|
-
iatoolkit/iatoolkit.py,sha256=OwlGujwtNLBYtfZuCpcX_yzrgB8BVo9Jfh72owM8FFc,15651
|
|
6
|
-
iatoolkit/system_prompts/arquitectura.prompt,sha256=2W-7NWy6P6y1Gh5_-zD1iK-BWq1Siu8TuvGCouP67bQ,1267
|
|
7
|
-
iatoolkit/system_prompts/format_styles.prompt,sha256=MSMe1qvR3cF_0IbFshn8R0z6Wx6VCHQq1p37rpu5wwk,3576
|
|
8
|
-
iatoolkit/system_prompts/query_main.prompt,sha256=Eu5VOQzUygJ45Ct1WKYGbi0JMltgI6FQIZWlGmN1bdk,3214
|
|
9
|
-
iatoolkit/system_prompts/sql_rules.prompt,sha256=y4nURVnb9AyFwt-lrbMNBHHtZlhk6kC9grYoOhRnrJo,59174
|
|
10
|
-
services/__init__.py,sha256=fSvSfIcPW1dHwTBY1hQ5dBEhaoorzk_GzR4G46gD8tY,173
|
|
11
|
-
services/api_service.py,sha256=InIKTc64BWcp4U4tYKHz28x4ErPxIfvR9x3ZlxJZlXs,2911
|
|
12
|
-
services/benchmark_service.py,sha256=0Vgsx_FaUZL7igoBYbe1AZkIWOiEUx1FSCV_0Ut0mtk,5921
|
|
13
|
-
services/dispatcher_service.py,sha256=AOLsv9haFOxW5hV-6LOP8tryDdlfvjEGhNNGjgphH2c,15489
|
|
14
|
-
services/document_service.py,sha256=sm5QtbrKs2dF9hpLuSLMB-IMWYNBD7yWHv3rd80aD0o,5960
|
|
15
|
-
services/excel_service.py,sha256=wE9Udbyb96kGRSnZZ6KM2mbE484rKjTEhta9GKKpy-8,3630
|
|
16
|
-
services/file_processor_service.py,sha256=0CM4CQu6KKfcLVGkxs4hYxgdz8kKRWfkV5rDH9UoccM,4173
|
|
17
|
-
services/history_service.py,sha256=6fGSSWxy60nxtkwp_fodwDHoVKhpIUbHnzAzUSiNi-Y,1657
|
|
18
|
-
services/jwt_service.py,sha256=dC45Sn6FyzdzRiQJnzgkjN3Hy21V1imRxB0hTyWRvlA,3979
|
|
19
|
-
services/load_documents_service.py,sha256=AtB5VnSjOEILG3wcQm6YiXkRrVvL87JWeRPgon81hf4,11064
|
|
20
|
-
services/mail_service.py,sha256=ystFit1LuYUC4ekYYebyiy1rqYQmxeL6K8h58MxEkOY,2233
|
|
21
|
-
services/profile_service.py,sha256=vZV0cregZQiPKYcNLaD7xjez2y6-3Mq97cDndC8NL8w,17922
|
|
22
|
-
services/prompt_manager_service.py,sha256=bWG4SIgt0u45PVUfm0xRLbLfKC7bk6uozVHRdkdgCmc,7761
|
|
23
|
-
services/query_service.py,sha256=zpaDzjzh2HqAG1F2Ap8WHBpfAMVtzZ_6v1JGVEguwvs,15687
|
|
24
|
-
services/search_service.py,sha256=oJD6WRXCJBD7WUVHWWKxexRkhR8nQSrFtcPV3pFO2KQ,1153
|
|
25
|
-
services/sql_service.py,sha256=H7CIPpXTcxLXLojD2fBFr_mIAD0PW1vEJhKHLfJi4Hk,1418
|
|
26
|
-
services/tasks_service.py,sha256=hHJDlcsSOPtEleD6_Vv3pocfxWNmthIhmZSdnoWFpEM,6861
|
|
27
|
-
services/user_feedback_service.py,sha256=YtCndRBekDEWYEbac431Ksn2gMO5iBrI3WqKK0xtShE,2513
|
|
28
|
-
services/user_session_context_service.py,sha256=5qn7fqpuiU8KgMpU4M5-iRUsETumz1raBw-EeZLuE1A,3868
|
|
29
|
-
iatoolkit-0.4.1.dist-info/METADATA,sha256=rzRfT16CeRWOOnJorFb5kNJ3W0U98yKb1FZiB_L6p8M,9300
|
|
30
|
-
iatoolkit-0.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
31
|
-
iatoolkit-0.4.1.dist-info/top_level.txt,sha256=dqlBbmgo9okD9d_WMR9uYzdup7Rxgj26yFF85jRGeu4,19
|
|
32
|
-
iatoolkit-0.4.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|