smart-artificial-intelligence-local 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smart_artificial_intelligence_local-0.0.1/PKG-INFO +23 -0
- smart_artificial_intelligence_local-0.0.1/README.md +26 -0
- smart_artificial_intelligence_local-0.0.1/pyproject.toml +29 -0
- smart_artificial_intelligence_local-0.0.1/setup.cfg +4 -0
- smart_artificial_intelligence_local-0.0.1/setup.py +39 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/__init__.py +0 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/__init__.py +0 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/constants_src_smart_artificial_intelligence_local.py +21 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/llm_mode_enum.py +11 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/smart_artificial_intelligence_local.py +205 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/smart_llm_logger.py +44 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/smart_llm_result.py +22 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/tests/__init__.py +0 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/tests/constants_tests_smart_artificial_intelligence_local.py +15 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/tests/smart_artificial_intelligence_local_test.py +292 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/tests/smart_llm_system_test.py +98 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/PKG-INFO +23 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/SOURCES.txt +19 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/dependency_links.txt +1 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/requires.txt +2 -0
- smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: smart-artificial-intelligence-local
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: PyPI smart-artificial-intelligence-local Python Package owned by Circlez.ai
|
|
5
|
+
Home-page: https://github.com/circles-zone/smart-artificial-intelligence-local-python-package
|
|
6
|
+
Author: Circles
|
|
7
|
+
Author-email: info@circlez.ai
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: python-sdk-remote
|
|
12
|
+
Requires-Dist: logger-local
|
|
13
|
+
Dynamic: author
|
|
14
|
+
Dynamic: author-email
|
|
15
|
+
Dynamic: classifier
|
|
16
|
+
Dynamic: description
|
|
17
|
+
Dynamic: description-content-type
|
|
18
|
+
Dynamic: home-page
|
|
19
|
+
Dynamic: requires-dist
|
|
20
|
+
Dynamic: summary
|
|
21
|
+
|
|
22
|
+
PyPI smart-artificial-intelligence-local Python Package owned by Circlez.ai
|
|
23
|
+
GHA: https://github.com/circles-zone/smart-artificial-intelligence-local-python-package/actions
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# smart-artificial-intelligence-local-python-package
|
|
2
|
+
|
|
3
|
+
Unlike the README.md in the root directory, this is the README.md of this specific repo/package.<br>
|
|
4
|
+
|
|
5
|
+
## General
|
|
6
|
+
|
|
7
|
+
Smart LLM orchestration layer: run the same prompt against multiple LLM engines and either
|
|
8
|
+
return all the answers (MULTIPLE_ANSWERS mode) or let a designated judge engine pick the best
|
|
9
|
+
one (JUDGE mode).
|
|
10
|
+
|
|
11
|
+
LLM engines are injected as plain Python callables `(prompt: str) -> str`.
|
|
12
|
+
This keeps this package decoupled from the actual LLM gateway
|
|
13
|
+
(llm-local-python-package, e.g. LM Studio / Ollama / OpenRouter / AWS Bedrock),
|
|
14
|
+
which will be integrated in a separate work item once it exists.
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from smart_artificial_intelligence_local.src.smart_artificial_intelligence_local import (
|
|
20
|
+
SmartArtificialIntelligenceLocal,
|
|
21
|
+
)
|
|
22
|
+
from smart_artificial_intelligence_local.src.llm_mode_enum import LlmMode
|
|
23
|
+
|
|
24
|
+
smart_ai = SmartArtificialIntelligenceLocal()
|
|
25
|
+
|
|
26
|
+
engines = {"engine_a": engine_a_callable, "engine_b": en
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# This file should be in the future instead of setup.py
|
|
2
|
+
# https://python-poetry.org/docs/pyproject
|
|
3
|
+
# https://stackoverflow.com/questions/78048223/adding-folder-with-data-with-pyproject-toml
|
|
4
|
+
# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license
|
|
5
|
+
|
|
6
|
+
# This file is mandatory for the `poetry version patch`
|
|
7
|
+
|
|
8
|
+
# It seems we need to copy this file also to serverless-com repo, as required by dialog-workflow-python-package
|
|
9
|
+
|
|
10
|
+
[build-system]
|
|
11
|
+
requires = ["setuptools>=61.0"]
|
|
12
|
+
build-backend = "setuptools.build_meta"
|
|
13
|
+
|
|
14
|
+
[tool.pytest.ini_options]
|
|
15
|
+
pythonpath = ["."]
|
|
16
|
+
|
|
17
|
+
[tool.poetry]
|
|
18
|
+
name = "smart-artificial-intelligence-local"
|
|
19
|
+
# I believe we are still using the version from setup.py and not from here until Potery will work
|
|
20
|
+
version = "0.0.1" # https://pypi.org/project/smart-artificial-intelligence-local
|
|
21
|
+
description = "smart-artificial-intelligence-local Python Package"
|
|
22
|
+
readme = "README.md"
|
|
23
|
+
authors = [
|
|
24
|
+
"Circlez.ai <info@circlez.ai>",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[tool.poetry.dev-dependencies]
|
|
28
|
+
pytest = "^8.0"
|
|
29
|
+
pytest-cov = "^5.0"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import setuptools
|
|
2
|
+
|
|
3
|
+
# Package Name is identical to the inner directory name (with dashes).
|
|
4
|
+
# TODO Add the Jira Work Item (Jira Issue) number as beta suffix when
|
|
5
|
+
# available, i.e. version='0.0.1b2432' (no Jira access at the moment).
|
|
6
|
+
PACKAGE_NAME = "smart-artificial-intelligence-local"
|
|
7
|
+
|
|
8
|
+
package_dir = PACKAGE_NAME.replace("-", "_")
|
|
9
|
+
|
|
10
|
+
PACKAGE_DESCRIPTION = f"PyPI {PACKAGE_NAME} Python Package owned by Circlez.ai"
|
|
11
|
+
|
|
12
|
+
setuptools.setup(
|
|
13
|
+
name=PACKAGE_NAME,
|
|
14
|
+
# Increase this number every time you make a change you want to publish.
|
|
15
|
+
# After 0.0.9 switch to 0.0.10 and not 0.1.0.
|
|
16
|
+
version='0.0.1',
|
|
17
|
+
author="Circles",
|
|
18
|
+
author_email="info@circlez.ai",
|
|
19
|
+
description=PACKAGE_DESCRIPTION,
|
|
20
|
+
long_description=(
|
|
21
|
+
f"{PACKAGE_DESCRIPTION}\n"
|
|
22
|
+
f"GHA: https://github.com/circles-zone/{PACKAGE_NAME}"
|
|
23
|
+
"-python-package/actions"
|
|
24
|
+
),
|
|
25
|
+
long_description_content_type='text/markdown',
|
|
26
|
+
url=f"https://github.com/circles-zone/{PACKAGE_NAME}-python-package",
|
|
27
|
+
packages=setuptools.find_packages(),
|
|
28
|
+
package_data={package_dir: ['*.py']},
|
|
29
|
+
classifiers=[
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Operating System :: OS Independent",
|
|
32
|
+
],
|
|
33
|
+
# Production/runtime dependencies only (test packages belong in
|
|
34
|
+
# requirements-dev.txt).
|
|
35
|
+
install_requires=[
|
|
36
|
+
'python-sdk-remote',
|
|
37
|
+
'logger-local',
|
|
38
|
+
]
|
|
39
|
+
)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Constants of Smart Artificial Intelligence Local."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ConstantsSrcSmartArtificialIntelligenceLocal:
|
|
5
|
+
"""All the constants of Smart Artificial Intelligence Local."""
|
|
6
|
+
|
|
7
|
+
# TODO Replace the placeholder component ids with real ones from Slack
|
|
8
|
+
SMART_AI_CODE_COMPONENT_ID: int = 0
|
|
9
|
+
SMART_AI_COMPONENT_NAME = (
|
|
10
|
+
"Smart Artificial Intelligence Local Python package"
|
|
11
|
+
)
|
|
12
|
+
SMART_AI_DEVELOPER_EMAIL_ADDRESS: str = "lielchochabi@gmail.com"
|
|
13
|
+
|
|
14
|
+
SMART_AI_CODE_LOGGER_OBJECT: dict = {
|
|
15
|
+
'component_id': SMART_AI_CODE_COMPONENT_ID,
|
|
16
|
+
'component_name': SMART_AI_COMPONENT_NAME,
|
|
17
|
+
'component_category': 'Code',
|
|
18
|
+
'developer_email_address': SMART_AI_DEVELOPER_EMAIL_ADDRESS,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
DEFAULT_MAX_RESPONSE_TIME_SECONDS: float = 60.0
|
smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/src/llm_mode_enum.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import enum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class LlmMode(enum.Enum):
|
|
5
|
+
"""Smart LLM modes (from the challenge diagram).
|
|
6
|
+
|
|
7
|
+
Mode 1: LLM as a Judge - a designated judge engine picks the best answer.
|
|
8
|
+
Mode 2: return multiple answers (and get reaction).
|
|
9
|
+
"""
|
|
10
|
+
JUDGE = 1
|
|
11
|
+
MULTIPLE_ANSWERS = 2
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""Smart LLM orchestration (Smart LLM box in the challenge diagram).
|
|
2
|
+
|
|
3
|
+
Can call multiple LLM engines with the same prompt and either:
|
|
4
|
+
- Mode 1 (JUDGE): a designated judge engine picks the best answer.
|
|
5
|
+
- Mode 2 (MULTIPLE_ANSWERS): return all the answers (and get reaction).
|
|
6
|
+
|
|
7
|
+
Engines are injected as plain callables ``(prompt: str) -> str`` so this
|
|
8
|
+
package stays decoupled from the actual LLM gateway
|
|
9
|
+
(llm-local-python-package), which is a separate work item.
|
|
10
|
+
"""
|
|
11
|
+
import random
|
|
12
|
+
import re
|
|
13
|
+
import time
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from concurrent.futures import TimeoutError as FutureTimeoutError
|
|
16
|
+
from typing import Callable, Dict, Optional
|
|
17
|
+
|
|
18
|
+
from python_sdk_remote.our_object import OurObject
|
|
19
|
+
|
|
20
|
+
from .constants_src_smart_artificial_intelligence_local import (
|
|
21
|
+
ConstantsSrcSmartArtificialIntelligenceLocal as Constants,
|
|
22
|
+
)
|
|
23
|
+
from .llm_mode_enum import LlmMode
|
|
24
|
+
from .smart_llm_logger import create_smart_llm_logger
|
|
25
|
+
from .smart_llm_result import SmartLlmResult
|
|
26
|
+
|
|
27
|
+
ENTITY_NAME = "Smart Artificial Intelligence Local"
|
|
28
|
+
|
|
29
|
+
EngineCallable = Callable[[str], str]
|
|
30
|
+
|
|
31
|
+
logger = create_smart_llm_logger()
|
|
32
|
+
|
|
33
|
+
JUDGE_PROMPT_TEMPLATE = (
|
|
34
|
+
"You are a judge. Below is a prompt and several candidate answers,"
|
|
35
|
+
" each produced by a different LLM engine.\n"
|
|
36
|
+
"Prompt:\n{prompt}\n\n"
|
|
37
|
+
"Candidate answers:\n{candidates}\n"
|
|
38
|
+
"Reply with the name of the engine that produced the best answer."
|
|
39
|
+
" Reply with the engine name only."
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
JUDGE_CANDIDATE_TEMPLATE = "Engine name: {engine_name}\nAnswer: {answer}\n"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class SmartArtificialIntelligenceLocal(OurObject):
|
|
46
|
+
"""Smart LLM: run one prompt against multiple injected LLM engines."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, entity_name: str = ENTITY_NAME, **kwargs) -> None:
|
|
49
|
+
super().__init__(entity_name, **kwargs)
|
|
50
|
+
self.entity_name = entity_name
|
|
51
|
+
|
|
52
|
+
def get_name(self) -> str:
|
|
53
|
+
return self.entity_name
|
|
54
|
+
|
|
55
|
+
def ask(self, *, prompt: str, engines: Dict[str, EngineCallable],
|
|
56
|
+
llm_mode: LlmMode = LlmMode.MULTIPLE_ANSWERS,
|
|
57
|
+
judge_engine: Optional[EngineCallable] = None,
|
|
58
|
+
force_engine_name: Optional[str] = None,
|
|
59
|
+
max_response_time_seconds: Optional[float] = None
|
|
60
|
+
) -> SmartLlmResult:
|
|
61
|
+
"""Run the prompt against the engines according to llm_mode.
|
|
62
|
+
|
|
63
|
+
prompt: the prompt to send to every engine.
|
|
64
|
+
engines: engine name -> callable(prompt) -> answer.
|
|
65
|
+
llm_mode: JUDGE or MULTIPLE_ANSWERS.
|
|
66
|
+
judge_engine: callable used as the judge (mandatory in JUDGE mode).
|
|
67
|
+
force_engine_name: if set, only this engine is called
|
|
68
|
+
("Force LLM Engine" in the challenge diagram).
|
|
69
|
+
max_response_time_seconds: wall-clock cap per ask() fan-out; engines
|
|
70
|
+
that exceed it are reported in result.errors.
|
|
71
|
+
"""
|
|
72
|
+
logger.start("ask", object={
|
|
73
|
+
'llm_mode': str(llm_mode),
|
|
74
|
+
'engine_names': list(engines or {}),
|
|
75
|
+
'force_engine_name': force_engine_name,
|
|
76
|
+
})
|
|
77
|
+
self._validate_ask_arguments(
|
|
78
|
+
prompt=prompt, engines=engines, llm_mode=llm_mode,
|
|
79
|
+
judge_engine=judge_engine, force_engine_name=force_engine_name)
|
|
80
|
+
selected_engines = self._select_engines(
|
|
81
|
+
engines=engines, force_engine_name=force_engine_name)
|
|
82
|
+
answers, errors = self._collect_answers(
|
|
83
|
+
prompt=prompt, engines=selected_engines,
|
|
84
|
+
max_response_time_seconds=max_response_time_seconds)
|
|
85
|
+
if not answers:
|
|
86
|
+
logger.error("ask: all engines failed", object={'errors': errors})
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"All engines failed for prompt. errors={errors}")
|
|
89
|
+
result = SmartLlmResult(
|
|
90
|
+
prompt=prompt, llm_mode=llm_mode, answers=answers, errors=errors)
|
|
91
|
+
if llm_mode == LlmMode.JUDGE:
|
|
92
|
+
self._judge(result=result, judge_engine=judge_engine)
|
|
93
|
+
logger.end("ask", object={
|
|
94
|
+
'winner_engine_name': result.winner_engine_name,
|
|
95
|
+
'answered_engine_names': list(result.answers),
|
|
96
|
+
'error_engine_names': list(result.errors),
|
|
97
|
+
})
|
|
98
|
+
return result
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def _validate_ask_arguments(
|
|
102
|
+
*, prompt: str, engines: Dict[str, EngineCallable],
|
|
103
|
+
llm_mode: LlmMode, judge_engine: Optional[EngineCallable],
|
|
104
|
+
force_engine_name: Optional[str]) -> None:
|
|
105
|
+
if not prompt:
|
|
106
|
+
raise ValueError("prompt must be a non-empty string")
|
|
107
|
+
if not engines:
|
|
108
|
+
raise ValueError("engines must be a non-empty dict")
|
|
109
|
+
if llm_mode == LlmMode.JUDGE and judge_engine is None:
|
|
110
|
+
raise ValueError("judge_engine is mandatory in JUDGE llm_mode")
|
|
111
|
+
if force_engine_name is not None and force_engine_name not in engines:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"force_engine_name '{force_engine_name}' is not one of the"
|
|
114
|
+
f" engines {list(engines)}")
|
|
115
|
+
|
|
116
|
+
@staticmethod
|
|
117
|
+
def _select_engines(
|
|
118
|
+
*, engines: Dict[str, EngineCallable],
|
|
119
|
+
force_engine_name: Optional[str]) -> Dict[str, EngineCallable]:
|
|
120
|
+
if force_engine_name is None:
|
|
121
|
+
return engines
|
|
122
|
+
return {force_engine_name: engines[force_engine_name]}
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def _collect_answers(
|
|
126
|
+
*, prompt: str, engines: Dict[str, EngineCallable],
|
|
127
|
+
max_response_time_seconds: Optional[float]) -> tuple:
|
|
128
|
+
"""Fan the prompt out to all engines concurrently.
|
|
129
|
+
|
|
130
|
+
Returns (answers, errors) dicts keyed by engine name. An engine that
|
|
131
|
+
raises or exceeds max_response_time_seconds lands in errors, so one
|
|
132
|
+
bad engine never fails the whole run.
|
|
133
|
+
"""
|
|
134
|
+
if max_response_time_seconds is None:
|
|
135
|
+
max_response_time_seconds = (
|
|
136
|
+
Constants.DEFAULT_MAX_RESPONSE_TIME_SECONDS)
|
|
137
|
+
answers: Dict[str, str] = {}
|
|
138
|
+
errors: Dict[str, str] = {}
|
|
139
|
+
# Manual shutdown below; a context manager would wait for
|
|
140
|
+
# timed-out engines.
|
|
141
|
+
executor = ThreadPoolExecutor(max_workers=len(engines))
|
|
142
|
+
futures = {
|
|
143
|
+
engine_name: executor.submit(engine, prompt)
|
|
144
|
+
for engine_name, engine in engines.items()
|
|
145
|
+
}
|
|
146
|
+
# Shared deadline for the whole fan-out
|
|
147
|
+
deadline = time.monotonic() + max_response_time_seconds
|
|
148
|
+
for engine_name, future in futures.items():
|
|
149
|
+
remaining_seconds = max(0.0, deadline - time.monotonic())
|
|
150
|
+
try:
|
|
151
|
+
answers[engine_name] = future.result(
|
|
152
|
+
timeout=remaining_seconds)
|
|
153
|
+
except FutureTimeoutError:
|
|
154
|
+
errors[engine_name] = (
|
|
155
|
+
f"engine '{engine_name}' timed out after"
|
|
156
|
+
f" {max_response_time_seconds} seconds")
|
|
157
|
+
except Exception as exception: # noqa: BLE001
|
|
158
|
+
errors[engine_name] = (
|
|
159
|
+
f"engine '{engine_name}' failed: {exception}")
|
|
160
|
+
# Do not wait for timed-out engines
|
|
161
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
162
|
+
return answers, errors
|
|
163
|
+
|
|
164
|
+
def _judge(self, *, result: SmartLlmResult,
|
|
165
|
+
judge_engine: EngineCallable) -> None:
|
|
166
|
+
"""Ask the judge engine to pick the best answer (LLM as a Judge)."""
|
|
167
|
+
judge_prompt = self._build_judge_prompt(
|
|
168
|
+
prompt=result.prompt, answers=result.answers)
|
|
169
|
+
result.judge_raw_answer = judge_engine(judge_prompt)
|
|
170
|
+
result.winner_engine_name = self._parse_judge_verdict(
|
|
171
|
+
judge_raw_answer=result.judge_raw_answer,
|
|
172
|
+
engine_names=list(result.answers))
|
|
173
|
+
result.winner_answer = result.answers[result.winner_engine_name]
|
|
174
|
+
|
|
175
|
+
@staticmethod
|
|
176
|
+
def _build_judge_prompt(*, prompt: str, answers: Dict[str, str]) -> str:
|
|
177
|
+
# Randomize candidate order to reduce judge position bias
|
|
178
|
+
candidate_items = list(answers.items())
|
|
179
|
+
random.shuffle(candidate_items)
|
|
180
|
+
candidates = "\n".join(
|
|
181
|
+
JUDGE_CANDIDATE_TEMPLATE.format(
|
|
182
|
+
engine_name=engine_name, answer=answer)
|
|
183
|
+
for engine_name, answer in candidate_items
|
|
184
|
+
)
|
|
185
|
+
return JUDGE_PROMPT_TEMPLATE.format(
|
|
186
|
+
prompt=prompt, candidates=candidates)
|
|
187
|
+
|
|
188
|
+
@staticmethod
|
|
189
|
+
def _parse_judge_verdict(*, judge_raw_answer: str,
|
|
190
|
+
engine_names: list) -> str:
|
|
191
|
+
"""Extract the winning engine name from the judge's verdict text."""
|
|
192
|
+
verdict = (judge_raw_answer or "").strip()
|
|
193
|
+
if verdict in engine_names:
|
|
194
|
+
return verdict
|
|
195
|
+
mentioned_engine_names = [
|
|
196
|
+
engine_name for engine_name in engine_names
|
|
197
|
+
if re.search(
|
|
198
|
+
rf"(?<!\w){re.escape(engine_name)}(?!\w)", verdict)
|
|
199
|
+
]
|
|
200
|
+
if len(mentioned_engine_names) == 1:
|
|
201
|
+
return mentioned_engine_names[0]
|
|
202
|
+
raise ValueError(
|
|
203
|
+
"Could not parse the judge verdict into exactly one engine"
|
|
204
|
+
f" name. verdict='{verdict}' engines={engine_names}"
|
|
205
|
+
f" mentioned={mentioned_engine_names}")
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Logger for this package.
|
|
2
|
+
|
|
3
|
+
Primary path is the circles-zone logger-local package (per company
|
|
4
|
+
conventions). If it cannot be initialized (e.g. missing environment /
|
|
5
|
+
credentials in a developer machine), we fall back to the standard library
|
|
6
|
+
logging so the package stays importable and testable everywhere.
|
|
7
|
+
"""
|
|
8
|
+
import logging
|
|
9
|
+
|
|
10
|
+
from .constants_src_smart_artificial_intelligence_local import (
|
|
11
|
+
ConstantsSrcSmartArtificialIntelligenceLocal as Constants,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _FallbackLogger:
|
|
16
|
+
"""Minimal start/end/info/error/exception adapter over stdlib logging."""
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
self._logger = logging.getLogger(Constants.SMART_AI_COMPONENT_NAME)
|
|
20
|
+
|
|
21
|
+
def start(self, message: str = "", **kwargs) -> None:
|
|
22
|
+
self._logger.debug("START %s %s", message, kwargs or "")
|
|
23
|
+
|
|
24
|
+
def end(self, message: str = "", **kwargs) -> None:
|
|
25
|
+
self._logger.debug("END %s %s", message, kwargs or "")
|
|
26
|
+
|
|
27
|
+
def info(self, message: str = "", **kwargs) -> None:
|
|
28
|
+
self._logger.info("%s %s", message, kwargs or "")
|
|
29
|
+
|
|
30
|
+
def error(self, message: str = "", **kwargs) -> None:
|
|
31
|
+
self._logger.error("%s %s", message, kwargs or "")
|
|
32
|
+
|
|
33
|
+
def exception(self, message: str = "", **kwargs) -> None:
|
|
34
|
+
self._logger.exception("%s %s", message, kwargs or "")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def create_smart_llm_logger():
|
|
38
|
+
"""Create the circles-zone logger-local Logger, or a stdlib fallback."""
|
|
39
|
+
try:
|
|
40
|
+
from logger_local.LoggerLocal import Logger
|
|
41
|
+
return Logger.create_logger(
|
|
42
|
+
object=Constants.SMART_AI_CODE_LOGGER_OBJECT)
|
|
43
|
+
except Exception: # noqa: BLE001 - any init failure means fallback
|
|
44
|
+
return _FallbackLogger()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from .llm_mode_enum import LlmMode
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class SmartLlmResult:
|
|
9
|
+
"""Result of a SmartArtificialIntelligenceLocal.ask() call.
|
|
10
|
+
|
|
11
|
+
answers: successful answers per engine name.
|
|
12
|
+
errors: error message per engine name (failed or timed-out engines).
|
|
13
|
+
winner_engine_name / winner_answer: set only in JUDGE mode.
|
|
14
|
+
judge_raw_answer: the raw verdict text returned by the judge engine.
|
|
15
|
+
"""
|
|
16
|
+
prompt: str
|
|
17
|
+
llm_mode: LlmMode
|
|
18
|
+
answers: dict = field(default_factory=dict)
|
|
19
|
+
errors: dict = field(default_factory=dict)
|
|
20
|
+
winner_engine_name: Optional[str] = None
|
|
21
|
+
winner_answer: Optional[str] = None
|
|
22
|
+
judge_raw_answer: Optional[str] = None
|
smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local/tests/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
class SmartArtificialIntelligenceLocalTestsConstants:
|
|
2
|
+
# TODO Replace the placeholder component id with a real one from Slack
|
|
3
|
+
SMART_AI_TEST_COMPONENT_ID: int = 0
|
|
4
|
+
SMART_AI_TEST_COMPONENT_NAME = (
|
|
5
|
+
"Smart Artificial Intelligence Local Tests"
|
|
6
|
+
)
|
|
7
|
+
SMART_AI_DEVELOPER_EMAIL_ADDRESS = "lielchochabi@gmail.com"
|
|
8
|
+
|
|
9
|
+
SMART_AI_TEST_LOGGER_OBJECT = {
|
|
10
|
+
'component_id': SMART_AI_TEST_COMPONENT_ID,
|
|
11
|
+
'component_name': SMART_AI_TEST_COMPONENT_NAME,
|
|
12
|
+
'component_category': 'UnitTest',
|
|
13
|
+
'testing_framework': 'pytest',
|
|
14
|
+
'developer_email_address': SMART_AI_DEVELOPER_EMAIL_ADDRESS,
|
|
15
|
+
}
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
"""Unit tests (with mock engines) for SmartArtificialIntelligenceLocal.
|
|
2
|
+
|
|
3
|
+
TDD: these tests were written first (red) and then the implementation was
|
|
4
|
+
added to make them pass (green).
|
|
5
|
+
"""
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from unittest.mock import Mock
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from ..src.llm_mode_enum import LlmMode
|
|
13
|
+
from ..src.smart_artificial_intelligence_local import (
|
|
14
|
+
SmartArtificialIntelligenceLocal,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
PROMPT = "Summarize: the quick brown fox jumps over the lazy dog."
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.fixture
|
|
21
|
+
def smart_ai() -> SmartArtificialIntelligenceLocal:
|
|
22
|
+
return SmartArtificialIntelligenceLocal(is_test_data=True)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_llm_mode_enum_has_judge_and_multiple_answers():
|
|
26
|
+
# Mode 1 = LLM as a Judge, Mode 2 = return multiple answers (diagram)
|
|
27
|
+
assert LlmMode.JUDGE.value == 1
|
|
28
|
+
assert LlmMode.MULTIPLE_ANSWERS.value == 2
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_get_name_returns_entity_name(smart_ai):
|
|
32
|
+
assert "Smart Artificial Intelligence" in smart_ai.get_name()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_multiple_answers_mode_returns_all_answers(smart_ai):
|
|
36
|
+
engine_a = Mock(return_value="answer from a")
|
|
37
|
+
engine_b = Mock(return_value="answer from b")
|
|
38
|
+
result = smart_ai.ask(
|
|
39
|
+
prompt=PROMPT,
|
|
40
|
+
engines={"engine_a": engine_a, "engine_b": engine_b},
|
|
41
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
42
|
+
)
|
|
43
|
+
assert result.answers == {
|
|
44
|
+
"engine_a": "answer from a",
|
|
45
|
+
"engine_b": "answer from b",
|
|
46
|
+
}
|
|
47
|
+
assert result.errors == {}
|
|
48
|
+
assert result.winner_engine_name is None
|
|
49
|
+
engine_a.assert_called_once_with(PROMPT)
|
|
50
|
+
engine_b.assert_called_once_with(PROMPT)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_multiple_answers_mode_collects_engine_errors(smart_ai):
|
|
54
|
+
engine_ok = Mock(return_value="fine")
|
|
55
|
+
engine_bad = Mock(side_effect=RuntimeError("engine down"))
|
|
56
|
+
result = smart_ai.ask(
|
|
57
|
+
prompt=PROMPT,
|
|
58
|
+
engines={"engine_ok": engine_ok, "engine_bad": engine_bad},
|
|
59
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
60
|
+
)
|
|
61
|
+
assert result.answers == {"engine_ok": "fine"}
|
|
62
|
+
assert "engine_bad" in result.errors
|
|
63
|
+
assert "engine down" in result.errors["engine_bad"]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_all_engines_failing_raises(smart_ai):
|
|
67
|
+
engine_bad = Mock(side_effect=RuntimeError("down"))
|
|
68
|
+
with pytest.raises(ValueError):
|
|
69
|
+
smart_ai.ask(
|
|
70
|
+
prompt=PROMPT,
|
|
71
|
+
engines={"engine_bad": engine_bad},
|
|
72
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_empty_prompt_raises(smart_ai):
|
|
77
|
+
with pytest.raises(ValueError):
|
|
78
|
+
smart_ai.ask(
|
|
79
|
+
prompt="",
|
|
80
|
+
engines={"engine_a": Mock(return_value="x")},
|
|
81
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_empty_engines_raises(smart_ai):
|
|
86
|
+
with pytest.raises(ValueError):
|
|
87
|
+
smart_ai.ask(
|
|
88
|
+
prompt=PROMPT,
|
|
89
|
+
engines={},
|
|
90
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_judge_mode_returns_winner_picked_by_judge(smart_ai):
|
|
95
|
+
engine_a = Mock(return_value="short answer")
|
|
96
|
+
engine_b = Mock(return_value="a much better detailed answer")
|
|
97
|
+
judge_engine = Mock(return_value="engine_b")
|
|
98
|
+
result = smart_ai.ask(
|
|
99
|
+
prompt=PROMPT,
|
|
100
|
+
engines={"engine_a": engine_a, "engine_b": engine_b},
|
|
101
|
+
llm_mode=LlmMode.JUDGE,
|
|
102
|
+
judge_engine=judge_engine,
|
|
103
|
+
)
|
|
104
|
+
assert result.winner_engine_name == "engine_b"
|
|
105
|
+
assert result.winner_answer == "a much better detailed answer"
|
|
106
|
+
# All answers are still returned alongside the verdict
|
|
107
|
+
assert set(result.answers) == {"engine_a", "engine_b"}
|
|
108
|
+
judge_engine.assert_called_once()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_judge_prompt_contains_original_prompt_and_answers(smart_ai):
|
|
112
|
+
judge_engine = Mock(return_value="engine_a")
|
|
113
|
+
smart_ai.ask(
|
|
114
|
+
prompt=PROMPT,
|
|
115
|
+
engines={
|
|
116
|
+
"engine_a": Mock(return_value="alpha answer"),
|
|
117
|
+
"engine_b": Mock(return_value="beta answer"),
|
|
118
|
+
},
|
|
119
|
+
llm_mode=LlmMode.JUDGE,
|
|
120
|
+
judge_engine=judge_engine,
|
|
121
|
+
)
|
|
122
|
+
judge_prompt = judge_engine.call_args[0][0]
|
|
123
|
+
assert PROMPT in judge_prompt
|
|
124
|
+
assert "alpha answer" in judge_prompt
|
|
125
|
+
assert "beta answer" in judge_prompt
|
|
126
|
+
assert "engine_a" in judge_prompt
|
|
127
|
+
assert "engine_b" in judge_prompt
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_judge_verdict_embedded_in_sentence_is_parsed(smart_ai):
|
|
131
|
+
judge_engine = Mock(
|
|
132
|
+
return_value="The best answer is the one from engine_b."
|
|
133
|
+
)
|
|
134
|
+
result = smart_ai.ask(
|
|
135
|
+
prompt=PROMPT,
|
|
136
|
+
engines={
|
|
137
|
+
"engine_a": Mock(return_value="alpha"),
|
|
138
|
+
"engine_b": Mock(return_value="beta"),
|
|
139
|
+
},
|
|
140
|
+
llm_mode=LlmMode.JUDGE,
|
|
141
|
+
judge_engine=judge_engine,
|
|
142
|
+
)
|
|
143
|
+
assert result.winner_engine_name == "engine_b"
|
|
144
|
+
assert result.winner_answer == "beta"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_judge_mode_without_judge_engine_raises(smart_ai):
|
|
148
|
+
with pytest.raises(ValueError):
|
|
149
|
+
smart_ai.ask(
|
|
150
|
+
prompt=PROMPT,
|
|
151
|
+
engines={"engine_a": Mock(return_value="x")},
|
|
152
|
+
llm_mode=LlmMode.JUDGE,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_unparseable_judge_verdict_raises(smart_ai):
|
|
157
|
+
judge_engine = Mock(return_value="I cannot decide at all")
|
|
158
|
+
with pytest.raises(ValueError):
|
|
159
|
+
smart_ai.ask(
|
|
160
|
+
prompt=PROMPT,
|
|
161
|
+
engines={
|
|
162
|
+
"engine_a": Mock(return_value="alpha"),
|
|
163
|
+
"engine_b": Mock(return_value="beta"),
|
|
164
|
+
},
|
|
165
|
+
llm_mode=LlmMode.JUDGE,
|
|
166
|
+
judge_engine=judge_engine,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_force_engine_runs_only_that_engine(smart_ai):
|
|
171
|
+
engine_a = Mock(return_value="from a")
|
|
172
|
+
engine_b = Mock(return_value="from b")
|
|
173
|
+
result = smart_ai.ask(
|
|
174
|
+
prompt=PROMPT,
|
|
175
|
+
engines={"engine_a": engine_a, "engine_b": engine_b},
|
|
176
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
177
|
+
force_engine_name="engine_a",
|
|
178
|
+
)
|
|
179
|
+
assert result.answers == {"engine_a": "from a"}
|
|
180
|
+
engine_b.assert_not_called()
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def test_force_unknown_engine_raises(smart_ai):
|
|
184
|
+
with pytest.raises(ValueError):
|
|
185
|
+
smart_ai.ask(
|
|
186
|
+
prompt=PROMPT,
|
|
187
|
+
engines={"engine_a": Mock(return_value="x")},
|
|
188
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
189
|
+
force_engine_name="no_such_engine",
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def test_max_response_time_times_out_slow_engine(smart_ai):
|
|
194
|
+
def slow_engine(prompt: str) -> str:
|
|
195
|
+
time.sleep(2)
|
|
196
|
+
return "too late"
|
|
197
|
+
|
|
198
|
+
fast_engine = Mock(return_value="fast answer")
|
|
199
|
+
result = smart_ai.ask(
|
|
200
|
+
prompt=PROMPT,
|
|
201
|
+
engines={"slow": slow_engine, "fast": fast_engine},
|
|
202
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
203
|
+
max_response_time_seconds=0.3,
|
|
204
|
+
)
|
|
205
|
+
assert result.answers == {"fast": "fast answer"}
|
|
206
|
+
assert "slow" in result.errors
|
|
207
|
+
assert "timed out" in result.errors["slow"].lower()
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
if __name__ == "__main__":
|
|
211
|
+
pytest.main(sys.argv[1:])
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def test_create_smart_llm_logger_uses_logger_local_when_available(
|
|
215
|
+
monkeypatch):
|
|
216
|
+
import types
|
|
217
|
+
from ..src import smart_llm_logger
|
|
218
|
+
|
|
219
|
+
fake_logger = Mock()
|
|
220
|
+
fake_logger_class = Mock()
|
|
221
|
+
fake_logger_class.create_logger = Mock(return_value=fake_logger)
|
|
222
|
+
fake_module = types.ModuleType("logger_local.LoggerLocal")
|
|
223
|
+
fake_module.Logger = fake_logger_class
|
|
224
|
+
fake_package = types.ModuleType("logger_local")
|
|
225
|
+
fake_package.LoggerLocal = fake_module
|
|
226
|
+
monkeypatch.setitem(sys.modules, "logger_local", fake_package)
|
|
227
|
+
monkeypatch.setitem(sys.modules, "logger_local.LoggerLocal", fake_module)
|
|
228
|
+
|
|
229
|
+
created_logger = smart_llm_logger.create_smart_llm_logger()
|
|
230
|
+
assert created_logger is fake_logger
|
|
231
|
+
fake_logger_class.create_logger.assert_called_once()
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def test_create_smart_llm_logger_falls_back_when_logger_local_fails(
|
|
235
|
+
monkeypatch):
|
|
236
|
+
from ..src import smart_llm_logger
|
|
237
|
+
|
|
238
|
+
monkeypatch.setitem(sys.modules, "logger_local", None)
|
|
239
|
+
created_logger = smart_llm_logger.create_smart_llm_logger()
|
|
240
|
+
assert isinstance(created_logger, smart_llm_logger._FallbackLogger)
|
|
241
|
+
# The fallback logger must expose the full logging interface
|
|
242
|
+
created_logger.start("m")
|
|
243
|
+
created_logger.end("m")
|
|
244
|
+
created_logger.info("m")
|
|
245
|
+
created_logger.error("m")
|
|
246
|
+
created_logger.exception("m")
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def test_ask_returns_promptly_when_engine_exceeds_timeout(smart_ai):
|
|
250
|
+
"""max response time must cap wall-clock time of ask() itself.
|
|
251
|
+
|
|
252
|
+
A timed-out engine keeps running in its worker thread (Python cannot
|
|
253
|
+
kill it), but ask() must NOT wait for it to finish.
|
|
254
|
+
"""
|
|
255
|
+
def very_slow_engine(prompt: str) -> str:
|
|
256
|
+
time.sleep(3)
|
|
257
|
+
return "too late"
|
|
258
|
+
|
|
259
|
+
fast_engine = Mock(return_value="fast answer")
|
|
260
|
+
started_at = time.monotonic()
|
|
261
|
+
result = smart_ai.ask(
|
|
262
|
+
prompt=PROMPT,
|
|
263
|
+
engines={"very_slow": very_slow_engine, "fast": fast_engine},
|
|
264
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
265
|
+
max_response_time_seconds=0.3,
|
|
266
|
+
)
|
|
267
|
+
elapsed_seconds = time.monotonic() - started_at
|
|
268
|
+
assert elapsed_seconds < 1.5
|
|
269
|
+
assert result.answers == {"fast": "fast answer"}
|
|
270
|
+
assert "very_slow" in result.errors
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def test_judge_candidates_order_is_shuffled(smart_ai, monkeypatch):
|
|
274
|
+
"""Candidate order must be randomized to mitigate judge position bias."""
|
|
275
|
+
from ..src import smart_artificial_intelligence_local as module
|
|
276
|
+
|
|
277
|
+
monkeypatch.setattr(
|
|
278
|
+
module.random, "shuffle", lambda items: items.reverse())
|
|
279
|
+
judge_engine = Mock(return_value="engine_a")
|
|
280
|
+
smart_ai.ask(
|
|
281
|
+
prompt=PROMPT,
|
|
282
|
+
engines={
|
|
283
|
+
"engine_a": Mock(return_value="alpha"),
|
|
284
|
+
"engine_b": Mock(return_value="beta"),
|
|
285
|
+
},
|
|
286
|
+
llm_mode=LlmMode.JUDGE,
|
|
287
|
+
judge_engine=judge_engine,
|
|
288
|
+
)
|
|
289
|
+
judge_prompt = judge_engine.call_args[0][0]
|
|
290
|
+
# With shuffle patched to reverse, engine_b must come first
|
|
291
|
+
assert judge_prompt.index("Engine name: engine_b") < \
|
|
292
|
+
judge_prompt.index("Engine name: engine_a")
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""System tests (without mock) for SmartArtificialIntelligenceLocal.
|
|
2
|
+
|
|
3
|
+
Runs the full Smart LLM flow end-to-end through the public API with real
|
|
4
|
+
engine callables (deterministic fake LLM engines implemented as plain
|
|
5
|
+
functions) - no mocking framework anywhere in this file.
|
|
6
|
+
"""
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from ..src.llm_mode_enum import LlmMode
|
|
12
|
+
from ..src.smart_artificial_intelligence_local import (
|
|
13
|
+
SmartArtificialIntelligenceLocal,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
DOCUMENT = (
|
|
17
|
+
"Circles Zone builds reusable packages. The Smart LLM component calls"
|
|
18
|
+
" multiple LLM engines with the same prompt. A judge engine can pick"
|
|
19
|
+
" the best answer."
|
|
20
|
+
)
|
|
21
|
+
PROMPT = f"Summarize the following document:\n{DOCUMENT}"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def first_sentence_engine(prompt: str) -> str:
|
|
25
|
+
"""Fake LLM: 'summarizes' by returning the first sentence it finds."""
|
|
26
|
+
document = prompt.split("\n", 1)[1]
|
|
27
|
+
return document.split(".")[0].strip() + "."
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def keyword_engine(prompt: str) -> str:
|
|
31
|
+
"""Fake LLM: 'summarizes' by listing capitalized words."""
|
|
32
|
+
document = prompt.split("\n", 1)[1]
|
|
33
|
+
keywords = sorted({
|
|
34
|
+
word.strip(".,") for word in document.split()
|
|
35
|
+
if word[:1].isupper()
|
|
36
|
+
})
|
|
37
|
+
return "Keywords: " + ", ".join(keywords)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def longest_answer_judge(judge_prompt: str) -> str:
|
|
41
|
+
"""Fake judge LLM: picks the engine whose answer text is the longest.
|
|
42
|
+
|
|
43
|
+
Parses the judge prompt built by the package (Engine name: X / Answer: Y
|
|
44
|
+
blocks) exactly as a real LLM would read it.
|
|
45
|
+
"""
|
|
46
|
+
best_engine_name = ""
|
|
47
|
+
best_length = -1
|
|
48
|
+
current_engine_name = None
|
|
49
|
+
for line in judge_prompt.splitlines():
|
|
50
|
+
if line.startswith("Engine name: "):
|
|
51
|
+
current_engine_name = line.removeprefix("Engine name: ").strip()
|
|
52
|
+
elif line.startswith("Answer: ") and current_engine_name:
|
|
53
|
+
answer_length = len(line.removeprefix("Answer: "))
|
|
54
|
+
if answer_length > best_length:
|
|
55
|
+
best_length = answer_length
|
|
56
|
+
best_engine_name = current_engine_name
|
|
57
|
+
return best_engine_name
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_system_multiple_answers_end_to_end():
|
|
61
|
+
smart_ai = SmartArtificialIntelligenceLocal()
|
|
62
|
+
result = smart_ai.ask(
|
|
63
|
+
prompt=PROMPT,
|
|
64
|
+
engines={
|
|
65
|
+
"first_sentence": first_sentence_engine,
|
|
66
|
+
"keyword": keyword_engine,
|
|
67
|
+
},
|
|
68
|
+
llm_mode=LlmMode.MULTIPLE_ANSWERS,
|
|
69
|
+
)
|
|
70
|
+
assert result.answers["first_sentence"] == (
|
|
71
|
+
"Circles Zone builds reusable packages.")
|
|
72
|
+
assert result.answers["keyword"].startswith("Keywords: ")
|
|
73
|
+
assert "Circles" in result.answers["keyword"]
|
|
74
|
+
assert result.errors == {}
|
|
75
|
+
assert result.winner_engine_name is None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_system_judge_end_to_end_picks_expected_winner():
|
|
79
|
+
smart_ai = SmartArtificialIntelligenceLocal()
|
|
80
|
+
result = smart_ai.ask(
|
|
81
|
+
prompt=PROMPT,
|
|
82
|
+
engines={
|
|
83
|
+
"first_sentence": first_sentence_engine,
|
|
84
|
+
"keyword": keyword_engine,
|
|
85
|
+
},
|
|
86
|
+
llm_mode=LlmMode.JUDGE,
|
|
87
|
+
judge_engine=longest_answer_judge,
|
|
88
|
+
max_response_time_seconds=30,
|
|
89
|
+
)
|
|
90
|
+
# The keyword answer is longer, so the longest-answer judge must pick it
|
|
91
|
+
assert result.winner_engine_name == "keyword"
|
|
92
|
+
assert result.winner_answer == result.answers["keyword"]
|
|
93
|
+
assert result.judge_raw_answer == "keyword"
|
|
94
|
+
assert set(result.answers) == {"first_sentence", "keyword"}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
pytest.main(sys.argv[1:])
|
smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: smart-artificial-intelligence-local
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: PyPI smart-artificial-intelligence-local Python Package owned by Circlez.ai
|
|
5
|
+
Home-page: https://github.com/circles-zone/smart-artificial-intelligence-local-python-package
|
|
6
|
+
Author: Circles
|
|
7
|
+
Author-email: info@circlez.ai
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: python-sdk-remote
|
|
12
|
+
Requires-Dist: logger-local
|
|
13
|
+
Dynamic: author
|
|
14
|
+
Dynamic: author-email
|
|
15
|
+
Dynamic: classifier
|
|
16
|
+
Dynamic: description
|
|
17
|
+
Dynamic: description-content-type
|
|
18
|
+
Dynamic: home-page
|
|
19
|
+
Dynamic: requires-dist
|
|
20
|
+
Dynamic: summary
|
|
21
|
+
|
|
22
|
+
PyPI smart-artificial-intelligence-local Python Package owned by Circlez.ai
|
|
23
|
+
GHA: https://github.com/circles-zone/smart-artificial-intelligence-local-python-package/actions
|
smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
smart_artificial_intelligence_local/__init__.py
|
|
5
|
+
smart_artificial_intelligence_local.egg-info/PKG-INFO
|
|
6
|
+
smart_artificial_intelligence_local.egg-info/SOURCES.txt
|
|
7
|
+
smart_artificial_intelligence_local.egg-info/dependency_links.txt
|
|
8
|
+
smart_artificial_intelligence_local.egg-info/requires.txt
|
|
9
|
+
smart_artificial_intelligence_local.egg-info/top_level.txt
|
|
10
|
+
smart_artificial_intelligence_local/src/__init__.py
|
|
11
|
+
smart_artificial_intelligence_local/src/constants_src_smart_artificial_intelligence_local.py
|
|
12
|
+
smart_artificial_intelligence_local/src/llm_mode_enum.py
|
|
13
|
+
smart_artificial_intelligence_local/src/smart_artificial_intelligence_local.py
|
|
14
|
+
smart_artificial_intelligence_local/src/smart_llm_logger.py
|
|
15
|
+
smart_artificial_intelligence_local/src/smart_llm_result.py
|
|
16
|
+
smart_artificial_intelligence_local/tests/__init__.py
|
|
17
|
+
smart_artificial_intelligence_local/tests/constants_tests_smart_artificial_intelligence_local.py
|
|
18
|
+
smart_artificial_intelligence_local/tests/smart_artificial_intelligence_local_test.py
|
|
19
|
+
smart_artificial_intelligence_local/tests/smart_llm_system_test.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
smart_artificial_intelligence_local-0.0.1/smart_artificial_intelligence_local.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
smart_artificial_intelligence_local
|