aiverify-moonshot 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aiverify_moonshot-0.5.0.dist-info → aiverify_moonshot-0.5.1.dist-info}/METADATA +8 -5
- {aiverify_moonshot-0.5.0.dist-info → aiverify_moonshot-0.5.1.dist-info}/RECORD +12 -12
- {aiverify_moonshot-0.5.0.dist-info → aiverify_moonshot-0.5.1.dist-info}/WHEEL +1 -1
- moonshot/integrations/cli/common/dataset.py +5 -6
- moonshot/integrations/web_api/app.py +1 -1
- moonshot/integrations/web_api/schemas/cookbook_response_model.py +1 -1
- moonshot/integrations/web_api/schemas/recipe_response_model.py +1 -1
- moonshot/integrations/web_api/services/cookbook_service.py +52 -21
- moonshot/integrations/web_api/services/recipe_service.py +36 -15
- {aiverify_moonshot-0.5.0.dist-info → aiverify_moonshot-0.5.1.dist-info}/licenses/AUTHORS.md +0 -0
- {aiverify_moonshot-0.5.0.dist-info → aiverify_moonshot-0.5.1.dist-info}/licenses/LICENSE.md +0 -0
- {aiverify_moonshot-0.5.0.dist-info → aiverify_moonshot-0.5.1.dist-info}/licenses/NOTICES.md +0 -0
|
@@ -1,12 +1,15 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: aiverify-moonshot
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: AI Verify advances Gen AI testing with Project Moonshot.
|
|
5
5
|
Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
|
|
6
6
|
Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
|
|
7
7
|
Project-URL: Issues, https://github.com/aiverify-foundation/moonshot/issues
|
|
8
8
|
Author-email: AI Verify Foundation <info@aiverify.sg>
|
|
9
9
|
License: Apache-2.0
|
|
10
|
+
License-File: AUTHORS.md
|
|
11
|
+
License-File: LICENSE.md
|
|
12
|
+
License-File: NOTICES.md
|
|
10
13
|
Classifier: Development Status :: 3 - Alpha
|
|
11
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
15
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -26,7 +29,7 @@ Requires-Dist: xxhash>=3.5.0
|
|
|
26
29
|
Provides-Extra: all
|
|
27
30
|
Requires-Dist: cmd2>=2.4.3; extra == 'all'
|
|
28
31
|
Requires-Dist: dependency-injector>=4.41.0; extra == 'all'
|
|
29
|
-
Requires-Dist: fastapi>=0.
|
|
32
|
+
Requires-Dist: fastapi>=0.115.4; extra == 'all'
|
|
30
33
|
Requires-Dist: rich>=13.8.0; extra == 'all'
|
|
31
34
|
Requires-Dist: typing-extensions>=4.12.2; extra == 'all'
|
|
32
35
|
Requires-Dist: uvicorn>=0.30.6; extra == 'all'
|
|
@@ -35,7 +38,7 @@ Requires-Dist: cmd2>=2.4.3; extra == 'cli'
|
|
|
35
38
|
Requires-Dist: rich>=13.8.0; extra == 'cli'
|
|
36
39
|
Provides-Extra: web-api
|
|
37
40
|
Requires-Dist: dependency-injector>=4.41.0; extra == 'web-api'
|
|
38
|
-
Requires-Dist: fastapi>=0.
|
|
41
|
+
Requires-Dist: fastapi>=0.115.4; extra == 'web-api'
|
|
39
42
|
Requires-Dist: typing-extensions>=4.12.2; extra == 'web-api'
|
|
40
43
|
Requires-Dist: uvicorn>=0.30.6; extra == 'web-api'
|
|
41
44
|
Description-Content-Type: text/markdown
|
|
@@ -44,7 +47,7 @@ Description-Content-Type: text/markdown
|
|
|
44
47
|
|
|
45
48
|

|
|
46
49
|
|
|
47
|
-
**Version 0.5.
|
|
50
|
+
**Version 0.5.1**
|
|
48
51
|
|
|
49
52
|
A simple and modular tool to evaluate any LLM application.
|
|
50
53
|
|
|
@@ -19,7 +19,7 @@ moonshot/integrations/cli/benchmark/runner.py,sha256=Y4Vt6Qqn9QzsM6eLUM9m2_XKkW3
|
|
|
19
19
|
moonshot/integrations/cli/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
moonshot/integrations/cli/common/common.py,sha256=RjlIqV82DHVi5CeUjOkC1xYlVAq4mxelDD9MN4Tj4jU,3302
|
|
21
21
|
moonshot/integrations/cli/common/connectors.py,sha256=bmdZfDHKhii3ejZb5tkT0WnoSYtAGKPk8CpBgIvI5PY,20017
|
|
22
|
-
moonshot/integrations/cli/common/dataset.py,sha256=
|
|
22
|
+
moonshot/integrations/cli/common/dataset.py,sha256=_H7SgP80Y9WTBwHdSo5GlyqWL6w9nuBNti2ELgzxL3c,12646
|
|
23
23
|
moonshot/integrations/cli/common/display_helper.py,sha256=8rVowW33XK0j0C_X_H1jUbFlFk1Y2WpzxmIUE3Ca5Co,1459
|
|
24
24
|
moonshot/integrations/cli/common/prompt_template.py,sha256=2cXWeHh16EUvzYFre7juMxT2R0x3kqDJVucH0bKLmCU,6351
|
|
25
25
|
moonshot/integrations/cli/initialisation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -34,7 +34,7 @@ moonshot/integrations/cli/utils/process_data.py,sha256=QVL5vp2_8ZgGicmCAdeYEHkeb
|
|
|
34
34
|
moonshot/integrations/web_api/.env.dev,sha256=0z5_Ut8rF-UqFZtgjkH2qoqORhD5_nSs2w_OeX2SteI,182
|
|
35
35
|
moonshot/integrations/web_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
36
36
|
moonshot/integrations/web_api/__main__.py,sha256=MdnLi_ZF-olAAEJwTPU1iGYFYwo-fNWNT2qfchkH3y4,2050
|
|
37
|
-
moonshot/integrations/web_api/app.py,sha256=
|
|
37
|
+
moonshot/integrations/web_api/app.py,sha256=14_CWTSuLNyX3zH_vaqFUa4fugOUcOxLqx-IEv0-v34,3651
|
|
38
38
|
moonshot/integrations/web_api/container.py,sha256=DVkJG_qm7ItcG6tgMYOqIj07wpKhPWOOfy6-bEv72y4,5915
|
|
39
39
|
moonshot/integrations/web_api/logging_conf.py,sha256=t3EGRV6tZhV732KXe8_Tiy0fiwVAWxZX5Tt8VTgrrfg,3388
|
|
40
40
|
moonshot/integrations/web_api/log/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -56,7 +56,7 @@ moonshot/integrations/web_api/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
|
|
|
56
56
|
moonshot/integrations/web_api/schemas/benchmark_runner_dto.py,sha256=nfNMt_9Xg0YAL5f93dZamu7DxSLvAsz8-tdA_DTCXQQ,322
|
|
57
57
|
moonshot/integrations/web_api/schemas/bookmark_create_dto.py,sha256=C78vG8UG02N7Cmt6RSuS8e4sX_G-MLCiAWT-cF5BE8s,374
|
|
58
58
|
moonshot/integrations/web_api/schemas/cookbook_create_dto.py,sha256=00SPVw7lEpfY9yOFdt1XkvvNAzfFRd7d7CA90qguhuQ,670
|
|
59
|
-
moonshot/integrations/web_api/schemas/cookbook_response_model.py,sha256=
|
|
59
|
+
moonshot/integrations/web_api/schemas/cookbook_response_model.py,sha256=COLvaE4Hrz_w-C_HQkB7feztweIr0wkY9h8N6NKNIr8,332
|
|
60
60
|
moonshot/integrations/web_api/schemas/dataset_create_dto.py,sha256=Jr_EbIgqR7K__LQXpXSocfYLE6oN7pGRFzWcDVcFaus,963
|
|
61
61
|
moonshot/integrations/web_api/schemas/dataset_response_dto.py,sha256=s5x4-UXEWccWhK42E0FPXiHG6VqjuFuph-2t5atEkg4,171
|
|
62
62
|
moonshot/integrations/web_api/schemas/endpoint_create_dto.py,sha256=WS8AfRybrweoOgZx6K6jiNy1Z6J3IZS1PUNnrRxGKyM,678
|
|
@@ -64,7 +64,7 @@ moonshot/integrations/web_api/schemas/endpoint_response_model.py,sha256=OmmM2uaP
|
|
|
64
64
|
moonshot/integrations/web_api/schemas/prompt_response_model.py,sha256=S9PwxJERY1ppDaUKLlL9_skHcYcURIIvFnRZj24hLnE,303
|
|
65
65
|
moonshot/integrations/web_api/schemas/prompt_template_response_model.py,sha256=V7znK-QjQVUXUbsmEy5hZHzjnHYCN1kDtvOxgyxF83k,195
|
|
66
66
|
moonshot/integrations/web_api/schemas/recipe_create_dto.py,sha256=xXT1TlVM3jVGUH4FPrd64vRctoCV8ybnXga4Nb1VzEU,1117
|
|
67
|
-
moonshot/integrations/web_api/schemas/recipe_response_model.py,sha256=
|
|
67
|
+
moonshot/integrations/web_api/schemas/recipe_response_model.py,sha256=2__672E0tgicx6f_i34IxYqc9GnDkUY40bLsAkMWQMw,257
|
|
68
68
|
moonshot/integrations/web_api/schemas/session_create_dto.py,sha256=3Xhj5JOEEWwaoq1q4FZLBVUdpo59C6X6m4y_qLurEhg,493
|
|
69
69
|
moonshot/integrations/web_api/schemas/session_prompt_dto.py,sha256=AcMq3UhKrZJIEYDrEfK8xPHeTNk2Ex4IDXrDWZwn6Fo,218
|
|
70
70
|
moonshot/integrations/web_api/schemas/session_response_model.py,sha256=JCFJQ5k5IYPuYYUTRY-LPlVuV5xMclOtOghgUvWGkEg,1021
|
|
@@ -79,12 +79,12 @@ moonshot/integrations/web_api/services/benchmark_test_state.py,sha256=MyhTxpAhhP
|
|
|
79
79
|
moonshot/integrations/web_api/services/benchmarking_service.py,sha256=lJZeNTqxEPBLrZNX3Z9JIilgwetywSkv0deQkcb8mQs,1257
|
|
80
80
|
moonshot/integrations/web_api/services/bookmark_service.py,sha256=jI9nXs1hjzO0CLG2LKaXSzDApLThkfCvPUkaNNV9A5A,3546
|
|
81
81
|
moonshot/integrations/web_api/services/context_strategy_service.py,sha256=6YKnnG8JlE_1nlnr4Hq7rgz-sxI6oQglK0STaWPFQxQ,710
|
|
82
|
-
moonshot/integrations/web_api/services/cookbook_service.py,sha256=
|
|
82
|
+
moonshot/integrations/web_api/services/cookbook_service.py,sha256=AU2J_YCLIrRlufPgboZ27f89rVVR1zJHXXHwusf_ipc,8877
|
|
83
83
|
moonshot/integrations/web_api/services/dataset_service.py,sha256=ZWb3FqyDkA0C9qhlQ3X_zR0ohAlwlLsJi-mgKLvXpnI,2407
|
|
84
84
|
moonshot/integrations/web_api/services/endpoint_service.py,sha256=N5SXNAh44UNeBpMhA9baL0VZoTx4sHzpy4y7-Ch8O4E,2395
|
|
85
85
|
moonshot/integrations/web_api/services/metric_service.py,sha256=xWC5Dk8aiU7tuHsxYedTTrEkbA3Ug1pV2nbaBas6cAg,456
|
|
86
86
|
moonshot/integrations/web_api/services/prompt_template_service.py,sha256=5ds7pKDB2R0_0slVDwsCRIpIVdsgpqhI-3wQqSYcpuE,1226
|
|
87
|
-
moonshot/integrations/web_api/services/recipe_service.py,sha256=
|
|
87
|
+
moonshot/integrations/web_api/services/recipe_service.py,sha256=Pa3mE7RPY950YpkrepeXlOD703CtZRy-N63jewndwP0,7539
|
|
88
88
|
moonshot/integrations/web_api/services/runner_service.py,sha256=_ljFTVtMzt3fxlY0l2252KTxgVsqoQB6vsOXBIY0PIU,4584
|
|
89
89
|
moonshot/integrations/web_api/services/session_service.py,sha256=g0iMGkSAbc2Vx3c9m4aqe15LknsiHgzs7ErV1hHq5nA,13821
|
|
90
90
|
moonshot/integrations/web_api/services/utils/exceptions_handler.py,sha256=anaQQYLAXzbt-cL4nAZGm1TIlP9fKt7qYE7lrNtW_04,2070
|
|
@@ -172,9 +172,9 @@ moonshot/src/utils/import_modules.py,sha256=T9zTN59PFnvY2rjyWhSV9KSIAHxWV1pyBemF
|
|
|
172
172
|
moonshot/src/utils/log.py,sha256=YNgD7Eh2OT36XlmVBKCGUTAh9TRp4Akfe4kDdvHASgs,2502
|
|
173
173
|
moonshot/src/utils/pagination.py,sha256=5seymyRoqyENIhKllAatr1T91kMCGFslcvRnJHyMSvc,814
|
|
174
174
|
moonshot/src/utils/timeit.py,sha256=TvuF0w8KWhp0oZFY0cUU3UY0xlGKjchb0OkfYfgVTlc,866
|
|
175
|
-
aiverify_moonshot-0.5.
|
|
176
|
-
aiverify_moonshot-0.5.
|
|
177
|
-
aiverify_moonshot-0.5.
|
|
178
|
-
aiverify_moonshot-0.5.
|
|
179
|
-
aiverify_moonshot-0.5.
|
|
180
|
-
aiverify_moonshot-0.5.
|
|
175
|
+
aiverify_moonshot-0.5.1.dist-info/METADATA,sha256=EtBhJ0bmVoTktwS_p6OUI9JMgY6-gjmhUWfGK3Em7Vk,12419
|
|
176
|
+
aiverify_moonshot-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
177
|
+
aiverify_moonshot-0.5.1.dist-info/licenses/AUTHORS.md,sha256=mmAbe3i3sT8JZHJMBhxp3i1xRehV0g7WB4T_eyIBuBs,59
|
|
178
|
+
aiverify_moonshot-0.5.1.dist-info/licenses/LICENSE.md,sha256=53izDRmJZZCjpYGfyLqlxnGQN-aNWBxasuzuMXC5Ias,11347
|
|
179
|
+
aiverify_moonshot-0.5.1.dist-info/licenses/NOTICES.md,sha256=vS1zZYAnGjCJdwQ13xv3b2zc30wOS98ZnCKluT-AhHs,123266
|
|
180
|
+
aiverify_moonshot-0.5.1.dist-info/RECORD,,
|
|
@@ -5,13 +5,12 @@ from rich.console import Console
|
|
|
5
5
|
from rich.table import Table
|
|
6
6
|
|
|
7
7
|
from moonshot.api import (
|
|
8
|
+
api_convert_dataset,
|
|
8
9
|
api_delete_dataset,
|
|
10
|
+
api_download_dataset,
|
|
9
11
|
api_get_all_datasets,
|
|
10
12
|
api_get_all_datasets_name,
|
|
11
|
-
api_convert_dataset,
|
|
12
|
-
api_download_dataset
|
|
13
13
|
)
|
|
14
|
-
|
|
15
14
|
from moonshot.integrations.cli.cli_errors import (
|
|
16
15
|
ERROR_BENCHMARK_DELETE_DATASET_DATASET_VALIDATION,
|
|
17
16
|
ERROR_BENCHMARK_LIST_DATASETS_FIND_VALIDATION,
|
|
@@ -19,12 +18,12 @@ from moonshot.integrations.cli.cli_errors import (
|
|
|
19
18
|
ERROR_BENCHMARK_LIST_DATASETS_PAGINATION_VALIDATION_1,
|
|
20
19
|
ERROR_BENCHMARK_VIEW_DATASET_DATASET_FILENAME_VALIDATION,
|
|
21
20
|
)
|
|
22
|
-
|
|
23
21
|
from moonshot.integrations.cli.common.display_helper import display_view_str_format
|
|
24
22
|
from moonshot.integrations.cli.utils.process_data import filter_data
|
|
25
23
|
|
|
26
24
|
console = Console()
|
|
27
25
|
|
|
26
|
+
|
|
28
27
|
def list_datasets(args) -> list | None:
|
|
29
28
|
"""
|
|
30
29
|
List all available datasets.
|
|
@@ -155,6 +154,7 @@ def delete_dataset(args) -> None:
|
|
|
155
154
|
except Exception as e:
|
|
156
155
|
print(f"[delete_dataset]: {str(e)}")
|
|
157
156
|
|
|
157
|
+
|
|
158
158
|
def convert_dataset(args) -> None:
|
|
159
159
|
"""
|
|
160
160
|
Convert an existing dataset to a new format.
|
|
@@ -350,8 +350,7 @@ download_dataset_args.add_argument(
|
|
|
350
350
|
type=literal_eval,
|
|
351
351
|
help=(
|
|
352
352
|
"Params of the new dataset in dictionary format. For example: \n"
|
|
353
|
-
"
|
|
354
|
-
"2. For 'hf' method: \"{'dataset_name': 'cais_mmlu', 'dataset_config': 'college_biology', 'split': 'test', "
|
|
353
|
+
"{'dataset_name': 'cais_mmlu', 'dataset_config': 'college_biology', 'split': 'test', "
|
|
355
354
|
"'input_col': ['questions','choices'], 'target_col': 'answer'}\""
|
|
356
355
|
),
|
|
357
356
|
)
|
|
@@ -71,7 +71,7 @@ def create_app(cfg: providers.Configuration) -> CustomFastAPI:
|
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
app: CustomFastAPI = CustomFastAPI(
|
|
74
|
-
title="Project Moonshot", version="0.5.
|
|
74
|
+
title="Project Moonshot", version="0.5.1", **app_kwargs
|
|
75
75
|
)
|
|
76
76
|
|
|
77
77
|
if cfg.cors.enabled():
|
|
@@ -8,4 +8,4 @@ from moonshot.src.cookbooks.cookbook_arguments import (
|
|
|
8
8
|
class CookbookResponseModel(CookbookPydanticModel):
|
|
9
9
|
total_prompt_in_cookbook: Optional[int] = None
|
|
10
10
|
total_dataset_in_cookbook: Optional[int] = None
|
|
11
|
-
|
|
11
|
+
required_config: dict | None = None
|
|
@@ -5,7 +5,10 @@ from .... import api as moonshot_api
|
|
|
5
5
|
from ..schemas.cookbook_create_dto import CookbookCreateDTO, CookbookUpdateDTO
|
|
6
6
|
from ..schemas.cookbook_response_model import CookbookResponseModel
|
|
7
7
|
from ..services.base_service import BaseService
|
|
8
|
-
from ..services.recipe_service import
|
|
8
|
+
from ..services.recipe_service import (
|
|
9
|
+
get_metric_dependency_in_recipe,
|
|
10
|
+
get_total_prompt_in_recipe,
|
|
11
|
+
)
|
|
9
12
|
from ..services.utils.exceptions_handler import exception_handler
|
|
10
13
|
|
|
11
14
|
|
|
@@ -63,25 +66,28 @@ class CookbookService(BaseService):
|
|
|
63
66
|
if cookbook not in cookbooks_list:
|
|
64
67
|
cookbooks_list.append(cookbook)
|
|
65
68
|
if count:
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
(
|
|
70
|
+
cookbook.total_prompt_in_cookbook,
|
|
71
|
+
cookbook.total_dataset_in_cookbook,
|
|
72
|
+
) = get_total_prompt_and_dataset_in_cookbook(cookbook)
|
|
69
73
|
|
|
70
74
|
if tags and cookbooks_recipe_has_tags(tags, cookbook):
|
|
71
75
|
if cookbook not in cookbooks_list:
|
|
72
76
|
cookbooks_list.append(cookbook)
|
|
73
77
|
if count:
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
78
|
+
(
|
|
79
|
+
cookbook.total_prompt_in_cookbook,
|
|
80
|
+
cookbook.total_dataset_in_cookbook,
|
|
81
|
+
) = get_total_prompt_and_dataset_in_cookbook(cookbook)
|
|
77
82
|
|
|
78
83
|
if categories and cookbooks_recipe_has_categories(categories, cookbook):
|
|
79
84
|
if cookbook not in cookbooks_list:
|
|
80
85
|
cookbooks_list.append(cookbook)
|
|
81
86
|
if count:
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
87
|
+
(
|
|
88
|
+
cookbook.total_prompt_in_cookbook,
|
|
89
|
+
cookbook.total_dataset_in_cookbook,
|
|
90
|
+
) = get_total_prompt_and_dataset_in_cookbook(cookbook)
|
|
85
91
|
|
|
86
92
|
if categories_excluded and cookbooks_recipe_has_categories(
|
|
87
93
|
categories_excluded, cookbook
|
|
@@ -89,7 +95,7 @@ class CookbookService(BaseService):
|
|
|
89
95
|
cookbooks_list.remove(cookbook)
|
|
90
96
|
|
|
91
97
|
for cookbook in cookbooks_list:
|
|
92
|
-
cookbook.
|
|
98
|
+
cookbook.required_config = cookbook_metrics_dependency(cookbook)
|
|
93
99
|
|
|
94
100
|
return cookbooks_list
|
|
95
101
|
|
|
@@ -147,7 +153,9 @@ def get_total_prompt_and_dataset_in_cookbook(cookbook: Cookbook) -> tuple[int, i
|
|
|
147
153
|
int: The total count of prompts within the cookbook.
|
|
148
154
|
"""
|
|
149
155
|
recipes = moonshot_api.api_read_recipes(cookbook.recipes)
|
|
150
|
-
total_prompts, total_datasets = zip(
|
|
156
|
+
total_prompts, total_datasets = zip(
|
|
157
|
+
*(get_total_prompt_in_recipe(Recipe(**recipe)) for recipe in recipes)
|
|
158
|
+
)
|
|
151
159
|
return sum(total_prompts), sum(total_datasets)
|
|
152
160
|
|
|
153
161
|
|
|
@@ -197,25 +205,48 @@ def cookbooks_recipe_has_categories(categories: str, cookbook: Cookbook) -> bool
|
|
|
197
205
|
return True
|
|
198
206
|
return False
|
|
199
207
|
|
|
208
|
+
|
|
200
209
|
@staticmethod
|
|
201
|
-
def
|
|
210
|
+
def cookbook_metrics_dependency(cookbook: Cookbook) -> dict | None:
|
|
202
211
|
"""
|
|
203
|
-
Retrieve a list of endpoint dependencies for all recipes in a given cookbook.
|
|
212
|
+
Retrieve a list of endpoint and configuration dependencies for all recipes in a given cookbook.
|
|
204
213
|
|
|
205
214
|
Args:
|
|
206
215
|
cookbook (Cookbook): The cookbook object containing the recipe IDs.
|
|
207
216
|
|
|
208
217
|
Returns:
|
|
209
|
-
list[str] | None: A
|
|
218
|
+
dict[str, list[str]] | None: A dictionary with aggregated endpoint and configuration dependencies
|
|
219
|
+
if any are found, otherwise None.
|
|
210
220
|
"""
|
|
211
221
|
recipes_in_cookbook = cookbook.recipes
|
|
212
222
|
recipes = moonshot_api.api_read_recipes(recipes_in_cookbook)
|
|
213
|
-
|
|
223
|
+
aggregated_endpoints = set()
|
|
224
|
+
aggregated_configurations = {}
|
|
214
225
|
|
|
215
226
|
for recipe in recipes:
|
|
216
227
|
recipe = Recipe(**recipe)
|
|
217
|
-
|
|
218
|
-
if
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
228
|
+
dependency = get_metric_dependency_in_recipe(recipe)
|
|
229
|
+
if dependency:
|
|
230
|
+
# Aggregate endpoints
|
|
231
|
+
aggregated_endpoints.update(dependency.get("endpoints", []))
|
|
232
|
+
|
|
233
|
+
# Aggregate configurations
|
|
234
|
+
for key, value in dependency.get("configurations", {}).items():
|
|
235
|
+
if key in aggregated_configurations:
|
|
236
|
+
aggregated_configurations[key].extend(
|
|
237
|
+
v for v in value if v not in aggregated_configurations[key]
|
|
238
|
+
)
|
|
239
|
+
else:
|
|
240
|
+
aggregated_configurations[key] = value
|
|
241
|
+
|
|
242
|
+
aggregated_dependencies = {
|
|
243
|
+
"endpoints": list(aggregated_endpoints),
|
|
244
|
+
"configurations": aggregated_configurations,
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return (
|
|
248
|
+
aggregated_dependencies
|
|
249
|
+
if aggregated_dependencies["endpoints"]
|
|
250
|
+
or aggregated_dependencies["configurations"]
|
|
251
|
+
else None
|
|
252
|
+
)
|
|
@@ -85,7 +85,7 @@ class RecipeService(BaseService):
|
|
|
85
85
|
filtered_recipes.sort(key=lambda x: x.id)
|
|
86
86
|
|
|
87
87
|
for recipe in filtered_recipes:
|
|
88
|
-
recipe.
|
|
88
|
+
recipe.required_config = get_metric_dependency_in_recipe(recipe)
|
|
89
89
|
|
|
90
90
|
return filtered_recipes
|
|
91
91
|
|
|
@@ -157,28 +157,49 @@ def get_total_prompt_in_recipe(recipe: Recipe) -> tuple[int, int]:
|
|
|
157
157
|
|
|
158
158
|
return total_prompt_count, int(recipe.stats.get("num_of_datasets", 0))
|
|
159
159
|
|
|
160
|
+
|
|
160
161
|
@staticmethod
|
|
161
|
-
def
|
|
162
|
+
def get_metric_dependency_in_recipe(recipe: Recipe) -> dict | None:
|
|
162
163
|
"""
|
|
163
|
-
Retrieve
|
|
164
|
+
Retrieve endpoint and configuration dependencies for a given recipe.
|
|
164
165
|
|
|
165
|
-
This function
|
|
166
|
-
|
|
167
|
-
of endpoint dependencies.
|
|
166
|
+
This function gathers all available metrics along with their endpoints and configurations,
|
|
167
|
+
then identifies and compiles the dependencies based on the metrics present in the provided recipe.
|
|
168
168
|
|
|
169
169
|
Args:
|
|
170
|
-
recipe (Recipe): The recipe object containing
|
|
170
|
+
recipe (Recipe): The recipe object containing metrics information.
|
|
171
171
|
|
|
172
172
|
Returns:
|
|
173
|
-
list[str] | None: A
|
|
173
|
+
dict[str, dict[str, list[str]]] | None: A dictionary with 'endpoints' and 'configurations' as keys,
|
|
174
|
+
where 'endpoints' is a list of endpoint dependencies and 'configurations' is a dictionary of configuration
|
|
175
|
+
dependencies. Returns None if no dependencies are found.
|
|
174
176
|
"""
|
|
175
177
|
metrics = recipe.metrics
|
|
176
178
|
all_metrics = moonshot_api.api_get_all_metric()
|
|
177
|
-
|
|
178
|
-
|
|
179
|
+
aggregated_endpoints = set()
|
|
180
|
+
aggregated_configurations = {}
|
|
181
|
+
|
|
179
182
|
for metric in metrics:
|
|
180
|
-
for m in all_metrics
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
183
|
+
metric_data = next((m for m in all_metrics if m["id"] == metric), None)
|
|
184
|
+
if metric_data:
|
|
185
|
+
# Aggregate endpoints
|
|
186
|
+
aggregated_endpoints.update(metric_data.get("endpoints", []))
|
|
187
|
+
|
|
188
|
+
# Aggregate configurations
|
|
189
|
+
for key, value in metric_data.get("configurations", {}).items():
|
|
190
|
+
if key in aggregated_configurations:
|
|
191
|
+
aggregated_configurations[key].extend(
|
|
192
|
+
v for v in value if v not in aggregated_configurations[key]
|
|
193
|
+
)
|
|
194
|
+
else:
|
|
195
|
+
aggregated_configurations[key] = value
|
|
196
|
+
|
|
197
|
+
aggregated_data = {
|
|
198
|
+
"endpoints": list(aggregated_endpoints),
|
|
199
|
+
"configurations": aggregated_configurations,
|
|
200
|
+
}
|
|
201
|
+
return (
|
|
202
|
+
aggregated_data
|
|
203
|
+
if aggregated_data["endpoints"] or aggregated_data["configurations"]
|
|
204
|
+
else None
|
|
205
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|