vis-agent 0.2.42.dev7627__tar.gz → 0.2.43__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/PKG-INFO +4 -5
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/README.md +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/config.json +10 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/gateway.json +45 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/pyproject.toml +10 -5
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/__init__.py +6 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/agent.py +2 -3
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/bench.py +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/check.py +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/environment.py +7 -7
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/grade.py +0 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/harbor.py +203 -30
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/judge.py +1 -2
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/extension.py +19 -6
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/conftest.py +0 -10
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_engine.py +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_environment.py +21 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_guards.py +2 -5
- vis_agent-0.2.43/tests/test_evals_tasks.py +387 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_local.py +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_method_typing.py +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_providers.py +22 -2
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_sdk_typing.py +1 -1
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/.gitignore +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/LICENSE +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-arguments.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-cases.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-copy.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-execution.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-groups.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-read-session.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-reads.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-results.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity-tables.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/activity.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/diff.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/live-primitives.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/session-health.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/fixtures/view.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/activity.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/agents.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/automations.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/common.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/content.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/council.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/diff.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/evals.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/improve.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/provider.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/rooms.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/symbol.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/toggle.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/contracts/schema/view.json +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/examples/greeter/tests/test_greeter.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/hatch_build.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/__init__.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/_contracts.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/_outside.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/_wire.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/activity.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/__init__.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/_agent.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/_agents.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/_client.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/_council.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/_extensions.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/engine/_local.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/_gateway.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/compare.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/judge-rubric.toml +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/labels.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/leaderboard.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/repeats.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/rounds.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/stages.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/stats.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/evals/trajectory.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/extension_package.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/py.typed +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/rooms.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/src/blockether/vis/views.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/evals_atif.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_activity.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_agent.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_agent_extension_lifecycle.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_agent_extensions.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_agents.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_api_guides.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_attachments.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_authoring.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_catalog.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_catalog_recipe.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_ci_report_recipe.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_client.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_client_extensions_engine.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_code_quality_recipe.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_contracts.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_council.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_declarations.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_engine_docstrings.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_agent.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_bench.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_check.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_compare.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_gateway.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_grade.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_harbor.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_judge.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_labels.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_repeats.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_rounds.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_stages.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_stats.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_evals_trajectory.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_extension_package.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_extension_reference.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_gateway_performance.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_goals.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_live_primitives.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_live_wait.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_monitor_recipe.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_namespace.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_outside.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_read_session.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_record_arguments.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_registration.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_repair_hooks.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_result_sequence.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_rooms.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_scoped_settings.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_sdk_bootstrap.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_sdk_guide.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_structured_live.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_symbol_contract.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_testing.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_text_list_arguments.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_tools.py +0 -0
- {vis_agent-0.2.42.dev7627 → vis_agent-0.2.43}/tests/test_views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: vis-agent
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.43
|
|
4
4
|
Summary: The Vis Python SDK — author extensions and control local or remote engines.
|
|
5
5
|
Project-URL: Homepage, https://vis.blockether.com
|
|
6
6
|
Project-URL: Documentation, https://vis.blockether.com
|
|
@@ -15,15 +15,14 @@ Classifier: Intended Audience :: Developers
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
16
|
Classifier: Topic :: Software Development :: Build Tools
|
|
17
17
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Requires-Dist: harbor==0.24.0
|
|
19
20
|
Requires-Dist: jsonschema<5,>=4.23
|
|
20
21
|
Requires-Dist: packaging>=24
|
|
21
22
|
Requires-Dist: pydantic<3,>=2.13.5
|
|
22
23
|
Requires-Dist: referencing>=0.28.4
|
|
23
24
|
Provides-Extra: docs
|
|
24
25
|
Requires-Dist: pdoc==16.0.0; extra == 'docs'
|
|
25
|
-
Provides-Extra: harbor
|
|
26
|
-
Requires-Dist: harbor==0.24.0; (python_version >= '3.12') and extra == 'harbor'
|
|
27
26
|
Description-Content-Type: text/markdown
|
|
28
27
|
|
|
29
28
|
# vis-agent
|
|
@@ -34,7 +33,7 @@ Python SDK for Vis extensions and agent sessions.
|
|
|
34
33
|
pip install vis-agent
|
|
35
34
|
```
|
|
36
35
|
|
|
37
|
-
Requires Python 3.
|
|
36
|
+
Requires Python 3.12+. Install the Vis executable and configure a provider
|
|
38
37
|
separately for local use. The execution-layer API below is unreleased: build
|
|
39
38
|
the SDK and engine from the same source revision.
|
|
40
39
|
|
|
@@ -6,7 +6,7 @@ Python SDK for Vis extensions and agent sessions.
|
|
|
6
6
|
pip install vis-agent
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
-
Requires Python 3.
|
|
9
|
+
Requires Python 3.12+. Install the Vis executable and configure a provider
|
|
10
10
|
separately for local use. The execution-layer API below is unreleased: build
|
|
11
11
|
the SDK and engine from the same source revision.
|
|
12
12
|
|
|
@@ -158,10 +158,16 @@
|
|
|
158
158
|
"type": "integer",
|
|
159
159
|
"minimum": 1
|
|
160
160
|
},
|
|
161
|
+
"is_tool_calls_supported": {
|
|
162
|
+
"type": "boolean"
|
|
163
|
+
},
|
|
161
164
|
"is_tool_call": {
|
|
162
165
|
"type": "boolean"
|
|
163
166
|
},
|
|
164
|
-
"
|
|
167
|
+
"is_vision_supported": {
|
|
168
|
+
"type": "boolean"
|
|
169
|
+
},
|
|
170
|
+
"is_image_input": {
|
|
165
171
|
"type": "boolean"
|
|
166
172
|
},
|
|
167
173
|
"is_reasoning": {
|
|
@@ -255,6 +261,9 @@
|
|
|
255
261
|
"is_stateless": {
|
|
256
262
|
"type": "boolean"
|
|
257
263
|
},
|
|
264
|
+
"is_vision_supported": {
|
|
265
|
+
"type": "boolean"
|
|
266
|
+
},
|
|
258
267
|
"is_image_input": {
|
|
259
268
|
"type": "boolean"
|
|
260
269
|
},
|
|
@@ -2718,6 +2718,24 @@
|
|
|
2718
2718
|
"is_override": {
|
|
2719
2719
|
"type": "boolean"
|
|
2720
2720
|
},
|
|
2721
|
+
"decided_in": {
|
|
2722
|
+
"type": "object",
|
|
2723
|
+
"description": "The closest wider scope that decides this inherited row. The row is read-only here.",
|
|
2724
|
+
"required": [
|
|
2725
|
+
"scope"
|
|
2726
|
+
],
|
|
2727
|
+
"properties": {
|
|
2728
|
+
"scope": {
|
|
2729
|
+
"$ref": "https://contract.blockether.com/vis/schema/toggle.json#/$defs/scope"
|
|
2730
|
+
},
|
|
2731
|
+
"target_id": {
|
|
2732
|
+
"type": "string"
|
|
2733
|
+
},
|
|
2734
|
+
"label": {
|
|
2735
|
+
"type": "string"
|
|
2736
|
+
}
|
|
2737
|
+
}
|
|
2738
|
+
},
|
|
2721
2739
|
"overridden_by": {
|
|
2722
2740
|
"type": "object",
|
|
2723
2741
|
"required": [
|
|
@@ -2852,6 +2870,33 @@
|
|
|
2852
2870
|
"type": "object"
|
|
2853
2871
|
}
|
|
2854
2872
|
},
|
|
2873
|
+
"project_extensions": {
|
|
2874
|
+
"type": "object",
|
|
2875
|
+
"description": "Project extensions of the context session. Only Machine settings show them.",
|
|
2876
|
+
"required": [
|
|
2877
|
+
"target_id",
|
|
2878
|
+
"count",
|
|
2879
|
+
"names"
|
|
2880
|
+
],
|
|
2881
|
+
"properties": {
|
|
2882
|
+
"target_id": {
|
|
2883
|
+
"type": "string"
|
|
2884
|
+
},
|
|
2885
|
+
"label": {
|
|
2886
|
+
"type": "string"
|
|
2887
|
+
},
|
|
2888
|
+
"count": {
|
|
2889
|
+
"type": "integer",
|
|
2890
|
+
"minimum": 1
|
|
2891
|
+
},
|
|
2892
|
+
"names": {
|
|
2893
|
+
"type": "array",
|
|
2894
|
+
"items": {
|
|
2895
|
+
"type": "string"
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
}
|
|
2899
|
+
},
|
|
2855
2900
|
"groups": {
|
|
2856
2901
|
"type": "array",
|
|
2857
2902
|
"items": {
|
|
@@ -8,13 +8,20 @@ build-backend = "hatchling.build"
|
|
|
8
8
|
|
|
9
9
|
[project]
|
|
10
10
|
name = "vis-agent"
|
|
11
|
-
version = "0.2.
|
|
11
|
+
version = "0.2.43"
|
|
12
12
|
description = "The Vis Python SDK — author extensions and control local or remote engines."
|
|
13
13
|
readme = "README.md"
|
|
14
|
-
requires-python = ">=3.
|
|
14
|
+
requires-python = ">=3.12"
|
|
15
15
|
license = "Apache-2.0"
|
|
16
16
|
license-files = ["LICENSE"]
|
|
17
|
-
|
|
17
|
+
# Harbor runs the tasks of `blockether.vis.evals`: the agent, host environment, check and judge.
|
|
18
|
+
dependencies = [
|
|
19
|
+
"harbor==0.24.0",
|
|
20
|
+
"jsonschema>=4.23,<5",
|
|
21
|
+
"packaging>=24",
|
|
22
|
+
"pydantic>=2.13.5,<3",
|
|
23
|
+
"referencing>=0.28.4",
|
|
24
|
+
]
|
|
18
25
|
authors = [{ name = "Blockether" }]
|
|
19
26
|
keywords = ["vis", "agent", "extension", "automation", "human-in-the-loop"]
|
|
20
27
|
classifiers = [
|
|
@@ -27,8 +34,6 @@ classifiers = [
|
|
|
27
34
|
|
|
28
35
|
[project.optional-dependencies]
|
|
29
36
|
docs = ["pdoc==16.0.0"]
|
|
30
|
-
# The Harbor agent, host environment, check and judge of `blockether.vis.evals`.
|
|
31
|
-
harbor = ["harbor==0.24.0; python_version >= '3.12'"]
|
|
32
37
|
|
|
33
38
|
[project.urls]
|
|
34
39
|
Homepage = "https://vis.blockether.com"
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
"""Evaluation tools:
|
|
1
|
+
"""Evaluation tools: write and run Harbor tasks, read the results and compare them.
|
|
2
2
|
|
|
3
|
+
Write a Vis task with `write_task`, and run tasks with the Vis agent with `run_tasks`.
|
|
3
4
|
Read one Vis run with `load_trajectory` or `parse_trajectory`. Read the cases of a
|
|
4
5
|
Harbor dataset with `read_tasks`, and the trials of a job or round with `read_trials`.
|
|
5
6
|
Count the trials with `case_results` or `summarize`. Seal a finished round with
|
|
@@ -46,10 +47,12 @@ from .harbor import (
|
|
|
46
47
|
read_tasks,
|
|
47
48
|
read_trial,
|
|
48
49
|
read_trials,
|
|
50
|
+
run_tasks,
|
|
49
51
|
seal_digest,
|
|
50
52
|
seal_round,
|
|
51
53
|
seal_rows,
|
|
52
54
|
summarize,
|
|
55
|
+
write_task,
|
|
53
56
|
)
|
|
54
57
|
from .labels import (
|
|
55
58
|
KAPPA_REVIEW,
|
|
@@ -171,6 +174,7 @@ __all__ = [
|
|
|
171
174
|
"read_trials",
|
|
172
175
|
"repeat_stats",
|
|
173
176
|
"round_summary",
|
|
177
|
+
"run_tasks",
|
|
174
178
|
"sample_agreement",
|
|
175
179
|
"save_label",
|
|
176
180
|
"save_stage_label",
|
|
@@ -184,4 +188,5 @@ __all__ = [
|
|
|
184
188
|
"task_attempts",
|
|
185
189
|
"task_note",
|
|
186
190
|
"wilson_interval",
|
|
191
|
+
"write_task",
|
|
187
192
|
]
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"""Run the Vis CLI as a Harbor agent on the host.
|
|
2
2
|
|
|
3
3
|
Select it with the Harbor agent import path ``blockether.vis.evals.agent:VisAgent``.
|
|
4
|
-
It runs only in ``blockether.vis.evals.environment:HostEnvironment``.
|
|
5
|
-
|
|
6
|
-
database. With ``--persist`` the
|
|
4
|
+
It runs only in ``blockether.vis.evals.environment:HostEnvironment``. Each trial
|
|
5
|
+
starts one CLI process with its own database. With ``--persist`` the
|
|
7
6
|
CLI runs the gateway in the same process, so no shared gateway and no operator
|
|
8
7
|
extension changes the measurement. The agent writes these files to its logs
|
|
9
8
|
directory:
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Run it with ``python -m blockether.vis.evals.check TASK_DIR --provider P --model M``.
|
|
4
4
|
``TASK_DIR`` is one task or a directory of tasks, for example ``e2e`` of the Vis
|
|
5
|
-
repository.
|
|
5
|
+
repository. Each check makes paid model calls.
|
|
6
6
|
|
|
7
7
|
``harbor check`` runs Claude Code in Docker. This module uses the same rubric,
|
|
8
8
|
prompt and result validation of Harbor. The Vis agent runs each check in the host
|
|
@@ -29,7 +29,7 @@ from harbor.environments.base import BaseEnvironment, ExecResult
|
|
|
29
29
|
from harbor.environments.capabilities import EnvironmentCapabilities
|
|
30
30
|
from harbor.models.trial.paths import EnvironmentPaths
|
|
31
31
|
|
|
32
|
-
from .
|
|
32
|
+
from .harbor import WORKSPACE_FILESYSTEM_FILE
|
|
33
33
|
|
|
34
34
|
WORKSPACE_DIRECTORY = "workspace"
|
|
35
35
|
GIT_SETUP = (
|
|
@@ -81,10 +81,10 @@ class HostEnvironment(BaseEnvironment):
|
|
|
81
81
|
paths = EnvironmentPaths.for_os(self.os)
|
|
82
82
|
table = {
|
|
83
83
|
paths.logs_dir: self.root / "logs",
|
|
84
|
-
paths.agent_dir: self.trial_paths.agent_dir,
|
|
85
|
-
paths.user_agent_dir: self.trial_paths.user_agent_dir,
|
|
86
|
-
paths.verifier_dir: self.trial_paths.verifier_dir,
|
|
87
|
-
paths.artifacts_dir: self.trial_paths.artifacts_dir,
|
|
84
|
+
paths.agent_dir: self.trial_paths.agent_dir.resolve(),
|
|
85
|
+
paths.user_agent_dir: self.trial_paths.user_agent_dir.resolve(),
|
|
86
|
+
paths.verifier_dir: self.trial_paths.verifier_dir.resolve(),
|
|
87
|
+
paths.artifacts_dir: self.trial_paths.artifacts_dir.resolve(),
|
|
88
88
|
paths.tests_dir: self.root / "tests",
|
|
89
89
|
paths.solution_dir: self.root / "solution",
|
|
90
90
|
paths.default_skills_dir: self.root / "skills",
|
|
@@ -121,8 +121,8 @@ class HostEnvironment(BaseEnvironment):
|
|
|
121
121
|
"VIS_HARBOR_TASK": self.environment_dir.parent.name,
|
|
122
122
|
"VIS_HARBOR_WORKSPACE": str(self.workspace),
|
|
123
123
|
"VIS_HARBOR_TESTS": str(self.host_path(paths.tests_dir)),
|
|
124
|
-
"VIS_HARBOR_AGENT_LOGS": str(self.trial_paths.agent_dir),
|
|
125
|
-
"VIS_HARBOR_VERIFIER_LOGS": str(self.trial_paths.verifier_dir),
|
|
124
|
+
"VIS_HARBOR_AGENT_LOGS": str(self.trial_paths.agent_dir.resolve()),
|
|
125
|
+
"VIS_HARBOR_VERIFIER_LOGS": str(self.trial_paths.verifier_dir.resolve()),
|
|
126
126
|
"VIS_HARBOR_PYTHON": sys.executable,
|
|
127
127
|
}
|
|
128
128
|
|
|
@@ -1,21 +1,25 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Write and run Harbor tasks, and read Harbor jobs: trials, cases, a summary and seals.
|
|
2
2
|
|
|
3
|
-
A Harbor
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
A Harbor task is a folder with `task.toml`, `instruction.md`, `environment/` and
|
|
4
|
+
`tests/`. `write_task` writes a Vis task, and `run_tasks` runs tasks with the Vis
|
|
5
|
+
agent on this host. A Harbor job directory holds `config.json`, `result.json` and
|
|
6
|
+
one directory for each trial. A trial directory holds `result.json`,
|
|
7
|
+
`agent/trajectory.json` and `verifier/reward.json`. A round directory holds one or
|
|
8
|
+
more job directories.
|
|
6
9
|
|
|
7
|
-
|
|
10
|
+
The read functions read the JSON files directly. Only `run_tasks` imports Harbor.
|
|
8
11
|
"""
|
|
9
12
|
|
|
10
13
|
from __future__ import annotations
|
|
11
14
|
|
|
12
15
|
import hashlib
|
|
13
16
|
import json
|
|
17
|
+
import re
|
|
14
18
|
import time
|
|
15
19
|
import tomllib
|
|
16
20
|
from collections.abc import Iterable, Mapping, Sequence
|
|
17
21
|
from dataclasses import dataclass
|
|
18
|
-
from pathlib import Path
|
|
22
|
+
from pathlib import Path, PurePosixPath
|
|
19
23
|
from types import MappingProxyType
|
|
20
24
|
from typing import Any
|
|
21
25
|
|
|
@@ -29,15 +33,29 @@ __all__ = [
|
|
|
29
33
|
"read_tasks",
|
|
30
34
|
"read_trial",
|
|
31
35
|
"read_trials",
|
|
36
|
+
"run_tasks",
|
|
32
37
|
"seal_digest",
|
|
33
38
|
"seal_round",
|
|
34
39
|
"seal_rows",
|
|
35
40
|
"summarize",
|
|
41
|
+
"write_task",
|
|
36
42
|
]
|
|
37
43
|
|
|
38
44
|
RESULT_FILE = "result.json"
|
|
39
45
|
TASK_FILE = "task.toml"
|
|
40
46
|
INSTRUCTION_FILE = "instruction.md"
|
|
47
|
+
CHECKS_FILE = "checks.json"
|
|
48
|
+
TEST_SCRIPT_FILE = "test.sh"
|
|
49
|
+
WORKSPACE_FILESYSTEM_FILE = "workspace_filesystem.json"
|
|
50
|
+
WORKSPACE_DIRECTORY = "workspace"
|
|
51
|
+
TASK_NAME = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*")
|
|
52
|
+
AGENT = "blockether.vis.evals.agent:VisAgent"
|
|
53
|
+
ENVIRONMENT = "blockether.vis.evals.environment:HostEnvironment"
|
|
54
|
+
GRADER_SCRIPT = """#!/usr/bin/env bash
|
|
55
|
+
# Grade the trial. The Vis host environment sets the VIS_HARBOR_* paths.
|
|
56
|
+
set -euo pipefail
|
|
57
|
+
exec "$VIS_HARBOR_PYTHON" -m blockether.vis.evals.grade "$VIS_HARBOR_TESTS/checks.json"
|
|
58
|
+
"""
|
|
41
59
|
|
|
42
60
|
|
|
43
61
|
@dataclass(frozen=True, slots=True)
|
|
@@ -334,6 +352,25 @@ def _seconds(table: object) -> float | None:
|
|
|
334
352
|
return float(value)
|
|
335
353
|
|
|
336
354
|
|
|
355
|
+
def _read_task(config: Path) -> HarborTask:
|
|
356
|
+
info = tomllib.loads(config.read_text(encoding="utf-8"))
|
|
357
|
+
instruction = config.parent / INSTRUCTION_FILE
|
|
358
|
+
environment = info.get("environment")
|
|
359
|
+
gpus = environment.get("gpus") if isinstance(environment, dict) else None
|
|
360
|
+
metadata = info.get("metadata")
|
|
361
|
+
return HarborTask(
|
|
362
|
+
name=config.parent.name,
|
|
363
|
+
instruction=instruction.read_text(encoding="utf-8")
|
|
364
|
+
if instruction.is_file()
|
|
365
|
+
else "",
|
|
366
|
+
agent_timeout_sec=_seconds(info.get("agent")),
|
|
367
|
+
verifier_timeout_sec=_seconds(info.get("verifier")),
|
|
368
|
+
gpus=gpus if isinstance(gpus, int) and not isinstance(gpus, bool) else 0,
|
|
369
|
+
metadata=MappingProxyType(dict(metadata) if isinstance(metadata, dict) else {}),
|
|
370
|
+
path=config.parent,
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
|
|
337
374
|
def read_tasks(dataset: Path | str) -> tuple[HarborTask, ...]:
|
|
338
375
|
"""Read the tasks of a Harbor dataset directory, sorted by name.
|
|
339
376
|
|
|
@@ -344,28 +381,164 @@ def read_tasks(dataset: Path | str) -> tuple[HarborTask, ...]:
|
|
|
344
381
|
root = Path(dataset)
|
|
345
382
|
if not root.is_dir():
|
|
346
383
|
raise FileNotFoundError(f"no Harbor dataset directory at {root}")
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
384
|
+
return tuple(_read_task(config) for config in sorted(root.glob(f"*/{TASK_FILE}")))
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _toml_value(key: str, value: object) -> str:
|
|
388
|
+
if isinstance(value, bool):
|
|
389
|
+
return "true" if value else "false"
|
|
390
|
+
if isinstance(value, (int, float)):
|
|
391
|
+
return repr(value)
|
|
392
|
+
if isinstance(value, str):
|
|
393
|
+
return json.dumps(value)
|
|
394
|
+
raise ValueError(f"metadata `{key}` must be a string, a number or a boolean")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _seed_path(name: str) -> PurePosixPath:
|
|
398
|
+
path = PurePosixPath(name)
|
|
399
|
+
if not name or path.is_absolute() or ".." in path.parts or name != str(path):
|
|
400
|
+
raise ValueError(f"seed file path must stay inside the workspace: {name!r}")
|
|
401
|
+
return path
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def write_task(
|
|
405
|
+
dataset: Path | str,
|
|
406
|
+
name: str,
|
|
407
|
+
instruction: str,
|
|
408
|
+
checks: Mapping[str, Any],
|
|
409
|
+
*,
|
|
410
|
+
files: Mapping[str, str] | None = None,
|
|
411
|
+
agent_timeout_sec: float = 300.0,
|
|
412
|
+
verifier_timeout_sec: float = 300.0,
|
|
413
|
+
metadata: Mapping[str, str | int | float | bool] | None = None,
|
|
414
|
+
artifacts: Sequence[str] = (),
|
|
415
|
+
filesystem_roots: Mapping[str, str] | None = None,
|
|
416
|
+
) -> HarborTask:
|
|
417
|
+
"""Write the Vis task `name` in the folder `dataset` and return it.
|
|
418
|
+
|
|
419
|
+
`name` is lowercase words joined by hyphens. `instruction` is the prompt of the
|
|
420
|
+
agent. `checks` is the `tests/checks.json` object that the Vis grader reads, for
|
|
421
|
+
example `{"want": {"m.py": ["def c("]}}`. `files` maps each seed file path to its
|
|
422
|
+
text. The paths are relative to `environment/workspace/`. `artifacts` lists the
|
|
423
|
+
workspace files that Harbor copies to `artifacts/` in the trial folder after the
|
|
424
|
+
check, for example the call record of a mock extension. `filesystem_roots` maps each
|
|
425
|
+
filesystem root id to a workspace folder; the trial registers them in `vis.yml`. The
|
|
426
|
+
task gets `tests/test.sh`, which runs the grader. An existing task folder, an invalid
|
|
427
|
+
name, empty checks or a path outside the workspace raise an error.
|
|
428
|
+
"""
|
|
429
|
+
if not TASK_NAME.fullmatch(name):
|
|
430
|
+
raise ValueError(
|
|
431
|
+
f"task name must be lowercase words joined by hyphens: {name!r}"
|
|
370
432
|
)
|
|
371
|
-
|
|
433
|
+
if not isinstance(checks, Mapping) or not checks:
|
|
434
|
+
raise ValueError("checks must be a nonempty object")
|
|
435
|
+
if not instruction.strip():
|
|
436
|
+
raise ValueError("instruction must not be empty")
|
|
437
|
+
seeds = {_seed_path(path): text for path, text in (files or {}).items()}
|
|
438
|
+
kept = [str(_seed_path(path)) for path in artifacts]
|
|
439
|
+
roots = {
|
|
440
|
+
root: str(_seed_path(path)) for root, path in (filesystem_roots or {}).items()
|
|
441
|
+
}
|
|
442
|
+
lines = [
|
|
443
|
+
f"{key} = {_toml_value(key, value)}" for key, value in (metadata or {}).items()
|
|
444
|
+
]
|
|
445
|
+
# Harbor reads `artifacts` only as a root key, so it comes before every table.
|
|
446
|
+
root = f"artifacts = {json.dumps(kept)}\n\n" if kept else ""
|
|
447
|
+
tables = [
|
|
448
|
+
("metadata", lines),
|
|
449
|
+
("agent", [f"timeout_sec = {float(agent_timeout_sec)!r}"]),
|
|
450
|
+
("verifier", [f"timeout_sec = {float(verifier_timeout_sec)!r}"]),
|
|
451
|
+
]
|
|
452
|
+
task = Path(dataset) / name
|
|
453
|
+
task.mkdir(parents=True, exist_ok=False)
|
|
454
|
+
(task / TASK_FILE).write_text(
|
|
455
|
+
root
|
|
456
|
+
+ "\n".join(
|
|
457
|
+
f"[{table}]\n" + "".join(f"{line}\n" for line in rows)
|
|
458
|
+
for table, rows in tables
|
|
459
|
+
),
|
|
460
|
+
encoding="utf-8",
|
|
461
|
+
)
|
|
462
|
+
(task / INSTRUCTION_FILE).write_text(
|
|
463
|
+
instruction.rstrip("\n") + "\n", encoding="utf-8"
|
|
464
|
+
)
|
|
465
|
+
workspace = task / "environment" / WORKSPACE_DIRECTORY
|
|
466
|
+
workspace.mkdir(parents=True)
|
|
467
|
+
for path, text in seeds.items():
|
|
468
|
+
target = workspace.joinpath(*path.parts)
|
|
469
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
470
|
+
target.write_text(text, encoding="utf-8")
|
|
471
|
+
if roots:
|
|
472
|
+
(workspace.parent / WORKSPACE_FILESYSTEM_FILE).write_text(
|
|
473
|
+
json.dumps(roots, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
|
474
|
+
)
|
|
475
|
+
tests = task / "tests"
|
|
476
|
+
tests.mkdir()
|
|
477
|
+
(tests / CHECKS_FILE).write_text(
|
|
478
|
+
json.dumps(checks, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
|
479
|
+
)
|
|
480
|
+
script = tests / TEST_SCRIPT_FILE
|
|
481
|
+
script.write_text(GRADER_SCRIPT, encoding="utf-8")
|
|
482
|
+
script.chmod(0o755)
|
|
483
|
+
return _read_task(task / TASK_FILE)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def run_tasks(
|
|
487
|
+
path: Path | str,
|
|
488
|
+
model: str,
|
|
489
|
+
*,
|
|
490
|
+
jobs_dir: Path | str = "jobs",
|
|
491
|
+
job_name: str | None = None,
|
|
492
|
+
attempts: int = 1,
|
|
493
|
+
concurrency: int = 1,
|
|
494
|
+
agent_options: Mapping[str, str] | None = None,
|
|
495
|
+
) -> Path:
|
|
496
|
+
"""Run the tasks in `path` with the Vis agent on this host and return the job folder.
|
|
497
|
+
|
|
498
|
+
`path` is one task folder or a dataset folder of tasks. `model` is the
|
|
499
|
+
`provider/model` route of the agent. Harbor writes the job to
|
|
500
|
+
`jobs_dir/job_name`. Without `job_name`, Harbor names the job with the time.
|
|
501
|
+
`attempts` is the number of trials for each task. `concurrency` is the number
|
|
502
|
+
of trials that run at the same time. `agent_options` are the agent options of
|
|
503
|
+
`harbor run --ak`, for example `{"source": "/path/to/vis"}`. The call sets
|
|
504
|
+
`HARBOR_TELEMETRY=off`, so Harbor sends no usage data. Each trial makes paid
|
|
505
|
+
model calls. Call it outside a running event loop.
|
|
506
|
+
"""
|
|
507
|
+
import asyncio
|
|
508
|
+
import os
|
|
509
|
+
|
|
510
|
+
from harbor.job import Job
|
|
511
|
+
from harbor.models.job.config import JobConfig
|
|
512
|
+
from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig
|
|
513
|
+
|
|
514
|
+
root = Path(path)
|
|
515
|
+
if (root / TASK_FILE).is_file():
|
|
516
|
+
folders = [root]
|
|
517
|
+
else:
|
|
518
|
+
folders = [task.path for task in read_tasks(root)]
|
|
519
|
+
if not folders:
|
|
520
|
+
raise ValueError(f"no Harbor tasks in {root}")
|
|
521
|
+
os.environ["HARBOR_TELEMETRY"] = "off"
|
|
522
|
+
config = JobConfig(
|
|
523
|
+
jobs_dir=Path(jobs_dir),
|
|
524
|
+
n_attempts=attempts,
|
|
525
|
+
n_concurrent_trials=concurrency,
|
|
526
|
+
quiet=True,
|
|
527
|
+
agents=[
|
|
528
|
+
AgentConfig(
|
|
529
|
+
import_path=AGENT, model_name=model, kwargs=dict(agent_options or {})
|
|
530
|
+
)
|
|
531
|
+
],
|
|
532
|
+
environment=EnvironmentConfig(import_path=ENVIRONMENT),
|
|
533
|
+
tasks=[TaskConfig(path=folder) for folder in folders],
|
|
534
|
+
datasets=[],
|
|
535
|
+
)
|
|
536
|
+
if job_name is not None:
|
|
537
|
+
config = config.model_copy(update={"job_name": job_name})
|
|
538
|
+
|
|
539
|
+
async def run() -> Path:
|
|
540
|
+
job = await Job.create(config)
|
|
541
|
+
await job.run()
|
|
542
|
+
return job.job_dir
|
|
543
|
+
|
|
544
|
+
return asyncio.run(run())
|
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Run it with ``python -m blockether.vis.evals.judge PATH --provider P --model M``.
|
|
4
4
|
``PATH`` is one trial directory or a Harbor job directory, for example a job of
|
|
5
|
-
``harbor run``.
|
|
6
|
-
calls.
|
|
5
|
+
``harbor run``. Each judgement makes paid model calls.
|
|
7
6
|
|
|
8
7
|
``harbor analyze`` runs Claude Code in Docker. This module uses the same prompt,
|
|
9
8
|
result validation and ``analysis.json`` file of Harbor. The Vis agent runs each
|
|
@@ -2940,20 +2940,33 @@ class ProviderStatus(_ProviderValue):
|
|
|
2940
2940
|
|
|
2941
2941
|
@dataclass(frozen=True, slots=True)
|
|
2942
2942
|
class ProviderModel(_ProviderValue):
|
|
2943
|
-
"""Model metadata returned by enrichment. extra keeps additional router fields.
|
|
2943
|
+
"""Model metadata returned by enrichment. extra keeps additional router fields.
|
|
2944
|
+
|
|
2945
|
+
is_tool_call and is_image_input are old names of is_tool_calls_supported and
|
|
2946
|
+
is_vision_supported. They still work. A new name wins over its old name.
|
|
2947
|
+
"""
|
|
2944
2948
|
|
|
2945
2949
|
name: str
|
|
2946
2950
|
context: int | None = None
|
|
2947
|
-
|
|
2948
|
-
|
|
2951
|
+
is_tool_calls_supported: bool | None = None
|
|
2952
|
+
is_vision_supported: bool | None = None
|
|
2949
2953
|
extra: Mapping[str, ProviderJSON] | None = None
|
|
2954
|
+
is_tool_call: bool | None = field(default=None, repr=False, compare=False)
|
|
2955
|
+
is_image_input: bool | None = field(default=None, repr=False, compare=False)
|
|
2950
2956
|
|
|
2951
2957
|
def __post_init__(self):
|
|
2952
2958
|
_provider_text(self.name, "model name", required=True)
|
|
2953
2959
|
_provider_number(self.context, "context", integer=True, minimum=1)
|
|
2954
|
-
for
|
|
2955
|
-
|
|
2956
|
-
|
|
2960
|
+
for new, old in (
|
|
2961
|
+
("is_tool_calls_supported", "is_tool_call"),
|
|
2962
|
+
("is_vision_supported", "is_image_input"),
|
|
2963
|
+
):
|
|
2964
|
+
for name in (new, old):
|
|
2965
|
+
if getattr(self, name) is not None:
|
|
2966
|
+
_provider_bool(getattr(self, name), name)
|
|
2967
|
+
if getattr(self, new) is None:
|
|
2968
|
+
object.__setattr__(self, new, getattr(self, old))
|
|
2969
|
+
object.__setattr__(self, old, None)
|
|
2957
2970
|
_provider_extra(self)
|
|
2958
2971
|
|
|
2959
2972
|
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
"""Make the package importable from the checkout, and give every test a clean host."""
|
|
2
2
|
|
|
3
|
-
import importlib.util
|
|
4
3
|
import os
|
|
5
4
|
import sys
|
|
6
5
|
from pathlib import Path
|
|
@@ -15,15 +14,6 @@ sys.modules.pop("blockether.vis._outside", None)
|
|
|
15
14
|
import blockether.vis.extension as vis # noqa: E402
|
|
16
15
|
import pytest # noqa: E402 (the path above is what makes `vis` importable)
|
|
17
16
|
|
|
18
|
-
# The Harbor agent, environment, check and judge need the `harbor` extra.
|
|
19
|
-
if importlib.util.find_spec("harbor") is None:
|
|
20
|
-
collect_ignore = [
|
|
21
|
-
"test_evals_agent.py",
|
|
22
|
-
"test_evals_check.py",
|
|
23
|
-
"test_evals_environment.py",
|
|
24
|
-
"test_evals_judge.py",
|
|
25
|
-
]
|
|
26
|
-
|
|
27
17
|
|
|
28
18
|
@pytest.fixture(autouse=True)
|
|
29
19
|
def outside_home(tmp_path, monkeypatch):
|
|
@@ -72,7 +72,7 @@ def credential():
|
|
|
72
72
|
llm_headers=__CREDENTIAL_HEADERS__)
|
|
73
73
|
|
|
74
74
|
def enrich(provider, router_opts):
|
|
75
|
-
return [vis.ProviderModel(m["name"], context=200000,
|
|
75
|
+
return [vis.ProviderModel(m["name"], context=200000, is_tool_calls_supported=True)
|
|
76
76
|
for m in provider["models"]]
|
|
77
77
|
|
|
78
78
|
vis.register_extension(vis.Extension(name="sdk-provider", description="Typed provider integration",
|