verifiers 0.1.15.dev8__tar.gz → 0.1.15.dev9__tar.gz

{verifiers-0.1.15.dev8 → verifiers-0.1.15.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev8
+Version: 0.1.15.dev9
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -227,7 +227,8 @@ class MyTasksetConfig(vf.TasksetConfig):
     split: str = "train"
 
 
-class MyTaskset(vf.Taskset[MyTasksetConfig]):
+class MyTaskset(vf.Taskset):
+    config: MyTasksetConfig
     _default_rewards = (contains_answer,)
 
     def rows(self) -> list[dict[str, object]]:
@@ -242,12 +243,15 @@ class MyTaskset(vf.Taskset[MyTasksetConfig]):
         return [row for row in rows if row["split"] == self.config.split]
 
 
-class MyEnvConfig(vf.EnvConfig):
-    taskset: MyTasksetConfig = MyTasksetConfig()
+def load_taskset(config: MyTasksetConfig) -> MyTaskset:
+    assert isinstance(config, MyTasksetConfig)
+    return MyTaskset(config=config)
 
 
-def load_environment(config: MyEnvConfig) -> vf.Env:
-    return vf.Env(taskset=MyTaskset(config=config.taskset))
+def load_environment(config: vf.EnvConfig) -> vf.Env:
+    taskset_config = config.taskset
+    assert isinstance(taskset_config, MyTasksetConfig)
+    return vf.Env(taskset=load_taskset(taskset_config))
 ```
 If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.

{verifiers-0.1.15.dev8 → verifiers-0.1.15.dev9}/README.md

@@ -151,7 +151,8 @@ class MyTasksetConfig(vf.TasksetConfig):
     split: str = "train"
 
 
-class MyTaskset(vf.Taskset[MyTasksetConfig]):
+class MyTaskset(vf.Taskset):
+    config: MyTasksetConfig
     _default_rewards = (contains_answer,)
 
     def rows(self) -> list[dict[str, object]]:
@@ -166,12 +167,15 @@ class MyTaskset(vf.Taskset[MyTasksetConfig]):
         return [row for row in rows if row["split"] == self.config.split]
 
 
-class MyEnvConfig(vf.EnvConfig):
-    taskset: MyTasksetConfig = MyTasksetConfig()
+def load_taskset(config: MyTasksetConfig) -> MyTaskset:
+    assert isinstance(config, MyTasksetConfig)
+    return MyTaskset(config=config)
 
 
-def load_environment(config: MyEnvConfig) -> vf.Env:
-    return vf.Env(taskset=MyTaskset(config=config.taskset))
+def load_environment(config: vf.EnvConfig) -> vf.Env:
+    taskset_config = config.taskset
+    assert isinstance(taskset_config, MyTasksetConfig)
+    return vf.Env(taskset=load_taskset(taskset_config))
 ```
 If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.

{verifiers-0.1.15.dev8 → verifiers-0.1.15.dev9}/tests/test_envs.py

@@ -216,10 +216,14 @@ def help_test_can_load_env(tmp_venv_dir: Path, env_dir: Path):
 
 def help_test_can_eval_env(tmp_venv_dir: Path, env_dir: Path):
     """Test that the environment can be run via vf-eval."""
-    if os.getenv("OPENAI_API_KEY"):
-        model_flags = "-m gpt-4.1-mini -b https://api.openai.com/v1 -k OPENAI_API_KEY"
-    elif os.getenv("PRIME_API_KEY"):
+    if env_dir.name == "tau2_bench_v1" and not os.getenv("PRIME_API_KEY"):
+        pytest.skip(
+            "Skipping tau2 default eval because PRIME_API_KEY is not configured"
+        )
+    if os.getenv("PRIME_API_KEY"):
         model_flags = "-m openai/gpt-4.1-mini -b https://api.pinference.ai/api/v1 -k PRIME_API_KEY"
+    elif os.getenv("OPENAI_API_KEY"):
+        model_flags = "-m gpt-4.1-mini -b https://api.openai.com/v1 -k OPENAI_API_KEY"
     else:
         pytest.skip("Skipping vf-eval smoke test because no API key is configured")
 

{verifiers-0.1.15.dev8 → verifiers-0.1.15.dev9}/tests/test_imports.py

@@ -1,6 +1,26 @@
+import importlib
+import sys
+
 import verifiers
 
 
+def test_v1_taskset_imports_do_not_import_textarena():
+    textarena_module = "verifiers.v1.packages.tasksets.textarena"
+    sys.modules.pop(textarena_module, None)
+
+    tasksets = importlib.import_module("verifiers.v1.packages.tasksets")
+    tasksets.__dict__.pop("TextArenaTaskset", None)
+    tasksets.__dict__.pop("TextArenaTasksetConfig", None)
+    importlib.reload(tasksets)
+    assert textarena_module not in sys.modules
+
+    v1 = importlib.import_module("verifiers.v1")
+    v1.__dict__.pop("TextArenaTaskset", None)
+    v1.__dict__.pop("TextArenaTasksetConfig", None)
+    importlib.reload(v1)
+    assert textarena_module not in sys.modules
+
+
 class TestImports:
     """Test that all public API imports work correctly.
     This was inspired by issue #349.

{verifiers-0.1.15.dev8 → verifiers-0.1.15.dev9}/tests/test_rlm_env.py

@@ -1467,46 +1467,7 @@ class TestSubLLMTrajectorySteps:
 
 
 # =============================================================================
-# 13. Tunnel Utils (kept for coverage)
-# =============================================================================
-
-
-class TestExtractTunnelUrlFromLine:
-    def test_extract_valid_url(self):
-        from verifiers.utils.tunnel_utils import extract_tunnel_url_from_line
-
-        line = (
-            "2024-01-01 12:00:00 INF https://random-words.trycloudflare.com registered"
-        )
-        url = extract_tunnel_url_from_line(line)
-        assert url == "https://random-words.trycloudflare.com"
-
-    def test_return_none_for_no_url(self):
-        from verifiers.utils.tunnel_utils import extract_tunnel_url_from_line
-
-        line = "Starting cloudflared tunnel..."
-        url = extract_tunnel_url_from_line(line)
-        assert url is None
-
-    def test_handle_trailing_characters(self):
-        from verifiers.utils.tunnel_utils import extract_tunnel_url_from_line
-
-        line = "https://test-tunnel.trycloudflare.com/path?query=1 some text"
-        url = extract_tunnel_url_from_line(line)
-        assert url is not None
-        assert url.startswith("https://")
-        assert ".trycloudflare.com" in url
-
-    def test_no_https_prefix(self):
-        from verifiers.utils.tunnel_utils import extract_tunnel_url_from_line
-
-        line = "something.trycloudflare.com without https"
-        url = extract_tunnel_url_from_line(line)
-        assert url is None
-
-
-# =============================================================================
-# 14. RLM Exception Hierarchy
+# 13. RLM Exception Hierarchy
 # =============================================================================
 
 

{verifiers-0.1.15.dev8 → verifiers-0.1.15.dev9}/tests/test_tool_utils.py

@@ -7,6 +7,10 @@ from pydantic import Field
 from verifiers.utils.tool_utils import convert_func_to_tool_def
 
 
+def tool_dump_without_strict(tool):
+    return tool.model_dump(exclude={"strict"}, exclude_none=True)
+
+
 class TestToolUtils:
     """Test cases for the tool_utils module."""
 
@@ -28,7 +32,8 @@ class TestToolUtils:
             return 1.0
 
         result = convert_func_to_tool_def(test_func)
-        assert result.model_dump(exclude_none=True) == {
+        assert result.strict is True
+        assert tool_dump_without_strict(result) == {
             "name": "test_func",
             "description": "This is a test function.",
             "parameters": {
@@ -63,7 +68,7 @@ class TestToolUtils:
             return 1.0
 
         result = convert_func_to_tool_def(test_func)
-        assert result.model_dump(exclude_none=True) == {
+        assert tool_dump_without_strict(result) == {
             "name": "test_func",
             "description": "",
             "parameters": {
@@ -94,7 +99,7 @@ class TestToolUtils:
             return None
 
         result = convert_func_to_tool_def(test_func)
-        assert result.model_dump(exclude_none=True) == {
+        assert tool_dump_without_strict(result) == {
             "name": "test_func",
             "description": "",
             "parameters": {
@@ -121,7 +126,7 @@ class TestToolUtils:
             return None
 
         result = convert_func_to_tool_def(test_func)
-        assert result.model_dump(exclude_none=True) == {
+        assert tool_dump_without_strict(result) == {
             "name": "test_func",
             "description": "",
             "parameters": {
@@ -147,7 +152,7 @@ class TestToolUtils:
             return None
 
         result = convert_func_to_tool_def(test_func)
-        assert result.model_dump(exclude_none=True) == {
+        assert tool_dump_without_strict(result) == {
             "name": "test_func",
             "description": "This is a test function.",
             "parameters": {
@@ -171,7 +176,7 @@ class TestToolUtils:
             return None
 
         result = convert_func_to_tool_def(test_func)
-        assert result.model_dump(exclude_none=True) == {
+        assert tool_dump_without_strict(result) == {
             "name": "test_func",
             "description": "Search pages.",
             "parameters": {