sciworld 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sciworld-0.1.0/LICENSE +21 -0
- sciworld-0.1.0/PKG-INFO +156 -0
- sciworld-0.1.0/README.md +131 -0
- sciworld-0.1.0/pyproject.toml +44 -0
- sciworld-0.1.0/sciworld/__init__.py +32 -0
- sciworld-0.1.0/sciworld/__main__.py +5 -0
- sciworld-0.1.0/sciworld/_api.py +242 -0
- sciworld-0.1.0/sciworld/cache.py +42 -0
- sciworld-0.1.0/sciworld/cli.py +1398 -0
- sciworld-0.1.0/sciworld/config.py +97 -0
- sciworld-0.1.0/sciworld/containers/agent.Dockerfile +59 -0
- sciworld-0.1.0/sciworld/containers/agent.def +52 -0
- sciworld-0.1.0/sciworld/evaluate_cmd.py +403 -0
- sciworld-0.1.0/sciworld/format_stream.py +526 -0
- sciworld-0.1.0/sciworld/github.py +315 -0
- sciworld-0.1.0/sciworld/info_cmd.py +234 -0
- sciworld-0.1.0/sciworld/launcher.py +1022 -0
- sciworld-0.1.0/sciworld/milestones.py +34 -0
- sciworld-0.1.0/sciworld/normscore.py +150 -0
- sciworld-0.1.0/sciworld/package.py +198 -0
- sciworld-0.1.0/sciworld/prompts/continue.md +3 -0
- sciworld-0.1.0/sciworld/prompts/environment.md +7 -0
- sciworld-0.1.0/sciworld/prompts/improve.md +60 -0
- sciworld-0.1.0/sciworld/prompts/scratch.md +60 -0
- sciworld-0.1.0/sciworld/prompts/system_prompt.md +1 -0
- sciworld-0.1.0/sciworld/prompts/test.md +7 -0
- sciworld-0.1.0/sciworld/research_cmd.py +842 -0
- sciworld-0.1.0/sciworld/select_cmd.py +231 -0
- sciworld-0.1.0/sciworld/session.py +47 -0
- sciworld-0.1.0/sciworld/style.py +719 -0
- sciworld-0.1.0/sciworld/widgets.py +720 -0
- sciworld-0.1.0/sciworld.egg-info/PKG-INFO +156 -0
- sciworld-0.1.0/sciworld.egg-info/SOURCES.txt +36 -0
- sciworld-0.1.0/sciworld.egg-info/dependency_links.txt +1 -0
- sciworld-0.1.0/sciworld.egg-info/entry_points.txt +3 -0
- sciworld-0.1.0/sciworld.egg-info/requires.txt +1 -0
- sciworld-0.1.0/sciworld.egg-info/top_level.txt +1 -0
- sciworld-0.1.0/setup.cfg +4 -0
sciworld-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Taiming Lu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
sciworld-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sciworld
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Turn published papers into research environments an AI agent can work in
|
|
5
|
+
Author: Taiming Lu
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/TaiMingLu/sciworld
|
|
8
|
+
Project-URL: Repository, https://github.com/TaiMingLu/sciworld
|
|
9
|
+
Project-URL: Issues, https://github.com/TaiMingLu/sciworld/issues
|
|
10
|
+
Keywords: ai,agents,research,reproducibility,benchmark,llm
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: prompt_toolkit>=3.0
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# sciworld
|
|
27
|
+
|
|
28
|
+
Turn published papers into research environments an AI agent can actually work in.
|
|
29
|
+
|
|
30
|
+
Every paper with public code is a frozen research trajectory — problem, method,
|
|
31
|
+
baseline, metric, result. `sciworld` packages one of those into an isolated
|
|
32
|
+
workspace, hands the keys to an agent (Claude Code or Codex), and lets it try
|
|
33
|
+
to **reproduce**, **improve**, or **solve from scratch** — then scores the result.
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install sciworld
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Or from source:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
git clone https://github.com/TaiMingLu/sciworld.git
|
|
43
|
+
cd sciworld
|
|
44
|
+
pip install -e .
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## CLI
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
sw init # one-time setup wizard
|
|
51
|
+
sw config [section] # view/edit (paths, compute, prompt, test-prompt, …)
|
|
52
|
+
sw select [--id ID] # browse papers; clones <data_dir>/envs/<id>/source/
|
|
53
|
+
sw info --id <arxiv_id> [--full] # show paper overview/problem/experiments
|
|
54
|
+
sw research --id <arxiv_id> [flags] # build + launch a research env
|
|
55
|
+
sw evaluate --id <arxiv_id> [--run NAME] # compare run scores to paper scores
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### `sw research` flags
|
|
59
|
+
|
|
60
|
+
| Flag | What it does |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `--id ARXIV_ID` | paper to work on (required for non-interactive use) |
|
|
63
|
+
| `--mode improve \| scratch` | agent gets the method + results (`improve`) vs problem-only (`scratch`) |
|
|
64
|
+
| `--agent claude-code \| codex` | which CLI runs inside the container |
|
|
65
|
+
| `--tag TAG` | suffix for the run folder (e.g. `v2`, `ablation`) |
|
|
66
|
+
| `--path DIR` | override workspace destination |
|
|
67
|
+
| `--loop` | keep resuming the agent's session until `status.json.done=true` |
|
|
68
|
+
| `--max-loops N` | safety cap for `--loop` (default 10) |
|
|
69
|
+
| `--setup-only` | build the env + write `launch.sh` but **don't launch the agent** |
|
|
70
|
+
| `--refresh` | bypass paper cache |
|
|
71
|
+
|
|
72
|
+
`sw research` always writes a standalone `launch_<run_name>.sh` as a sibling
|
|
73
|
+
of the workspace. Normal runs execute it inside tmux/screen/nohup. With
|
|
74
|
+
`--setup-only`, nothing starts — the env is prepared and you can run
|
|
75
|
+
`bash <path>/launch_<run_name>.sh` yourself (inside your own job scheduler,
|
|
76
|
+
cron, or another session manager). Re-running the script is safe.
|
|
77
|
+
|
|
78
|
+
### Train / test split (anti-shortcut)
|
|
79
|
+
|
|
80
|
+
After the agent sets `status.json.done=true`, sciworld silently enters a
|
|
81
|
+
test phase: copies the hidden `source/eval/test/` + `scripts/evaluate_test.sh`
|
|
82
|
+
into the workspace, resumes the agent with a strict "be honest, do not
|
|
83
|
+
cheat, do not modify your method" prompt, and records the test-side scores
|
|
84
|
+
to `scoring/scores_test.json`. The test phase locks `method/` read-only.
|
|
85
|
+
Nothing test-related is present in the workspace during main-phase operation.
|
|
86
|
+
|
|
87
|
+
### Trajectory and token capture
|
|
88
|
+
|
|
89
|
+
Each CLI invocation writes raw JSONL events to
|
|
90
|
+
`<env_dir>/trajectory/turn_NNNN_raw.jsonl` and merges a unified summary
|
|
91
|
+
into `<env_dir>/trajectory/stats.json` with `{input_tokens, output_tokens,
|
|
92
|
+
num_turns, duration_s, cost_usd, tool_uses}` — identical shape for Claude
|
|
93
|
+
Code and Codex.
|
|
94
|
+
|
|
95
|
+
## Python API
|
|
96
|
+
|
|
97
|
+
Every CLI command is also a Python function with the same name and matching
|
|
98
|
+
kwargs. Required args missing → `TypeError`; non-zero exit → `RuntimeError`
|
|
99
|
+
(no interactive prompts — notebook/script-safe).
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
import sciworld
|
|
103
|
+
from pathlib import Path
|
|
104
|
+
|
|
105
|
+
info = sciworld.research(
|
|
106
|
+
id="2505.19252", agent="claude-code", mode="scratch",
|
|
107
|
+
loop=True, max_loops=5,
|
|
108
|
+
system_prompt=Path("~/my_prompts/strict.md").expanduser(),
|
|
109
|
+
test_prompt="Run evaluate_test.sh, write reflection.md, stop.",
|
|
110
|
+
setup_only=False, # True → write launch.sh, don't launch
|
|
111
|
+
)
|
|
112
|
+
# → {"arxiv_id", "run_name", "env_dir", "source_dir", "launch_script"}
|
|
113
|
+
|
|
114
|
+
sciworld.evaluate(id="2505.19252", run=info["run_name"])
|
|
115
|
+
sciworld.info(id="2505.19252")
|
|
116
|
+
sciworld.select(id="2505.19252")
|
|
117
|
+
sciworld.config() # returns current config dict
|
|
118
|
+
sciworld.config(section="compute") # opens the interactive editor
|
|
119
|
+
sciworld.init(force=False) # first-time setup wizard
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Prompt kwargs (`system_prompt`, `mode_prompt`, `continue_prompt`,
|
|
123
|
+
`test_prompt`, `compute`) accept either a literal `str` or a
|
|
124
|
+
`pathlib.Path` — paths are read at call time. Use `import sciworld as sw`
|
|
125
|
+
if you prefer the two-letter alias.
|
|
126
|
+
|
|
127
|
+
## Where things live
|
|
128
|
+
|
|
129
|
+
| Path | What's there |
|
|
130
|
+
|---|---|
|
|
131
|
+
| `~/.sciworld/config.json` | your config (paths, container, session, env bindings) |
|
|
132
|
+
| `~/.sciworld/instructions/*.md` | your customized prompts (system, research_scratch, continue, test, compute) |
|
|
133
|
+
| `<data_dir>/envs/<id>/source/` | cloned paper repo |
|
|
134
|
+
| `<data_dir>/envs/<id>/<run_name>/` | per-run agent workspace (bind-mounted into container) |
|
|
135
|
+
| `<data_dir>/envs/<id>/launch_<run_name>.sh` | launcher — **outside** the workspace, not visible to the agent |
|
|
136
|
+
| `<run_workspace>/agent.log` | full pretty-printed session log |
|
|
137
|
+
| `<run_workspace>/trajectory/` | raw per-turn JSONL + merged stats.json |
|
|
138
|
+
| `<run_workspace>/scoring/scores_train.json` | agent's train-side scores |
|
|
139
|
+
| `<run_workspace>/scoring/scores_test.json` | agent's test-side scores (written during test phase) |
|
|
140
|
+
| `<run_workspace>/idea.md`, `reflection.md` | agent's end-of-run notes |
|
|
141
|
+
|
|
142
|
+
## Default prompts
|
|
143
|
+
|
|
144
|
+
All bundled prompts live as markdown under `sciworld/prompts/` — nothing
|
|
145
|
+
prompt-related is hardcoded in Python. Per-user overrides go into
|
|
146
|
+
`~/.sciworld/instructions/` via `sw config …`.
|
|
147
|
+
|
|
148
|
+
| File | When it's used |
|
|
149
|
+
|---|---|
|
|
150
|
+
| `system_prompt.md` | base role description, prepended to every launch |
|
|
151
|
+
| `improve.md`, `scratch.md` | mode-specific task instructions |
|
|
152
|
+
| `environment.md` | container housekeeping note (appended to every launch) |
|
|
153
|
+
| `continue.md` | follow-up message for each `--loop` iteration |
|
|
154
|
+
| `test.md` | strict test-phase prompt (sent only at test-phase transition) |
|
|
155
|
+
|
|
156
|
+
Run any command with `--help` for details.
|
sciworld-0.1.0/README.md
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# sciworld
|
|
2
|
+
|
|
3
|
+
Turn published papers into research environments an AI agent can actually work in.
|
|
4
|
+
|
|
5
|
+
Every paper with public code is a frozen research trajectory — problem, method,
|
|
6
|
+
baseline, metric, result. `sciworld` packages one of those into an isolated
|
|
7
|
+
workspace, hands the keys to an agent (Claude Code or Codex), and lets it try
|
|
8
|
+
to **reproduce**, **improve**, or **solve from scratch** — then scores the result.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install sciworld
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Or from source:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
git clone https://github.com/TaiMingLu/sciworld.git
|
|
18
|
+
cd sciworld
|
|
19
|
+
pip install -e .
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## CLI
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
sw init # one-time setup wizard
|
|
26
|
+
sw config [section] # view/edit (paths, compute, prompt, test-prompt, …)
|
|
27
|
+
sw select [--id ID] # browse papers; clones <data_dir>/envs/<id>/source/
|
|
28
|
+
sw info --id <arxiv_id> [--full] # show paper overview/problem/experiments
|
|
29
|
+
sw research --id <arxiv_id> [flags] # build + launch a research env
|
|
30
|
+
sw evaluate --id <arxiv_id> [--run NAME] # compare run scores to paper scores
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### `sw research` flags
|
|
34
|
+
|
|
35
|
+
| Flag | What it does |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `--id ARXIV_ID` | paper to work on (required for non-interactive use) |
|
|
38
|
+
| `--mode improve \| scratch` | agent gets the method + results (`improve`) vs problem-only (`scratch`) |
|
|
39
|
+
| `--agent claude-code \| codex` | which CLI runs inside the container |
|
|
40
|
+
| `--tag TAG` | suffix for the run folder (e.g. `v2`, `ablation`) |
|
|
41
|
+
| `--path DIR` | override workspace destination |
|
|
42
|
+
| `--loop` | keep resuming the agent's session until `status.json.done=true` |
|
|
43
|
+
| `--max-loops N` | safety cap for `--loop` (default 10) |
|
|
44
|
+
| `--setup-only` | build the env + write `launch.sh` but **don't launch the agent** |
|
|
45
|
+
| `--refresh` | bypass paper cache |
|
|
46
|
+
|
|
47
|
+
`sw research` always writes a standalone `launch_<run_name>.sh` as a sibling
|
|
48
|
+
of the workspace. Normal runs execute it inside tmux/screen/nohup. With
|
|
49
|
+
`--setup-only`, nothing starts — the env is prepared and you can run
|
|
50
|
+
`bash <path>/launch_<run_name>.sh` yourself (inside your own job scheduler,
|
|
51
|
+
cron, or another session manager). Re-running the script is safe.
|
|
52
|
+
|
|
53
|
+
### Train / test split (anti-shortcut)
|
|
54
|
+
|
|
55
|
+
After the agent sets `status.json.done=true`, sciworld silently enters a
|
|
56
|
+
test phase: copies the hidden `source/eval/test/` + `scripts/evaluate_test.sh`
|
|
57
|
+
into the workspace, resumes the agent with a strict "be honest, do not
|
|
58
|
+
cheat, do not modify your method" prompt, and records the test-side scores
|
|
59
|
+
to `scoring/scores_test.json`. The test phase locks `method/` read-only.
|
|
60
|
+
Nothing test-related is present in the workspace during main-phase operation.
|
|
61
|
+
|
|
62
|
+
### Trajectory and token capture
|
|
63
|
+
|
|
64
|
+
Each CLI invocation writes raw JSONL events to
|
|
65
|
+
`<env_dir>/trajectory/turn_NNNN_raw.jsonl` and merges a unified summary
|
|
66
|
+
into `<env_dir>/trajectory/stats.json` with `{input_tokens, output_tokens,
|
|
67
|
+
num_turns, duration_s, cost_usd, tool_uses}` — identical shape for Claude
|
|
68
|
+
Code and Codex.
|
|
69
|
+
|
|
70
|
+
## Python API
|
|
71
|
+
|
|
72
|
+
Every CLI command is also a Python function with the same name and matching
|
|
73
|
+
kwargs. Required args missing → `TypeError`; non-zero exit → `RuntimeError`
|
|
74
|
+
(no interactive prompts — notebook/script-safe).
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import sciworld
|
|
78
|
+
from pathlib import Path
|
|
79
|
+
|
|
80
|
+
info = sciworld.research(
|
|
81
|
+
id="2505.19252", agent="claude-code", mode="scratch",
|
|
82
|
+
loop=True, max_loops=5,
|
|
83
|
+
system_prompt=Path("~/my_prompts/strict.md").expanduser(),
|
|
84
|
+
test_prompt="Run evaluate_test.sh, write reflection.md, stop.",
|
|
85
|
+
setup_only=False, # True → write launch.sh, don't launch
|
|
86
|
+
)
|
|
87
|
+
# → {"arxiv_id", "run_name", "env_dir", "source_dir", "launch_script"}
|
|
88
|
+
|
|
89
|
+
sciworld.evaluate(id="2505.19252", run=info["run_name"])
|
|
90
|
+
sciworld.info(id="2505.19252")
|
|
91
|
+
sciworld.select(id="2505.19252")
|
|
92
|
+
sciworld.config() # returns current config dict
|
|
93
|
+
sciworld.config(section="compute") # opens the interactive editor
|
|
94
|
+
sciworld.init(force=False) # first-time setup wizard
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Prompt kwargs (`system_prompt`, `mode_prompt`, `continue_prompt`,
|
|
98
|
+
`test_prompt`, `compute`) accept either a literal `str` or a
|
|
99
|
+
`pathlib.Path` — paths are read at call time. Use `import sciworld as sw`
|
|
100
|
+
if you prefer the two-letter alias.
|
|
101
|
+
|
|
102
|
+
## Where things live
|
|
103
|
+
|
|
104
|
+
| Path | What's there |
|
|
105
|
+
|---|---|
|
|
106
|
+
| `~/.sciworld/config.json` | your config (paths, container, session, env bindings) |
|
|
107
|
+
| `~/.sciworld/instructions/*.md` | your customized prompts (system, research_scratch, continue, test, compute) |
|
|
108
|
+
| `<data_dir>/envs/<id>/source/` | cloned paper repo |
|
|
109
|
+
| `<data_dir>/envs/<id>/<run_name>/` | per-run agent workspace (bind-mounted into container) |
|
|
110
|
+
| `<data_dir>/envs/<id>/launch_<run_name>.sh` | launcher — **outside** the workspace, not visible to the agent |
|
|
111
|
+
| `<run_workspace>/agent.log` | full pretty-printed session log |
|
|
112
|
+
| `<run_workspace>/trajectory/` | raw per-turn JSONL + merged stats.json |
|
|
113
|
+
| `<run_workspace>/scoring/scores_train.json` | agent's train-side scores |
|
|
114
|
+
| `<run_workspace>/scoring/scores_test.json` | agent's test-side scores (written during test phase) |
|
|
115
|
+
| `<run_workspace>/idea.md`, `reflection.md` | agent's end-of-run notes |
|
|
116
|
+
|
|
117
|
+
## Default prompts
|
|
118
|
+
|
|
119
|
+
All bundled prompts live as markdown under `sciworld/prompts/` — nothing
|
|
120
|
+
prompt-related is hardcoded in Python. Per-user overrides go into
|
|
121
|
+
`~/.sciworld/instructions/` via `sw config …`.
|
|
122
|
+
|
|
123
|
+
| File | When it's used |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `system_prompt.md` | base role description, prepended to every launch |
|
|
126
|
+
| `improve.md`, `scratch.md` | mode-specific task instructions |
|
|
127
|
+
| `environment.md` | container housekeeping note (appended to every launch) |
|
|
128
|
+
| `continue.md` | follow-up message for each `--loop` iteration |
|
|
129
|
+
| `test.md` | strict test-phase prompt (sent only at test-phase transition) |
|
|
130
|
+
|
|
131
|
+
Run any command with `--help` for details.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "sciworld"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Turn published papers into research environments an AI agent can work in"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.8"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Taiming Lu" },
|
|
10
|
+
]
|
|
11
|
+
keywords = ["ai", "agents", "research", "reproducibility", "benchmark", "llm"]
|
|
12
|
+
dependencies = [
|
|
13
|
+
"prompt_toolkit>=3.0",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Operating System :: POSIX :: Linux",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/TaiMingLu/sciworld"
|
|
29
|
+
Repository = "https://github.com/TaiMingLu/sciworld"
|
|
30
|
+
Issues = "https://github.com/TaiMingLu/sciworld/issues"
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
sciworld = "sciworld.cli:main"
|
|
34
|
+
sw = "sciworld.cli:main"
|
|
35
|
+
|
|
36
|
+
[build-system]
|
|
37
|
+
requires = ["setuptools>=64"]
|
|
38
|
+
build-backend = "setuptools.build_meta"
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
include = ["sciworld*"]
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.package-data]
|
|
44
|
+
sciworld = ["containers/*", "prompts/*.md", "format_stream.py"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""sciworld — research environments for paper reproduction.
|
|
2
|
+
|
|
3
|
+
This package exposes the same commands the ``sw`` CLI provides as Python
|
|
4
|
+
functions. Typical usage from a script or notebook:
|
|
5
|
+
|
|
6
|
+
import sciworld
|
|
7
|
+
|
|
8
|
+
sciworld.research(id="2505.19252", agent="claude-code", mode="scratch",
|
|
9
|
+
loop=True, max_loops=5)
|
|
10
|
+
sciworld.evaluate(id="2505.19252", run="claude-code-scratch")
|
|
11
|
+
|
|
12
|
+
Prompt kwargs accept either a literal string or a ``pathlib.Path`` — the
|
|
13
|
+
file is read at call time.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from sciworld._api import (
|
|
17
|
+
research,
|
|
18
|
+
evaluate,
|
|
19
|
+
select,
|
|
20
|
+
info,
|
|
21
|
+
init,
|
|
22
|
+
config,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"research",
|
|
27
|
+
"evaluate",
|
|
28
|
+
"select",
|
|
29
|
+
"info",
|
|
30
|
+
"init",
|
|
31
|
+
"config",
|
|
32
|
+
]
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Python API mirror of the ``sw`` CLI.
|
|
2
|
+
|
|
3
|
+
Every top-level CLI command has a sibling Python function here. The
|
|
4
|
+
function signature reuses the CLI's kwargs; interactive prompts are
|
|
5
|
+
replaced with ``TypeError`` on missing required args so calling the
|
|
6
|
+
API from a notebook or script never blocks on stdin.
|
|
7
|
+
|
|
8
|
+
Typical usage:
|
|
9
|
+
|
|
10
|
+
import sciworld
|
|
11
|
+
result = sciworld.research(
|
|
12
|
+
id="2505.19252", agent="claude-code", mode="scratch",
|
|
13
|
+
loop=True, max_loops=5,
|
|
14
|
+
system_prompt=Path("~/my_prompts/strict.md"),
|
|
15
|
+
)
|
|
16
|
+
# result = {"arxiv_id": ..., "run_name": ..., "env_dir": Path(...),
|
|
17
|
+
# "launch_script": Path(...), "source_dir": Path(...)}
|
|
18
|
+
|
|
19
|
+
sciworld.evaluate(id="2505.19252", run=result["run_name"])
|
|
20
|
+
|
|
21
|
+
Prompt kwargs (``system_prompt``, ``mode_prompt``, ``continue_prompt``,
|
|
22
|
+
``test_prompt``, ``compute``) accept either a literal string or a
|
|
23
|
+
``pathlib.Path``. Paths are read at call time — the resolved content
|
|
24
|
+
is used for that call; later file edits do not affect the running
|
|
25
|
+
session (for live per-iteration binding, see the `live_prompts`
|
|
26
|
+
roadmap item).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from types import SimpleNamespace
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
from sciworld import config as _config
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# ── Helpers ────────────────────────────────────────────────
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _resolve_prompt(value: Any) -> str | None:
|
|
42
|
+
"""Normalize a prompt kwarg: Path → file content; str → as-is; None → None."""
|
|
43
|
+
if value is None:
|
|
44
|
+
return None
|
|
45
|
+
if isinstance(value, Path):
|
|
46
|
+
return value.expanduser().read_text()
|
|
47
|
+
return str(value)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _require(**kwargs) -> None:
|
|
51
|
+
"""Raise TypeError for any kwarg that's None (required but missing)."""
|
|
52
|
+
missing = [k for k, v in kwargs.items() if v is None]
|
|
53
|
+
if missing:
|
|
54
|
+
raise TypeError(
|
|
55
|
+
f"Missing required argument(s): {', '.join(missing)}"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _run_with_overrides(callable_, args_ns, overrides: dict) -> int:
|
|
60
|
+
"""Run a cmd_* function with prompt overrides active for this call."""
|
|
61
|
+
if not overrides:
|
|
62
|
+
return callable_(args_ns)
|
|
63
|
+
token = _config._instruction_overrides.set(overrides)
|
|
64
|
+
try:
|
|
65
|
+
return callable_(args_ns)
|
|
66
|
+
finally:
|
|
67
|
+
_config._instruction_overrides.reset(token)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# ── research ──────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def research(
|
|
74
|
+
*,
|
|
75
|
+
id: str,
|
|
76
|
+
agent: str,
|
|
77
|
+
mode: str,
|
|
78
|
+
tag: str | None = None,
|
|
79
|
+
path: str | Path | None = None,
|
|
80
|
+
loop: bool = False,
|
|
81
|
+
max_loops: int = 10,
|
|
82
|
+
setup_only: bool = False,
|
|
83
|
+
refresh: bool = False,
|
|
84
|
+
org: str | None = None,
|
|
85
|
+
system_prompt: str | Path | None = None,
|
|
86
|
+
mode_prompt: str | Path | None = None,
|
|
87
|
+
continue_prompt: str | Path | None = None,
|
|
88
|
+
test_prompt: str | Path | None = None,
|
|
89
|
+
compute: str | Path | None = None,
|
|
90
|
+
) -> dict:
|
|
91
|
+
"""Build (and optionally launch) a research environment.
|
|
92
|
+
|
|
93
|
+
Mirrors ``sw research``. Returns a dict with ``arxiv_id``, ``run_name``,
|
|
94
|
+
``env_dir``, ``source_dir``, ``launch_script`` — all the paths you need
|
|
95
|
+
to inspect the workspace or tail the agent log.
|
|
96
|
+
|
|
97
|
+
Raises ``TypeError`` if any of the three required kwargs (``id``,
|
|
98
|
+
``agent``, ``mode``) is missing, and ``RuntimeError`` if the command
|
|
99
|
+
itself exits with a non-zero return code.
|
|
100
|
+
"""
|
|
101
|
+
_require(id=id, agent=agent, mode=mode)
|
|
102
|
+
|
|
103
|
+
from sciworld.research_cmd import cmd_research # lazy import
|
|
104
|
+
|
|
105
|
+
args_ns = SimpleNamespace(
|
|
106
|
+
paper_id=id,
|
|
107
|
+
agent=agent,
|
|
108
|
+
mode=mode,
|
|
109
|
+
tag=tag,
|
|
110
|
+
dest_path=str(path) if path else None,
|
|
111
|
+
loop=loop,
|
|
112
|
+
max_loops=max_loops,
|
|
113
|
+
setup_only=setup_only,
|
|
114
|
+
refresh=refresh,
|
|
115
|
+
org=org,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
overrides: dict[str, str] = {}
|
|
119
|
+
if system_prompt is not None:
|
|
120
|
+
overrides["system_prompt"] = _resolve_prompt(system_prompt)
|
|
121
|
+
if mode_prompt is not None:
|
|
122
|
+
overrides[f"research_{mode}"] = _resolve_prompt(mode_prompt)
|
|
123
|
+
if continue_prompt is not None:
|
|
124
|
+
overrides["continue"] = _resolve_prompt(continue_prompt)
|
|
125
|
+
if test_prompt is not None:
|
|
126
|
+
overrides["test"] = _resolve_prompt(test_prompt)
|
|
127
|
+
if compute is not None:
|
|
128
|
+
overrides["compute"] = _resolve_prompt(compute)
|
|
129
|
+
|
|
130
|
+
rc = _run_with_overrides(cmd_research, args_ns, overrides)
|
|
131
|
+
if rc != 0:
|
|
132
|
+
raise RuntimeError(f"sciworld.research exited with code {rc}")
|
|
133
|
+
|
|
134
|
+
result = getattr(args_ns, "_sw_result", None)
|
|
135
|
+
if result is None:
|
|
136
|
+
raise RuntimeError(
|
|
137
|
+
"sciworld.research completed but produced no result metadata"
|
|
138
|
+
)
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ── evaluate ──────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def evaluate(*, id: str, run: str | None = None) -> int:
|
|
146
|
+
"""Compare a run's scores to the paper's scores.
|
|
147
|
+
|
|
148
|
+
Mirrors ``sw evaluate``. Returns the command's exit code (0 = success).
|
|
149
|
+
Raises ``TypeError`` if ``id`` is missing.
|
|
150
|
+
"""
|
|
151
|
+
_require(id=id)
|
|
152
|
+
from sciworld.evaluate_cmd import cmd_evaluate # lazy import
|
|
153
|
+
|
|
154
|
+
args_ns = SimpleNamespace(arxiv_id=id, run=run)
|
|
155
|
+
return cmd_evaluate(args_ns)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ── select ────────────────────────────────────────────────
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def select(
|
|
162
|
+
*,
|
|
163
|
+
id: str | None = None,
|
|
164
|
+
path: str | Path | None = None,
|
|
165
|
+
refresh: bool = False,
|
|
166
|
+
) -> int:
|
|
167
|
+
"""Select a paper and clone its repo.
|
|
168
|
+
|
|
169
|
+
Mirrors ``sw select``. When ``id`` is given, fetches that paper
|
|
170
|
+
directly; when missing, raises ``TypeError`` (no interactive fallback
|
|
171
|
+
in the Python API).
|
|
172
|
+
"""
|
|
173
|
+
_require(id=id)
|
|
174
|
+
from sciworld.select_cmd import cmd_select # lazy import
|
|
175
|
+
|
|
176
|
+
args_ns = SimpleNamespace(
|
|
177
|
+
paper_id=id,
|
|
178
|
+
dest_path=str(path) if path else None,
|
|
179
|
+
refresh=refresh,
|
|
180
|
+
)
|
|
181
|
+
return cmd_select(args_ns)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# ── info ──────────────────────────────────────────────────
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def info(*, id: str, full: bool = False) -> int:
|
|
188
|
+
"""Show a paper's metadata (overview, problem, experiments, …).
|
|
189
|
+
|
|
190
|
+
Mirrors ``sw info``. Raises ``TypeError`` if ``id`` is missing.
|
|
191
|
+
"""
|
|
192
|
+
_require(id=id)
|
|
193
|
+
from sciworld.info_cmd import cmd_info # lazy import
|
|
194
|
+
|
|
195
|
+
args_ns = SimpleNamespace(arxiv_id=id, full=full)
|
|
196
|
+
return cmd_info(args_ns)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# ── config ────────────────────────────────────────────────
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def config(section: str | None = None, **overrides) -> dict:
|
|
203
|
+
"""View or update configuration.
|
|
204
|
+
|
|
205
|
+
``sciworld.config()`` → returns the current config as a dict.
|
|
206
|
+
``sciworld.config(section="...")`` → runs the interactive editor for
|
|
207
|
+
that section; raises ``ValueError`` if the section is unknown.
|
|
208
|
+
"""
|
|
209
|
+
if section is None and not overrides:
|
|
210
|
+
return dict(_config.load_config())
|
|
211
|
+
|
|
212
|
+
if section is not None:
|
|
213
|
+
from sciworld.cli import _CONFIG_SECTIONS, cmd_config
|
|
214
|
+
if section not in _CONFIG_SECTIONS:
|
|
215
|
+
valid = ", ".join(_CONFIG_SECTIONS)
|
|
216
|
+
raise ValueError(f"Unknown config section {section!r}. Valid: {valid}")
|
|
217
|
+
args_ns = SimpleNamespace(section=section)
|
|
218
|
+
cmd_config(args_ns)
|
|
219
|
+
return dict(_config.load_config())
|
|
220
|
+
|
|
221
|
+
# If we got here, someone passed raw kwargs with no section. Write them
|
|
222
|
+
# straight into config.json. (Low-level escape hatch.)
|
|
223
|
+
cfg = dict(_config.load_config())
|
|
224
|
+
cfg.update(overrides)
|
|
225
|
+
_config.save_config(cfg)
|
|
226
|
+
return cfg
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# ── init ──────────────────────────────────────────────────
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def init(*, force: bool = False) -> int:
|
|
233
|
+
"""Run the first-time setup wizard (builds the agent SIF, etc.).
|
|
234
|
+
|
|
235
|
+
Mirrors ``sw init``. The underlying command is interactive; calling
|
|
236
|
+
it from a script typically only makes sense with the setup already
|
|
237
|
+
partially complete. ``force=True`` rebuilds the agent container.
|
|
238
|
+
"""
|
|
239
|
+
from sciworld.cli import cmd_init # lazy import
|
|
240
|
+
|
|
241
|
+
args_ns = SimpleNamespace(force=force)
|
|
242
|
+
return cmd_init(args_ns)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""TTL-based cache for GitHub repo data."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from sciworld.config import config_dir
|
|
9
|
+
|
|
10
|
+
CACHE_DIR = config_dir() / "cache"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def cache_path(org: str) -> Path:
|
|
14
|
+
return CACHE_DIR / f"{org}_repos.json"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def load_cached(org: str, ttl: int = 600) -> list[dict] | None:
|
|
18
|
+
"""Load cached repo data. Returns None if stale or missing."""
|
|
19
|
+
path = cache_path(org)
|
|
20
|
+
if not path.exists():
|
|
21
|
+
return None
|
|
22
|
+
try:
|
|
23
|
+
data = json.loads(path.read_text())
|
|
24
|
+
if time.time() - data.get("timestamp", 0) > ttl:
|
|
25
|
+
return None
|
|
26
|
+
return data.get("repos", [])
|
|
27
|
+
except (json.JSONDecodeError, KeyError):
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def save_cached(org: str, repos: list[dict]) -> None:
|
|
32
|
+
"""Write repos to cache with current timestamp."""
|
|
33
|
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
data = {"timestamp": time.time(), "repos": repos}
|
|
35
|
+
cache_path(org).write_text(json.dumps(data))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def invalidate(org: str) -> None:
|
|
39
|
+
"""Delete cache for an org."""
|
|
40
|
+
path = cache_path(org)
|
|
41
|
+
if path.exists():
|
|
42
|
+
path.unlink()
|