batchgrid 0.1.0b1__tar.gz → 0.2.0b2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/PKG-INFO +53 -4
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/README.md +52 -3
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/pyproject.toml +4 -1
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/src/batchgrid/__init__.py +5 -1
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/src/batchgrid/_cli.py +47 -3
- batchgrid-0.2.0b2/src/batchgrid/_ui.py +244 -0
- batchgrid-0.2.0b2/tests/test_ui.py +196 -0
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/.gitignore +0 -0
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/src/batchgrid/_api.py +0 -0
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/src/batchgrid/errors.py +0 -0
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/src/batchgrid/py.typed +0 -0
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/tests/conftest.py +0 -0
- {batchgrid-0.1.0b1 → batchgrid-0.2.0b2}/tests/test_api.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: batchgrid
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0b2
|
|
4
4
|
Summary: Run a prompt, or a whole pipeline, over every row of a DataFrame or spreadsheet - with cost estimates, retries and resumable runs.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mertguvencli/batchgrid
|
|
6
6
|
Project-URL: Source, https://github.com/mertguvencli/batchgrid/tree/main/python
|
|
@@ -81,6 +81,47 @@ result = batchgrid.run("next_week.csv", plan="sentiment.json", max_cost=10)
|
|
|
81
81
|
|
|
82
82
|
The file is the same one `batchgrid --save-plan` writes and `batchgrid --plan` reads.
|
|
83
83
|
|
|
84
|
+
## In the browser
|
|
85
|
+
|
|
86
|
+
`batchgrid.ui()` opens the batchgrid workspace with your data in the sheet. In a Jupyter notebook it
|
|
87
|
+
shows in the cell's output; elsewhere, or with `inline=False`, it opens in your browser. You can
|
|
88
|
+
chat, check the plan and run it there, while the log prints in the notebook cell or terminal as it
|
|
89
|
+
happens:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
df2 = batchgrid.ui(df)
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
batchgrid is open at http://127.0.0.1:4817
|
|
97
|
+
|
|
98
|
+
❯ classify the sentiment of each review
|
|
99
|
+
✔ Reading data.csv - 1,000 rows · 1 column: review
|
|
100
|
+
✔ Asking gpt-5.6 for a plan - 1 step · 2.1k tokens
|
|
101
|
+
✔ Estimating the cost - About ~$0.08 for 1,000 rows
|
|
102
|
+
batchgrid: 100%|██████████| 1000/1000 [00:42<00:00, failed=0]
|
|
103
|
+
✔ 1000/1000 rows, 0 failed
|
|
104
|
+
↳ Sheet synced: 1,000 rows × 2 columns
|
|
105
|
+
|
|
106
|
+
Finished in the browser.
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
To come back, press **Finish** in the workspace, close its tab, or interrupt the cell. Once it stops,
|
|
110
|
+
a note takes the workspace's place in the cell. `ui()` returns the
|
|
111
|
+
sheet as the tab last showed it. For a DataFrame, that is a copy of yours with the new columns and
|
|
112
|
+
any cells you edited; untouched columns keep their dtypes. `ui()` with no data starts on an empty
|
|
113
|
+
sheet.
|
|
114
|
+
|
|
115
|
+
The workspace runs on your machine and uses the keys from `batchgrid config` or your environment. Your
|
|
116
|
+
browser has to be able to reach the machine running Python, so this does not work for a notebook on
|
|
117
|
+
a remote server.
|
|
118
|
+
|
|
119
|
+
From a terminal, the `batchgrid` command this package installs does the same:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
batchgrid ui data.csv # or: batchgrid --ui data.csv
|
|
123
|
+
```
|
|
124
|
+
|
|
84
125
|
## Reference
|
|
85
126
|
|
|
86
127
|
### `batchgrid.run(data, prompt=None, *, plan=None, ...) -> Result`
|
|
@@ -124,6 +165,12 @@ except KeyboardInterrupt:
|
|
|
124
165
|
result = batchgrid.resume(df)
|
|
125
166
|
```
|
|
126
167
|
|
|
168
|
+
### `batchgrid.ui(data=None, *, inline=None, height=720, open_browser=True, port=None, log=True) -> DataFrame | None`
|
|
169
|
+
|
|
170
|
+
See [In the browser](#in-the-browser). `inline` shows the workspace in the cell, `height` pixels tall;
|
|
171
|
+
by default it does in a Jupyter notebook and opens a browser tab anywhere else. `on_event` receives the same events as in `run()`, plus `ui`,
|
|
172
|
+
`sheet`, `browser_error` and `closed`.
|
|
173
|
+
|
|
127
174
|
### Errors
|
|
128
175
|
|
|
129
176
|
Everything raises `batchgrid.BatchgridError`, and its `code` says what went wrong:
|
|
@@ -140,12 +187,14 @@ Everything raises `batchgrid.BatchgridError`, and its `code` says what went wron
|
|
|
140
187
|
|
|
141
188
|
### Choosing the CLI
|
|
142
189
|
|
|
143
|
-
The
|
|
190
|
+
The `batchgrid` command pip installs hands everything to the Node CLI, so `batchgrid config`,
|
|
191
|
+
`batchgrid ui` and the rest work the same whether batchgrid came from pip or npm. The package looks
|
|
192
|
+
for the CLI in this order:
|
|
144
193
|
|
|
145
194
|
1. the `cli=[...]` argument
|
|
146
195
|
2. the `BATCHGRID_CLI` environment variable, e.g. `node /path/to/cli/dist/index.js`
|
|
147
|
-
3.
|
|
148
|
-
4. `npx "batchgrid@>=0.
|
|
196
|
+
3. the npm `batchgrid` on the PATH (never this package's own command)
|
|
197
|
+
4. `npx "batchgrid@>=0.3.0 <1"`
|
|
149
198
|
|
|
150
199
|
## Development
|
|
151
200
|
|
|
@@ -55,6 +55,47 @@ result = batchgrid.run("next_week.csv", plan="sentiment.json", max_cost=10)
|
|
|
55
55
|
|
|
56
56
|
The file is the same one `batchgrid --save-plan` writes and `batchgrid --plan` reads.
|
|
57
57
|
|
|
58
|
+
## In the browser
|
|
59
|
+
|
|
60
|
+
`batchgrid.ui()` opens the batchgrid workspace with your data in the sheet. In a Jupyter notebook it
|
|
61
|
+
shows in the cell's output; elsewhere, or with `inline=False`, it opens in your browser. You can
|
|
62
|
+
chat, check the plan and run it there, while the log prints in the notebook cell or terminal as it
|
|
63
|
+
happens:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
df2 = batchgrid.ui(df)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
batchgrid is open at http://127.0.0.1:4817
|
|
71
|
+
|
|
72
|
+
❯ classify the sentiment of each review
|
|
73
|
+
✔ Reading data.csv - 1,000 rows · 1 column: review
|
|
74
|
+
✔ Asking gpt-5.6 for a plan - 1 step · 2.1k tokens
|
|
75
|
+
✔ Estimating the cost - About ~$0.08 for 1,000 rows
|
|
76
|
+
batchgrid: 100%|██████████| 1000/1000 [00:42<00:00, failed=0]
|
|
77
|
+
✔ 1000/1000 rows, 0 failed
|
|
78
|
+
↳ Sheet synced: 1,000 rows × 2 columns
|
|
79
|
+
|
|
80
|
+
Finished in the browser.
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
To come back, press **Finish** in the workspace, close its tab, or interrupt the cell. Once it stops,
|
|
84
|
+
a note takes the workspace's place in the cell. `ui()` returns the
|
|
85
|
+
sheet as the tab last showed it. For a DataFrame, that is a copy of yours with the new columns and
|
|
86
|
+
any cells you edited; untouched columns keep their dtypes. `ui()` with no data starts on an empty
|
|
87
|
+
sheet.
|
|
88
|
+
|
|
89
|
+
The workspace runs on your machine and uses the keys from `batchgrid config` or your environment. Your
|
|
90
|
+
browser has to be able to reach the machine running Python, so this does not work for a notebook on
|
|
91
|
+
a remote server.
|
|
92
|
+
|
|
93
|
+
From a terminal, the `batchgrid` command this package installs does the same:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
batchgrid ui data.csv # or: batchgrid --ui data.csv
|
|
97
|
+
```
|
|
98
|
+
|
|
58
99
|
## Reference
|
|
59
100
|
|
|
60
101
|
### `batchgrid.run(data, prompt=None, *, plan=None, ...) -> Result`
|
|
@@ -98,6 +139,12 @@ except KeyboardInterrupt:
|
|
|
98
139
|
result = batchgrid.resume(df)
|
|
99
140
|
```
|
|
100
141
|
|
|
142
|
+
### `batchgrid.ui(data=None, *, inline=None, height=720, open_browser=True, port=None, log=True) -> DataFrame | None`
|
|
143
|
+
|
|
144
|
+
See [In the browser](#in-the-browser). `inline` shows the workspace in the cell, `height` pixels tall;
|
|
145
|
+
by default it does in a Jupyter notebook and opens a browser tab anywhere else. `on_event` receives the same events as in `run()`, plus `ui`,
|
|
146
|
+
`sheet`, `browser_error` and `closed`.
|
|
147
|
+
|
|
101
148
|
### Errors
|
|
102
149
|
|
|
103
150
|
Everything raises `batchgrid.BatchgridError`, and its `code` says what went wrong:
|
|
@@ -114,12 +161,14 @@ Everything raises `batchgrid.BatchgridError`, and its `code` says what went wron
|
|
|
114
161
|
|
|
115
162
|
### Choosing the CLI
|
|
116
163
|
|
|
117
|
-
The
|
|
164
|
+
The `batchgrid` command pip installs hands everything to the Node CLI, so `batchgrid config`,
|
|
165
|
+
`batchgrid ui` and the rest work the same whether batchgrid came from pip or npm. The package looks
|
|
166
|
+
for the CLI in this order:
|
|
118
167
|
|
|
119
168
|
1. the `cli=[...]` argument
|
|
120
169
|
2. the `BATCHGRID_CLI` environment variable, e.g. `node /path/to/cli/dist/index.js`
|
|
121
|
-
3.
|
|
122
|
-
4. `npx "batchgrid@>=0.
|
|
170
|
+
3. the npm `batchgrid` on the PATH (never this package's own command)
|
|
171
|
+
4. `npx "batchgrid@>=0.3.0 <1"`
|
|
123
172
|
|
|
124
173
|
## Development
|
|
125
174
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "batchgrid"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0b2"
|
|
8
8
|
description = "Run a prompt, or a whole pipeline, over every row of a DataFrame or spreadsheet - with cost estimates, retries and resumable runs."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -26,6 +26,9 @@ pandas = ["pandas>=1.5"]
|
|
|
26
26
|
progress = ["tqdm>=4.60"]
|
|
27
27
|
dev = ["pytest>=7", "pandas>=1.5"]
|
|
28
28
|
|
|
29
|
+
[project.scripts]
|
|
30
|
+
batchgrid = "batchgrid._cli:main"
|
|
31
|
+
|
|
29
32
|
[project.urls]
|
|
30
33
|
Homepage = "https://github.com/mertguvencli/batchgrid"
|
|
31
34
|
Source = "https://github.com/mertguvencli/batchgrid/tree/main/python"
|
|
@@ -5,15 +5,18 @@
|
|
|
5
5
|
result = batchgrid.run(df, "classify the sentiment of each review", max_cost=5)
|
|
6
6
|
result.data # df with the new columns
|
|
7
7
|
|
|
8
|
+
df2 = batchgrid.ui(df) # the same, in the browser; its log prints here
|
|
9
|
+
|
|
8
10
|
The work is done by the batchgrid CLI (Node.js 22+), which this package starts
|
|
9
11
|
and follows; see find_cli() for how it is located.
|
|
10
12
|
"""
|
|
11
13
|
|
|
12
14
|
from ._api import Plan, Result, plan, resume, run
|
|
15
|
+
from ._ui import ui
|
|
13
16
|
from ._cli import CLI_SPEC, find_cli
|
|
14
17
|
from .errors import BatchgridError, CostLimitError, MissingSecretsError, NoPlanError
|
|
15
18
|
|
|
16
|
-
__version__ = "0.
|
|
19
|
+
__version__ = "0.2.0b2"
|
|
17
20
|
|
|
18
21
|
__all__ = [
|
|
19
22
|
"BatchgridError",
|
|
@@ -27,4 +30,5 @@ __all__ = [
|
|
|
27
30
|
"plan",
|
|
28
31
|
"resume",
|
|
29
32
|
"run",
|
|
33
|
+
"ui",
|
|
30
34
|
]
|
|
@@ -14,12 +14,13 @@ import shutil
|
|
|
14
14
|
import signal
|
|
15
15
|
import subprocess
|
|
16
16
|
import sys
|
|
17
|
+
import sysconfig
|
|
17
18
|
import threading
|
|
18
19
|
from collections import deque
|
|
19
20
|
from typing import Callable, Deque, Dict, Iterator, List, Mapping, Optional, Sequence
|
|
20
21
|
|
|
21
|
-
#: The CLI releases this package speaks to: 0.
|
|
22
|
-
CLI_SPEC = "batchgrid@>=0.
|
|
22
|
+
#: The CLI releases this package speaks to: 0.3.0 was the first with ``ui``, and its events only gain fields
|
|
23
|
+
CLI_SPEC = "batchgrid@>=0.3.0 <1"
|
|
23
24
|
|
|
24
25
|
Event = Dict[str, object]
|
|
25
26
|
|
|
@@ -36,7 +37,7 @@ def find_cli(cli: Optional[Sequence[str]] = None) -> List[str]:
|
|
|
36
37
|
from_env = os.environ.get("BATCHGRID_CLI", "").strip()
|
|
37
38
|
if from_env:
|
|
38
39
|
return shlex.split(from_env, posix=os.name != "nt")
|
|
39
|
-
installed =
|
|
40
|
+
installed = _node_cli_on_path()
|
|
40
41
|
if installed:
|
|
41
42
|
return [installed]
|
|
42
43
|
npx = shutil.which("npx")
|
|
@@ -51,6 +52,49 @@ def find_cli(cli: Optional[Sequence[str]] = None) -> List[str]:
|
|
|
51
52
|
)
|
|
52
53
|
|
|
53
54
|
|
|
55
|
+
def _node_cli_on_path() -> Optional[str]:
|
|
56
|
+
"""The npm ``batchgrid`` on the PATH - not this package's own ``batchgrid``
|
|
57
|
+
command, which only hands over to it and would otherwise call itself."""
|
|
58
|
+
scripts = os.path.normcase(os.path.realpath(sysconfig.get_path("scripts") or ""))
|
|
59
|
+
for directory in os.environ.get("PATH", "").split(os.pathsep):
|
|
60
|
+
found = shutil.which("batchgrid", path=directory) if directory else None
|
|
61
|
+
if not found:
|
|
62
|
+
continue
|
|
63
|
+
if os.path.normcase(os.path.realpath(os.path.dirname(found))) == scripts or _is_python_script(found):
|
|
64
|
+
continue
|
|
65
|
+
return found
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _is_python_script(path: str) -> bool:
|
|
70
|
+
try:
|
|
71
|
+
with open(path, "rb") as fh:
|
|
72
|
+
first = fh.readline(200)
|
|
73
|
+
except OSError:
|
|
74
|
+
return False
|
|
75
|
+
return first.startswith(b"#!") and b"python" in first
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
79
|
+
"""The ``batchgrid`` command pip installs: the Node CLI, with the same arguments.
|
|
80
|
+
|
|
81
|
+
So ``batchgrid ui data.csv`` (or ``batchgrid --ui data.csv``) and every
|
|
82
|
+
other command work the same whether batchgrid came from pip or npm.
|
|
83
|
+
"""
|
|
84
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
85
|
+
try:
|
|
86
|
+
command = find_cli()
|
|
87
|
+
except Exception as err: # noqa: BLE001 - the message is the whole story here
|
|
88
|
+
print(f"batchgrid: {err}", file=sys.stderr)
|
|
89
|
+
return 1
|
|
90
|
+
if os.name != "nt":
|
|
91
|
+
# Become the CLI: its exit code, its signals, its terminal
|
|
92
|
+
os.execvp(command[0], [*command, *args])
|
|
93
|
+
# Ctrl+C reaches the CLI directly; this process only waits for it
|
|
94
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
95
|
+
return subprocess.call([*command, *args])
|
|
96
|
+
|
|
97
|
+
|
|
54
98
|
def stream_events(
|
|
55
99
|
command: Sequence[str],
|
|
56
100
|
args: Sequence[str],
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""ui() - the web workspace in the notebook or the browser, with its log here and its sheet coming back."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Callable, List, Mapping, Optional, Sequence
|
|
9
|
+
|
|
10
|
+
from ._api import Data, EventHandler, _input_file, _is_dataframe, _merge_output, _TempDir
|
|
11
|
+
from ._cli import Event, find_cli, stream_events
|
|
12
|
+
from .errors import BatchgridError
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
import pandas as pd
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def ui(
|
|
19
|
+
data: Optional[Data] = None,
|
|
20
|
+
*,
|
|
21
|
+
inline: Optional[bool] = None,
|
|
22
|
+
height: int = 720,
|
|
23
|
+
open_browser: bool = True,
|
|
24
|
+
port: Optional[int] = None,
|
|
25
|
+
log: bool = True,
|
|
26
|
+
on_event: Optional[EventHandler] = None,
|
|
27
|
+
cli: Optional[Sequence[str]] = None,
|
|
28
|
+
env: Optional[Mapping[str, str]] = None,
|
|
29
|
+
) -> Optional["pd.DataFrame"]:
|
|
30
|
+
"""Open the batchgrid workspace and wait until you are done there.
|
|
31
|
+
|
|
32
|
+
``data`` - a file path or a pandas DataFrame - opens in the sheet. In a
|
|
33
|
+
Jupyter notebook the workspace shows in the cell's output, ``height``
|
|
34
|
+
pixels tall; ``inline=False`` opens it in a browser tab instead, which is
|
|
35
|
+
what happens outside a notebook. Chat, plan and run there; what happens is
|
|
36
|
+
printed here as it happens, with a progress bar while rows run.
|
|
37
|
+
|
|
38
|
+
Press Finish in the workspace, close its tab, or interrupt the cell
|
|
39
|
+
(Ctrl+C) to come back. The sheet as the tab last showed it is returned as a DataFrame: for
|
|
40
|
+
a DataFrame you passed in, a copy with the new columns added and your
|
|
41
|
+
edits in it. None when nothing was opened, or pandas is not installed.
|
|
42
|
+
|
|
43
|
+
It runs on this machine, so it needs a browser that can reach it - not a
|
|
44
|
+
notebook running on a remote server.
|
|
45
|
+
"""
|
|
46
|
+
embed = _in_notebook() if inline is None else inline
|
|
47
|
+
frame_view = _Frame(height) if embed else None
|
|
48
|
+
printer = _Printer(link=not embed) if log else None
|
|
49
|
+
with _input_file(data) as (path, frame):
|
|
50
|
+
out = _TempDir()
|
|
51
|
+
try:
|
|
52
|
+
sheet = os.path.join(out.get(), "sheet.csv")
|
|
53
|
+
args: List[str] = ["ui", *([path] if path else []), "--output", sheet, "--json"]
|
|
54
|
+
if embed or not open_browser:
|
|
55
|
+
args.append("--no-open")
|
|
56
|
+
if port is not None:
|
|
57
|
+
args += ["--port", str(port)]
|
|
58
|
+
|
|
59
|
+
failure: Optional[Event] = None
|
|
60
|
+
started = False
|
|
61
|
+
ended: Optional[Event] = None
|
|
62
|
+
try:
|
|
63
|
+
for event in stream_events(find_cli(cli), args, env):
|
|
64
|
+
if event.get("type") == "error":
|
|
65
|
+
failure = event
|
|
66
|
+
elif event.get("type") == "ui":
|
|
67
|
+
started = True
|
|
68
|
+
if frame_view is not None:
|
|
69
|
+
frame_view.show(str(event.get("url")))
|
|
70
|
+
elif event.get("type") == "exit":
|
|
71
|
+
ended = event
|
|
72
|
+
if printer is not None:
|
|
73
|
+
printer.show(event)
|
|
74
|
+
if on_event is not None and event.get("type") != "exit":
|
|
75
|
+
on_event(event)
|
|
76
|
+
except KeyboardInterrupt:
|
|
77
|
+
# Interrupting is how a notebook says "done": the server has stopped, the sheet is what it was
|
|
78
|
+
pass
|
|
79
|
+
finally:
|
|
80
|
+
if printer is not None:
|
|
81
|
+
printer.close()
|
|
82
|
+
if frame_view is not None:
|
|
83
|
+
frame_view.close()
|
|
84
|
+
if failure is not None:
|
|
85
|
+
raise BatchgridError(str(failure.get("code")), str(failure.get("message")))
|
|
86
|
+
if not started and ended is not None and ended.get("code"):
|
|
87
|
+
stderr = str(ended.get("stderr") or "")
|
|
88
|
+
hint = " The installed batchgrid CLI is too old for ui() - update it." if "unknown" in stderr else ""
|
|
89
|
+
raise BatchgridError("cli_failed", f"batchgrid ui did not start.{hint}\n{stderr}".rstrip(), stderr)
|
|
90
|
+
return _read_sheet(frame, Path(sheet), Path(path) if path else None)
|
|
91
|
+
finally:
|
|
92
|
+
out.cleanup()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _read_sheet(frame: Any, sheet: Path, sent: Optional[Path]) -> Optional["pd.DataFrame"]:
|
|
96
|
+
"""The sheet as the tab last showed it.
|
|
97
|
+
|
|
98
|
+
For a DataFrame passed in, a copy of it: new columns are added, a column
|
|
99
|
+
edited in the tab takes the edited values, and a column left alone keeps
|
|
100
|
+
its dtype. When the tab reshaped the sheet, it comes back as it was shown.
|
|
101
|
+
"""
|
|
102
|
+
if not sheet.exists():
|
|
103
|
+
return None
|
|
104
|
+
try:
|
|
105
|
+
import pandas as pd
|
|
106
|
+
except ImportError:
|
|
107
|
+
return None
|
|
108
|
+
if frame is None or not _is_dataframe(frame) or sent is None:
|
|
109
|
+
return pd.read_csv(sheet, dtype=str, keep_default_na=False)
|
|
110
|
+
|
|
111
|
+
merged = _merge_output(frame, sheet)
|
|
112
|
+
shown = pd.read_csv(sheet, dtype=str, keep_default_na=False)
|
|
113
|
+
if merged is None or len(shown) != len(frame) or list(merged.columns[: len(frame.columns)]) != [str(c) for c in frame.columns]:
|
|
114
|
+
return merged
|
|
115
|
+
# What the tab was given, as text - the same text it would hand back for an untouched cell
|
|
116
|
+
given = pd.read_csv(sent, dtype=str, keep_default_na=False)
|
|
117
|
+
for column in given.columns:
|
|
118
|
+
if column in shown.columns and not shown[column].equals(given[column]):
|
|
119
|
+
merged[column] = shown[column].to_numpy()
|
|
120
|
+
return merged
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _in_notebook() -> bool:
|
|
124
|
+
"""Whether this runs in a Jupyter kernel, whose front end can show the workspace in a cell"""
|
|
125
|
+
try:
|
|
126
|
+
from IPython import get_ipython
|
|
127
|
+
|
|
128
|
+
shell = get_ipython()
|
|
129
|
+
return shell is not None and "IPKernelApp" in shell.config
|
|
130
|
+
except Exception: # noqa: BLE001 - no IPython means a plain console
|
|
131
|
+
return False
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class _Frame:
|
|
135
|
+
"""The workspace in the cell's output; once it stops, a note takes its place"""
|
|
136
|
+
|
|
137
|
+
def __init__(self, height: int) -> None:
|
|
138
|
+
self.height = height
|
|
139
|
+
self.handle: Any = None
|
|
140
|
+
|
|
141
|
+
def show(self, url: str) -> None:
|
|
142
|
+
from IPython.display import IFrame, display
|
|
143
|
+
|
|
144
|
+
self.handle = display(
|
|
145
|
+
IFrame(
|
|
146
|
+
url,
|
|
147
|
+
width="100%",
|
|
148
|
+
height=self.height,
|
|
149
|
+
extras=[
|
|
150
|
+
'style="border: 1px solid rgba(128, 128, 128, 0.35); border-radius: 6px"',
|
|
151
|
+
'allow="clipboard-read; clipboard-write"',
|
|
152
|
+
],
|
|
153
|
+
),
|
|
154
|
+
display_id=True,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
def close(self) -> None:
|
|
158
|
+
if self.handle is None:
|
|
159
|
+
return
|
|
160
|
+
from IPython.display import HTML
|
|
161
|
+
|
|
162
|
+
# The server behind the frame is gone, so a saved notebook keeps no dead workspace
|
|
163
|
+
self.handle.update(HTML("<em>batchgrid closed. Run the cell again to reopen it.</em>"))
|
|
164
|
+
self.handle = None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class _Printer:
|
|
168
|
+
"""The terminal's log, in a notebook cell or a Python console"""
|
|
169
|
+
|
|
170
|
+
def __init__(self, link: bool = True) -> None:
|
|
171
|
+
self.bar: Any = None
|
|
172
|
+
self.link = link
|
|
173
|
+
try:
|
|
174
|
+
from tqdm.auto import tqdm
|
|
175
|
+
|
|
176
|
+
self._tqdm: Optional[Callable[..., Any]] = tqdm
|
|
177
|
+
except ImportError:
|
|
178
|
+
self._tqdm = None
|
|
179
|
+
|
|
180
|
+
def show(self, event: Event) -> None:
|
|
181
|
+
kind = event.get("type")
|
|
182
|
+
if kind == "progress":
|
|
183
|
+
self._progress(event)
|
|
184
|
+
return
|
|
185
|
+
# Anything else ends the run's bar, so its lines are not drawn over it
|
|
186
|
+
if kind in ("result", "closed", "user", "plan"):
|
|
187
|
+
self.close()
|
|
188
|
+
if kind == "ui" and self.link:
|
|
189
|
+
self._link(str(event.get("url")))
|
|
190
|
+
elif kind == "user":
|
|
191
|
+
self._line(f"\n❯ {event.get('text')}")
|
|
192
|
+
elif kind == "trace":
|
|
193
|
+
mark = {"error": "✖", "stopped": "⊘"}.get(str(event.get("status")), "✔")
|
|
194
|
+
detail = f" - {event['detail']}" if event.get("detail") else ""
|
|
195
|
+
self._line(f" {mark} {event.get('text')}{detail}")
|
|
196
|
+
elif kind == "message":
|
|
197
|
+
self._line(f" {event.get('text')}")
|
|
198
|
+
elif kind == "plan":
|
|
199
|
+
self._line(f"\n{event.get('summary')}\n")
|
|
200
|
+
if event.get("costFormatted"):
|
|
201
|
+
self._line(str(event.get("costFormatted")))
|
|
202
|
+
elif kind == "result":
|
|
203
|
+
mark = {"done": "✔", "stopped": "⊘"}.get(str(event.get("status")), "✖")
|
|
204
|
+
self._line(f" {mark} {event.get('success')}/{event.get('total')} rows, {event.get('failed')} failed")
|
|
205
|
+
elif kind == "sheet":
|
|
206
|
+
columns = event.get("columns") or []
|
|
207
|
+
self._line(f" ↳ Sheet synced: {event.get('rows')} rows × {len(columns)} columns") # type: ignore[arg-type]
|
|
208
|
+
elif kind == "browser_error":
|
|
209
|
+
self._line(f" ✖ Browser: {event.get('message')}")
|
|
210
|
+
elif kind == "error":
|
|
211
|
+
self._line(f"✖ {event.get('message')}")
|
|
212
|
+
elif kind == "closed":
|
|
213
|
+
why = {"tab_closed": "The tab was closed", "finished": "Finished in the browser" if self.link else "Finished"}.get(
|
|
214
|
+
str(event.get("reason")), "Stopped"
|
|
215
|
+
)
|
|
216
|
+
self._line(f"\n{why}.")
|
|
217
|
+
|
|
218
|
+
def _progress(self, event: Event) -> None:
|
|
219
|
+
if self._tqdm is None:
|
|
220
|
+
return
|
|
221
|
+
total = int(event.get("total") or 0) # type: ignore[arg-type]
|
|
222
|
+
if self.bar is None:
|
|
223
|
+
self.bar = self._tqdm(total=total, unit="row", desc="batchgrid")
|
|
224
|
+
if total and self.bar.total != total:
|
|
225
|
+
self.bar.total = total
|
|
226
|
+
self.bar.n = int(event.get("completed") or 0) # type: ignore[arg-type]
|
|
227
|
+
self.bar.set_postfix(failed=int(event.get("failed") or 0), refresh=False) # type: ignore[arg-type]
|
|
228
|
+
self.bar.refresh()
|
|
229
|
+
|
|
230
|
+
def _link(self, url: str) -> None:
|
|
231
|
+
if _in_notebook():
|
|
232
|
+
from IPython.display import HTML, display
|
|
233
|
+
|
|
234
|
+
display(HTML(f'batchgrid is open at <a href="{url}" target="_blank">{url.split("?")[0]}</a>'))
|
|
235
|
+
return
|
|
236
|
+
self._line(f"batchgrid is open at {url}")
|
|
237
|
+
|
|
238
|
+
def _line(self, text: str) -> None:
|
|
239
|
+
print(text, flush=True, file=sys.stdout)
|
|
240
|
+
|
|
241
|
+
def close(self) -> None:
|
|
242
|
+
if self.bar is not None:
|
|
243
|
+
self.bar.close()
|
|
244
|
+
self.bar = None
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""ui() against the real CLI server, with this test standing in for the browser tab.
|
|
2
|
+
|
|
3
|
+
The tab's side is plain HTTP - the same requests the web workspace sends - so
|
|
4
|
+
the whole path from Python through the CLI and back is exercised without a
|
|
5
|
+
browser: the link, the log, the synced sheet, and each way of finishing.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import _thread
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import stat
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import types
|
|
18
|
+
import urllib.request
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Dict, List
|
|
21
|
+
|
|
22
|
+
import pandas as pd
|
|
23
|
+
import pytest
|
|
24
|
+
|
|
25
|
+
import batchgrid
|
|
26
|
+
from batchgrid._cli import _node_cli_on_path
|
|
27
|
+
|
|
28
|
+
pytestmark = pytest.mark.usefixtures("cli_env")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class FakeTab:
|
|
32
|
+
"""What the web workspace does once `batchgrid ui` opens it"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, url: str) -> None:
|
|
35
|
+
self.url = url
|
|
36
|
+
self.base = url.split("/?")[0]
|
|
37
|
+
self.token = url.split("token=")[1]
|
|
38
|
+
self.stream: Any = None
|
|
39
|
+
|
|
40
|
+
def call(self, path: str, body: Any = None) -> Any:
|
|
41
|
+
data = None if body is None else json.dumps(body).encode()
|
|
42
|
+
request = urllib.request.Request(
|
|
43
|
+
f"{self.base}/__batchgrid/{path}",
|
|
44
|
+
data=data,
|
|
45
|
+
method="POST" if data is not None or path == "finish" else "GET",
|
|
46
|
+
headers={"x-batchgrid-token": self.token, "Content-Type": "application/json"},
|
|
47
|
+
)
|
|
48
|
+
with urllib.request.urlopen(request, timeout=10) as response:
|
|
49
|
+
return response.read()
|
|
50
|
+
|
|
51
|
+
def connect(self) -> None:
|
|
52
|
+
self.stream = urllib.request.urlopen(f"{self.base}/__batchgrid/events?token={self.token}", timeout=10)
|
|
53
|
+
|
|
54
|
+
def run_classification(self, sheet_csv: str, rows: int, columns: List[str]) -> None:
|
|
55
|
+
plan = {"title": "Classify", "intent": "Label", "sourceColumns": ["review"], "steps": [], "output": {"format": "csv"}}
|
|
56
|
+
summary = {"status": "done", "total": rows, "success": rows, "failed": 0, "tokens": {"input": 1, "output": 1}, "durationMs": 5}
|
|
57
|
+
self.call("log", {"entries": [{"id": "u1", "kind": "user", "text": "classify each review", "at": 0}]})
|
|
58
|
+
self.call("log", {"entries": [{"id": "p1", "kind": "plan", "plan": plan}], "plan": {"costFormatted": "~$0.01", "cost": None, "rows": rows, "missingSecrets": []}})
|
|
59
|
+
self.call("log", {"progress": {"completed": 1, "total": rows, "failed": 0, "elapsed": 100}})
|
|
60
|
+
self.call("log", {"entries": [{"id": "r1", "kind": "run", "plan": plan, "summary": summary}]})
|
|
61
|
+
self.call("sheet", {"csv": sheet_csv, "rows": rows, "columns": columns})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def open_tab(act) -> Any:
|
|
65
|
+
"""An on_event hook that opens a fake tab once the link is printed, then runs `act` on it"""
|
|
66
|
+
|
|
67
|
+
def on_event(event: Dict[str, Any]) -> None:
|
|
68
|
+
if event["type"] == "ui":
|
|
69
|
+
tab = FakeTab(event["url"])
|
|
70
|
+
tab.connect()
|
|
71
|
+
threading.Thread(target=act, args=(tab,), daemon=True).start()
|
|
72
|
+
|
|
73
|
+
return on_event
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_the_sheet_comes_back_with_new_columns_and_edits(capsys: pytest.CaptureFixture[str]) -> None:
|
|
77
|
+
df = pd.DataFrame({"review": ["Great", "Bad", "Okay"], "stars": [5, 1, 3], "note": ["a", "b", "c"]}, index=["x", "y", "z"])
|
|
78
|
+
|
|
79
|
+
def act(tab: FakeTab) -> None:
|
|
80
|
+
# The run added a column, and the user fixed a note by hand
|
|
81
|
+
tab.run_classification(
|
|
82
|
+
"review,stars,note,sentiment\nGreat,5,a,positive\nBad,1,B!,negative\nOkay,3,c,positive\n",
|
|
83
|
+
3,
|
|
84
|
+
["review", "stars", "note", "sentiment"],
|
|
85
|
+
)
|
|
86
|
+
tab.call("finish", {})
|
|
87
|
+
|
|
88
|
+
result = batchgrid.ui(df, open_browser=False, on_event=open_tab(act))
|
|
89
|
+
|
|
90
|
+
assert result is not None
|
|
91
|
+
assert list(result.columns) == ["review", "stars", "note", "sentiment"]
|
|
92
|
+
assert list(result.index) == ["x", "y", "z"]
|
|
93
|
+
assert result["stars"].dtype == df["stars"].dtype
|
|
94
|
+
assert result.loc["y", "note"] == "B!"
|
|
95
|
+
assert result.loc["y", "sentiment"] == "negative"
|
|
96
|
+
|
|
97
|
+
printed = capsys.readouterr().out
|
|
98
|
+
assert "batchgrid is open at http://127.0.0.1:" in printed
|
|
99
|
+
assert "❯ classify each review" in printed
|
|
100
|
+
assert "~$0.01" in printed
|
|
101
|
+
assert "✔ 3/3 rows, 0 failed" in printed
|
|
102
|
+
assert "Sheet synced: 3 rows × 4 columns" in printed
|
|
103
|
+
assert "Finished in the browser." in printed
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_closing_the_tab_returns_what_it_last_synced(reviews_csv: Path) -> None:
|
|
107
|
+
def act(tab: FakeTab) -> None:
|
|
108
|
+
tab.call("sheet", {"csv": "review,len\nGreat product,13\n", "rows": 1, "columns": ["review", "len"]})
|
|
109
|
+
tab.stream.close()
|
|
110
|
+
|
|
111
|
+
result = batchgrid.ui(reviews_csv, open_browser=False, log=False, on_event=open_tab(act))
|
|
112
|
+
|
|
113
|
+
# A file in, a sheet out - as shown, since the tab reshaped it
|
|
114
|
+
assert result is not None
|
|
115
|
+
assert result.to_dict("records") == [{"review": "Great product", "len": "13"}]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@pytest.mark.skipif(sys.platform == "win32", reason="SIGINT forwarding is POSIX-only")
|
|
119
|
+
def test_interrupting_the_cell_returns_instead_of_raising() -> None:
|
|
120
|
+
df = pd.DataFrame({"review": ["Great"]})
|
|
121
|
+
|
|
122
|
+
def act(tab: FakeTab) -> None:
|
|
123
|
+
tab.call("sheet", {"csv": "review,sentiment\nGreat,positive\n", "rows": 1, "columns": ["review", "sentiment"]})
|
|
124
|
+
time.sleep(0.3)
|
|
125
|
+
_thread.interrupt_main()
|
|
126
|
+
|
|
127
|
+
result = batchgrid.ui(df, open_browser=False, log=False, on_event=open_tab(act))
|
|
128
|
+
|
|
129
|
+
assert result is not None
|
|
130
|
+
assert result.loc[0, "sentiment"] == "positive"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_in_a_notebook_the_workspace_shows_in_the_cell(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
|
134
|
+
shown: List[Any] = []
|
|
135
|
+
|
|
136
|
+
class Handle:
|
|
137
|
+
def update(self, obj: Any) -> None:
|
|
138
|
+
shown.append(obj)
|
|
139
|
+
|
|
140
|
+
def display(obj: Any, display_id: bool = False) -> Handle:
|
|
141
|
+
shown.append(obj)
|
|
142
|
+
return Handle()
|
|
143
|
+
|
|
144
|
+
# A stand-in for the IPython a Jupyter kernel runs
|
|
145
|
+
ipython = types.ModuleType("IPython")
|
|
146
|
+
ipython.get_ipython = lambda: types.SimpleNamespace(config={"IPKernelApp": {}}) # type: ignore[attr-defined]
|
|
147
|
+
ipython_display = types.ModuleType("IPython.display")
|
|
148
|
+
ipython_display.HTML = lambda html: html # type: ignore[attr-defined]
|
|
149
|
+
ipython_display.IFrame = lambda src, **options: {"src": src, **options} # type: ignore[attr-defined]
|
|
150
|
+
ipython_display.display = display # type: ignore[attr-defined]
|
|
151
|
+
monkeypatch.setitem(sys.modules, "IPython", ipython)
|
|
152
|
+
monkeypatch.setitem(sys.modules, "IPython.display", ipython_display)
|
|
153
|
+
|
|
154
|
+
urls: List[str] = []
|
|
155
|
+
|
|
156
|
+
def act(tab: FakeTab) -> None:
|
|
157
|
+
urls.append(tab.url)
|
|
158
|
+
tab.call("finish", {})
|
|
159
|
+
|
|
160
|
+
# No open_browser=False: showing it in the cell never opens a tab
|
|
161
|
+
batchgrid.ui(on_event=open_tab(act))
|
|
162
|
+
|
|
163
|
+
assert len(shown) == 2
|
|
164
|
+
assert shown[0]["src"] == urls[0] and shown[0]["height"] == 720
|
|
165
|
+
# The server behind it stopped, so a note takes the workspace's place
|
|
166
|
+
assert "batchgrid closed" in shown[1]
|
|
167
|
+
printed = capsys.readouterr().out
|
|
168
|
+
assert "is open at" not in printed
|
|
169
|
+
assert "Finished." in printed
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def test_nothing_synced_returns_none() -> None:
|
|
173
|
+
result = batchgrid.ui(open_browser=False, log=False, on_event=open_tab(lambda tab: tab.call("finish", {})))
|
|
174
|
+
assert result is None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def test_the_pip_command_does_not_find_itself(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
178
|
+
shim_dir, node_dir = tmp_path / "shim", tmp_path / "node"
|
|
179
|
+
shim_dir.mkdir()
|
|
180
|
+
node_dir.mkdir()
|
|
181
|
+
for directory, first_line in ((shim_dir, "#!/usr/bin/env python3"), (node_dir, "#!/usr/bin/env node")):
|
|
182
|
+
script = directory / "batchgrid"
|
|
183
|
+
script.write_text(f"{first_line}\n", encoding="utf-8")
|
|
184
|
+
script.chmod(script.stat().st_mode | stat.S_IEXEC)
|
|
185
|
+
monkeypatch.setenv("PATH", os.pathsep.join([str(shim_dir), str(node_dir)]))
|
|
186
|
+
|
|
187
|
+
assert _node_cli_on_path() == str(node_dir / "batchgrid")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def test_a_cli_without_ui_says_so() -> None:
|
|
191
|
+
# A stand-in for an old CLI: it knows no `ui` command
|
|
192
|
+
old_cli = [sys.executable, "-c", "import sys; print(\"error: unknown command 'ui'\", file=sys.stderr); sys.exit(1)"]
|
|
193
|
+
with pytest.raises(batchgrid.BatchgridError) as caught:
|
|
194
|
+
batchgrid.ui(open_browser=False, log=False, cli=old_cli)
|
|
195
|
+
assert caught.value.code == "cli_failed"
|
|
196
|
+
assert "too old" in caught.value.message
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|