resutil 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- resutil-0.2.3/.flake8 +3 -0
- {resutil-0.2.2 → resutil-0.2.3}/.github/workflows/publish.yaml +11 -0
- resutil-0.2.3/.github/workflows/tests.yaml +22 -0
- {resutil-0.2.2 → resutil-0.2.3}/CLAUDE.md +6 -1
- {resutil-0.2.2 → resutil-0.2.3}/PKG-INFO +28 -6
- {resutil-0.2.2 → resutil-0.2.3}/README.ja.md +25 -4
- {resutil-0.2.2 → resutil-0.2.3}/README.md +26 -5
- resutil-0.2.3/docs/releases/v0.2.3.md +34 -0
- resutil-0.2.3/docs/review-fixes.md +43 -0
- resutil-0.2.3/docs/server-login-state.patch +69 -0
- {resutil-0.2.2 → resutil-0.2.3}/migration-to-server.md +5 -5
- {resutil-0.2.2 → resutil-0.2.3}/pyproject.toml +14 -1
- resutil-0.2.3/src/resutil/__init__.py +3 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/cli/__init__.py +2 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/cli/cli_main.py +127 -149
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/cli/interactive.py +150 -102
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/config_file.py +36 -15
- resutil-0.2.3/src/resutil/console.py +20 -0
- resutil-0.2.3/src/resutil/core.py +286 -0
- resutil-0.2.3/src/resutil/ex_dir.py +164 -0
- resutil-0.2.3/src/resutil/exp_file.py +17 -0
- resutil-0.2.3/src/resutil/git.py +61 -0
- resutil-0.2.3/src/resutil/main.py +171 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/storage/__init__.py +3 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/storage/gcs/gcs.py +17 -10
- resutil-0.2.3/src/resutil/storage/gdrive/gdrive.py +151 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/storage/server/server.py +63 -13
- resutil-0.2.3/src/resutil/storage/storage.py +26 -0
- resutil-0.2.3/src/resutil/sync.py +74 -0
- resutil-0.2.3/src/resutil/utils.py +143 -0
- resutil-0.2.3/tests/conftest.py +66 -0
- resutil-0.2.3/tests/test_cli_smoke.py +84 -0
- resutil-0.2.3/tests/test_cli_tui.py +126 -0
- resutil-0.2.3/tests/test_config_git.py +70 -0
- resutil-0.2.3/tests/test_core.py +144 -0
- resutil-0.2.3/tests/test_ex_dir.py +66 -0
- resutil-0.2.3/tests/test_main.py +127 -0
- resutil-0.2.3/tests/test_storage.py +139 -0
- resutil-0.2.3/tests/test_utils.py +61 -0
- {resutil-0.2.2 → resutil-0.2.3}/uv.lock +25 -3
- resutil-0.2.2/src/resutil/__init__.py +0 -6
- resutil-0.2.2/src/resutil/core.py +0 -180
- resutil-0.2.2/src/resutil/ex_dir.py +0 -75
- resutil-0.2.2/src/resutil/exp_file.py +0 -15
- resutil-0.2.2/src/resutil/git.py +0 -48
- resutil-0.2.2/src/resutil/main.py +0 -229
- resutil-0.2.2/src/resutil/storage/gdrive/gdrive.py +0 -151
- resutil-0.2.2/src/resutil/storage/storage.py +0 -15
- resutil-0.2.2/src/resutil/utils.py +0 -86
- resutil-0.2.2/tests/test_ex_dir.py +0 -28
- resutil-0.2.2/tests/test_utils.py +0 -49
- {resutil-0.2.2 → resutil-0.2.3}/.gitignore +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/.python-version +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/.vscode/launch.json +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/.vscode/settings.json +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/LICENSE +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/results/.gitignore +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/sample.py +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/cli/__main__.py +0 -0
- {resutil-0.2.2 → resutil-0.2.3}/src/resutil/storage/server/__init__.py +0 -0
resutil-0.2.3/.flake8
ADDED
|
@@ -16,9 +16,20 @@ jobs:
|
|
|
16
16
|
- name: Set up uv
|
|
17
17
|
uses: astral-sh/setup-uv@v7
|
|
18
18
|
|
|
19
|
+
- name: Install and verify
|
|
20
|
+
run: |
|
|
21
|
+
uv sync --locked
|
|
22
|
+
uv run pytest -q
|
|
23
|
+
uv run black --check src tests
|
|
24
|
+
uv run flake8 src tests
|
|
25
|
+
uv run mypy src
|
|
26
|
+
|
|
19
27
|
- name: Build (wheel and sdist)
|
|
20
28
|
run: uv build --no-sources
|
|
21
29
|
|
|
30
|
+
- name: Validate distribution metadata
|
|
31
|
+
run: uvx --from twine==6.2.0 twine check dist/*
|
|
32
|
+
|
|
22
33
|
- name: Save build artifacts
|
|
23
34
|
uses: actions/upload-artifact@v4
|
|
24
35
|
with:
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
pull_request:
|
|
5
|
+
jobs:
|
|
6
|
+
test:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
os: [ubuntu-latest, windows-latest]
|
|
12
|
+
python: ["3.10", "3.12"]
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v5
|
|
15
|
+
- uses: astral-sh/setup-uv@v7
|
|
16
|
+
with:
|
|
17
|
+
python-version: ${{ matrix.python }}
|
|
18
|
+
- run: uv sync --locked
|
|
19
|
+
- run: uv run pytest -q
|
|
20
|
+
- run: uv run black --check src tests
|
|
21
|
+
- run: uv run flake8 src tests
|
|
22
|
+
- run: uv run mypy src
|
|
@@ -51,5 +51,10 @@ All backends implement the `Storage` base class (`src/resutil/storage/storage.py
|
|
|
51
51
|
|
|
52
52
|
- `RESUTIL_COMMENT` - Pre-set experiment comment (skips prompt)
|
|
53
53
|
- `RESUTIL_NO_INTERACTIVE` - Non-interactive mode (no user prompts)
|
|
54
|
-
- `
|
|
54
|
+
- `RESUTIL_NO_REMOTE` - Suppress cloud upload (legacy RESUTIIL_REMOTE is also accepted)
|
|
55
55
|
- `RESUTIL_DEBUG` - Use temp directory, skip upload
|
|
56
|
+
|
|
57
|
+
- `RESUTIL_TIMEZONE` - IANA time zone (default: local); metadata includes the UTC offset
|
|
58
|
+
- `RESUTIL_STORAGE_TYPE`, `RESUTIL_GCS_BUCKET_NAME`, `RESUTIL_GCS_KEY_FILE_PATH` - Config overrides
|
|
59
|
+
|
|
60
|
+
Reliability and server-login compatibility are documented in docs/review-fixes.md.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: resutil
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Resutil is a utility to manage experimental result data obtained from Python projects. It also manages dependency such as codes and input data with result data. Data is synced to the cloud for team sharing and collaboration.
|
|
5
5
|
Project-URL: Homepage, https://github.com/KatayamaLab/resutil
|
|
6
6
|
Project-URL: Issues, https://github.com/KatayamaLab/resutil/issues
|
|
@@ -40,6 +40,7 @@ Requires-Dist: pyreadline3>=3.5.4; platform_system == 'Windows'
|
|
|
40
40
|
Requires-Dist: pyyaml>=6.0.0
|
|
41
41
|
Requires-Dist: rich>=13.7.0
|
|
42
42
|
Requires-Dist: textual>=8.2.1
|
|
43
|
+
Requires-Dist: tzdata>=2024.1; sys_platform == 'win32'
|
|
43
44
|
Description-Content-Type: text/markdown
|
|
44
45
|
|
|
45
46
|
# Resutil
|
|
@@ -204,9 +205,9 @@ storage_config:
|
|
|
204
205
|
|
|
205
206
|
### `resutil push`
|
|
206
207
|
|
|
207
|
-
The `resutil push` command
|
|
208
|
+
The `resutil push EXPERIMENT` command uploads a local experiment, replacing its remote archive if it exists. `resutil push --all` uploads new or locally modified experiments.
|
|
208
209
|
|
|
209
|
-
`resutil push
|
|
210
|
+
`resutil push EXPERIMENT` uploads the specified experiment and any new or modified local dependencies listed in `resutil-exp.yaml`. Pass `--no-dependency` to upload only the specified experiment.
|
|
210
211
|
|
|
211
212
|
`resutil push --all` will upload all experimental data to the cloud.
|
|
212
213
|
|
|
@@ -216,7 +217,7 @@ This is useful for keeping your local data up-to-date with the data stored in th
|
|
|
216
217
|
|
|
217
218
|
The `resutil pull` command is used to download a specific experimental data from the cloud that does not exist in the local result directory.
|
|
218
219
|
|
|
219
|
-
`resutil pull [exp_name]` downloads experiment directory from cloud. Depending directorys included in `exp
|
|
220
|
+
`resutil pull [exp_name]` downloads experiment directory from cloud. Depending directorys included in `resutil-exp.yaml` are automtically downlowded. `--no-dependency` option restrain automatic dependency download.
|
|
220
221
|
|
|
221
222
|
`resutil pull --all` will download all experimental data from the cloud that is not currently in your local result directory.
|
|
222
223
|
|
|
@@ -240,7 +241,7 @@ The `resutil rm` command removes experiments. You can use it as follows: resutil
|
|
|
240
241
|
|
|
241
242
|
### `resutil comment` **EXPERIMENTAL**
|
|
242
243
|
|
|
243
|
-
`resutil comment [EXPERIMENT] [COMMENT]` adds or modifies a comment following the timestamp in the experiment name. Both local and cloud experiment names will change if they exist. Note that Resutil regards a different experiment name as a different experiment, so this does not affect the name of the same experiment other users have already pulled.
|
|
244
|
+
`resutil comment [EXPERIMENT] [COMMENT]` adds or modifies a comment following the timestamp in the experiment name. Both local and cloud experiment names will change if they exist. Dependency references and the experiment metadata are updated; remote archives are inspected to find references. Note that Resutil regards a different experiment name as a different experiment, so this does not affect the name of the same experiment other users have already pulled.
|
|
244
245
|
|
|
245
246
|
### `resutil login`
|
|
246
247
|
|
|
@@ -262,7 +263,7 @@ When running code that integrates Resutil, you can use the following environment
|
|
|
262
263
|
|
|
263
264
|
`RESUTIL_NO_INTERACTIVE` Enables non-interactive mode. This prevents any user prompts during execution. This is useful when running as a batch job. If `RESUTIL_COMMENT` is not specified, no comment will be added to the experiment directory.
|
|
264
265
|
|
|
265
|
-
`
|
|
266
|
+
`RESUTIL_NO_REMOTE` Suppresses uploading results to cloud storage.
|
|
266
267
|
|
|
267
268
|
`RESUTIL_DEBUG` Enables debug mode where a temporary directory is used as experiment directory. The temporary directory will not be uploaded to cloud storage.
|
|
268
269
|
|
|
@@ -320,3 +321,24 @@ dependency: Dependencies (automatically extracted from directories in the comman
|
|
|
320
321
|
## Bug Fixed
|
|
321
322
|
- Fix a bug resuitl does not works properly when git is not initialized
|
|
322
323
|
```
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
### Reliability and configuration
|
|
327
|
+
|
|
328
|
+
- `resutil rm` asks for confirmation; pass `--yes` for scripts. Without `--local`
|
|
329
|
+
or `--remote`, both copies are deleted.
|
|
330
|
+
- `RESUTIL_NO_REMOTE` disables storage authentication and uploads, including checkpoints.
|
|
331
|
+
The old spelling `RESUTIIL_REMOTE` is accepted as an alias.
|
|
332
|
+
- `RESUTIL_DEBUG` uses a unique temporary directory, cleans it up after execution,
|
|
333
|
+
and skips storage authentication and uploads.
|
|
334
|
+
- `RESUTIL_TIMEZONE=Asia/Tokyo` selects an IANA time zone. Metadata records the UTC offset.
|
|
335
|
+
- `RESUTIL_STORAGE_TYPE`, `RESUTIL_GCS_BUCKET_NAME` and `RESUTIL_GCS_KEY_FILE_PATH`
|
|
336
|
+
override the corresponding storage settings.
|
|
337
|
+
- Configuration paths resolve relative to `resutil-conf.yaml`; the working directory is preserved.
|
|
338
|
+
- Comments accept up to 200 UTF-8 bytes. Directory-name collisions receive a unique suffix.
|
|
339
|
+
- `params.save_checkpoint()` returns `False` when disabled or failed. It does not interrupt
|
|
340
|
+
computation. A failed final upload raises an error; retry with `resutil push EXPERIMENT`.
|
|
341
|
+
- New or modified local data is detected using `results/.resutil-sync.json`.
|
|
342
|
+
Missing receipts trigger a conservative upload; remote-only edits are not compared.
|
|
343
|
+
- The server login endpoint must echo the login `state` through its sign-in page
|
|
344
|
+
and callback. See [server compatibility](docs/review-fixes.md) and the included server patch.
|
|
@@ -158,9 +158,9 @@ storage_config:
|
|
|
158
158
|
|
|
159
159
|
### `resutil push`
|
|
160
160
|
|
|
161
|
-
`resutil push
|
|
161
|
+
`resutil push 実験名` はローカルの実験をアップロードし、同名のリモート実験があれば上書きします。`resutil push --all` は未送信またはローカルで変更された実験を送信します。
|
|
162
162
|
|
|
163
|
-
`resutil push [exp_name]`は、特定のディレクトリの実験データをクラウドにアップロードします。`exp
|
|
163
|
+
`resutil push [exp_name]`は、特定のディレクトリの実験データをクラウドにアップロードします。`resutil-exp.yaml`に含まれる依存ディレクトリは自動的にアップロードされます。`--no-dependency`オプションを使用すると、自動依存アップロードが制限されます。
|
|
164
164
|
|
|
165
165
|
`resutil push --all`は、すべての実験データをクラウドにアップロードします。
|
|
166
166
|
|
|
@@ -168,7 +168,7 @@ storage_config:
|
|
|
168
168
|
|
|
169
169
|
`resutil pull`コマンドは、ローカルの結果ディレクトリに存在しない特定の実験データをクラウドからダウンロードするために使用されます。
|
|
170
170
|
|
|
171
|
-
`resutil pull [exp_name]`は、クラウドから実験ディレクトリをダウンロードします。`exp
|
|
171
|
+
`resutil pull [exp_name]`は、クラウドから実験ディレクトリをダウンロードします。`resutil-exp.yaml`に含まれる依存ディレクトリは自動的にダウンロードされます。`--no-dependency`オプションを使用すると、自動依存ダウンロードが制限されます。
|
|
172
172
|
|
|
173
173
|
`resutil pull --all`は、現在ローカルの結果ディレクトリに存在しないすべての実験データをクラウドからダウンロードします。
|
|
174
174
|
|
|
@@ -214,7 +214,7 @@ Resutilを組み込んだコードを実行する際に、以下の環境変数
|
|
|
214
214
|
|
|
215
215
|
`RESUTIL_NO_INTERACTIVE` 非インタラクティブモードにします。実行時にユーザーへ問い合わせをしなくなります。バッチジョブとして実行する場合に利用します。`RESUTIL_COMMENT` が指定されていない場合には実験ディレクトリにコメントは付与されません。
|
|
216
216
|
|
|
217
|
-
`
|
|
217
|
+
`RESUTIL_NO_REMOTE` クラウドストレージへのアップロードを抑制します。
|
|
218
218
|
|
|
219
219
|
`RESUTIL_DEBUG` デバッグモードを有効にします。一時ディレクトリが実験ディレクトリとして使用され、クラウドストレージへのアップロードは行われません。
|
|
220
220
|
|
|
@@ -265,3 +265,24 @@ dependency: Dependencies (automatically extracted from directories in the comman
|
|
|
265
265
|
1. `pyproject.toml` のバージョン番号を変更
|
|
266
266
|
2. `main` ブランチにマージ
|
|
267
267
|
3. `v*.*.*` タグを付与すると自動的にPyPIにデプロイされます
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
### 安全性と追加設定
|
|
271
|
+
|
|
272
|
+
- `resutil rm` は削除前に確認します。スクリプトでは `--yes` を指定できます。
|
|
273
|
+
- `RESUTIL_NO_REMOTE` は認証とチェックポイントを含むアップロードを抑制します。
|
|
274
|
+
旧表記 `RESUTIIL_REMOTE` も互換用に受け付けます。
|
|
275
|
+
- `RESUTIL_DEBUG` は実行ごとの一時ディレクトリを使用し、終了時に削除します。
|
|
276
|
+
- `RESUTIL_TIMEZONE=Asia/Tokyo` のようにIANAタイムゾーンを指定できます。
|
|
277
|
+
- `RESUTIL_STORAGE_TYPE`、`RESUTIL_GCS_BUCKET_NAME`、
|
|
278
|
+
`RESUTIL_GCS_KEY_FILE_PATH` で設定を上書きできます。
|
|
279
|
+
- 設定内の相対パスは設定ファイルを基準に解決し、作業ディレクトリは変更しません。
|
|
280
|
+
- コメントはUTF-8で200バイトまでです。同名ディレクトリの衝突時は一意な接尾辞を付けます。
|
|
281
|
+
- `save_checkpoint()` は無効時・失敗時に `False` を返し、実験を止めません。
|
|
282
|
+
最終アップロードの失敗は例外になり、`resutil push 実験名` で再送できます。
|
|
283
|
+
- 同期記録 `results/.resutil-sync.json` により、チェックポイント後の変更も検出します。
|
|
284
|
+
記録のない既存実験は一度再送します。別クライアントによるリモートだけの変更は比較しません。
|
|
285
|
+
- コメント変更では依存参照とメタデータも更新します。リモートの依存参照を調べるため、
|
|
286
|
+
リモートアーカイブの読み取りが必要です。
|
|
287
|
+
- 安全なログインには、サーバー側も `state` をサインイン画面とコールバックへ
|
|
288
|
+
引き継ぐ必要があります。[互換性の説明](docs/review-fixes.md) とサーバー用パッチを参照してください。
|
|
@@ -160,9 +160,9 @@ storage_config:
|
|
|
160
160
|
|
|
161
161
|
### `resutil push`
|
|
162
162
|
|
|
163
|
-
The `resutil push` command
|
|
163
|
+
The `resutil push EXPERIMENT` command uploads a local experiment, replacing its remote archive if it exists. `resutil push --all` uploads new or locally modified experiments.
|
|
164
164
|
|
|
165
|
-
`resutil push
|
|
165
|
+
`resutil push EXPERIMENT` uploads the specified experiment and any new or modified local dependencies listed in `resutil-exp.yaml`. Pass `--no-dependency` to upload only the specified experiment.
|
|
166
166
|
|
|
167
167
|
`resutil push --all` will upload all experimental data to the cloud.
|
|
168
168
|
|
|
@@ -172,7 +172,7 @@ This is useful for keeping your local data up-to-date with the data stored in th
|
|
|
172
172
|
|
|
173
173
|
The `resutil pull` command is used to download a specific experimental data from the cloud that does not exist in the local result directory.
|
|
174
174
|
|
|
175
|
-
`resutil pull [exp_name]` downloads experiment directory from cloud. Depending directorys included in `exp
|
|
175
|
+
`resutil pull [exp_name]` downloads experiment directory from cloud. Depending directorys included in `resutil-exp.yaml` are automtically downlowded. `--no-dependency` option restrain automatic dependency download.
|
|
176
176
|
|
|
177
177
|
`resutil pull --all` will download all experimental data from the cloud that is not currently in your local result directory.
|
|
178
178
|
|
|
@@ -196,7 +196,7 @@ The `resutil rm` command removes experiments. You can use it as follows: resutil
|
|
|
196
196
|
|
|
197
197
|
### `resutil comment` **EXPERIMENTAL**
|
|
198
198
|
|
|
199
|
-
`resutil comment [EXPERIMENT] [COMMENT]` adds or modifies a comment following the timestamp in the experiment name. Both local and cloud experiment names will change if they exist. Note that Resutil regards a different experiment name as a different experiment, so this does not affect the name of the same experiment other users have already pulled.
|
|
199
|
+
`resutil comment [EXPERIMENT] [COMMENT]` adds or modifies a comment following the timestamp in the experiment name. Both local and cloud experiment names will change if they exist. Dependency references and the experiment metadata are updated; remote archives are inspected to find references. Note that Resutil regards a different experiment name as a different experiment, so this does not affect the name of the same experiment other users have already pulled.
|
|
200
200
|
|
|
201
201
|
### `resutil login`
|
|
202
202
|
|
|
@@ -218,7 +218,7 @@ When running code that integrates Resutil, you can use the following environment
|
|
|
218
218
|
|
|
219
219
|
`RESUTIL_NO_INTERACTIVE` Enables non-interactive mode. This prevents any user prompts during execution. This is useful when running as a batch job. If `RESUTIL_COMMENT` is not specified, no comment will be added to the experiment directory.
|
|
220
220
|
|
|
221
|
-
`
|
|
221
|
+
`RESUTIL_NO_REMOTE` Suppresses uploading results to cloud storage.
|
|
222
222
|
|
|
223
223
|
`RESUTIL_DEBUG` Enables debug mode where a temporary directory is used as experiment directory. The temporary directory will not be uploaded to cloud storage.
|
|
224
224
|
|
|
@@ -276,3 +276,24 @@ dependency: Dependencies (automatically extracted from directories in the comman
|
|
|
276
276
|
## Bug Fixed
|
|
277
277
|
- Fix a bug resuitl does not works properly when git is not initialized
|
|
278
278
|
```
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
### Reliability and configuration
|
|
282
|
+
|
|
283
|
+
- `resutil rm` asks for confirmation; pass `--yes` for scripts. Without `--local`
|
|
284
|
+
or `--remote`, both copies are deleted.
|
|
285
|
+
- `RESUTIL_NO_REMOTE` disables storage authentication and uploads, including checkpoints.
|
|
286
|
+
The old spelling `RESUTIIL_REMOTE` is accepted as an alias.
|
|
287
|
+
- `RESUTIL_DEBUG` uses a unique temporary directory, cleans it up after execution,
|
|
288
|
+
and skips storage authentication and uploads.
|
|
289
|
+
- `RESUTIL_TIMEZONE=Asia/Tokyo` selects an IANA time zone. Metadata records the UTC offset.
|
|
290
|
+
- `RESUTIL_STORAGE_TYPE`, `RESUTIL_GCS_BUCKET_NAME` and `RESUTIL_GCS_KEY_FILE_PATH`
|
|
291
|
+
override the corresponding storage settings.
|
|
292
|
+
- Configuration paths resolve relative to `resutil-conf.yaml`; the working directory is preserved.
|
|
293
|
+
- Comments accept up to 200 UTF-8 bytes. Directory-name collisions receive a unique suffix.
|
|
294
|
+
- `params.save_checkpoint()` returns `False` when disabled or failed. It does not interrupt
|
|
295
|
+
computation. A failed final upload raises an error; retry with `resutil push EXPERIMENT`.
|
|
296
|
+
- New or modified local data is detected using `results/.resutil-sync.json`.
|
|
297
|
+
Missing receipts trigger a conservative upload; remote-only edits are not compared.
|
|
298
|
+
- The server login endpoint must echo the login `state` through its sign-in page
|
|
299
|
+
and callback. See [server compatibility](docs/review-fixes.md) and the included server patch.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
## Fixes
|
|
2
|
+
|
|
3
|
+
- Prevent missing local experiments from overwriting remote results with empty archives.
|
|
4
|
+
- Validate experiment names and deletion paths; require confirmation before deletion.
|
|
5
|
+
- Fix legacy dependency paths, nested and cyclic transfers, and hidden transfer errors.
|
|
6
|
+
- Detect modified results after checkpoints and retry failed final uploads.
|
|
7
|
+
- Preserve function return values, failure exit codes, and the working directory.
|
|
8
|
+
- Refresh server credentials during long runs, bind credentials to their server, and encode API paths.
|
|
9
|
+
- Validate login callback state and tolerate unrelated callback requests.
|
|
10
|
+
- Update dependency references when renaming comments; restore local metadata if renaming fails.
|
|
11
|
+
- Fix Google Drive pagination, folder selection, concurrent access, and archive updates.
|
|
12
|
+
- Avoid Unicode logging failures with legacy Windows output encodings.
|
|
13
|
+
- Handle Git repositories without commits and record untracked files and deletions.
|
|
14
|
+
- Improve TUI network-error handling, UTF-8 support, timezone selection, and directory-name collisions.
|
|
15
|
+
|
|
16
|
+
## Compatibility
|
|
17
|
+
|
|
18
|
+
- Server login requires a resutil-server deployment that preserves the login state.
|
|
19
|
+
Update the server before upgrading clients. Existing matching credentials can still be refreshed.
|
|
20
|
+
See `docs/server-login-state.patch` and `docs/review-fixes.md`.
|
|
21
|
+
- Comments are limited to 200 UTF-8 bytes.
|
|
22
|
+
- `resutil rm` requires confirmation; use `--yes` for scripts.
|
|
23
|
+
- `RESUTIL_NO_REMOTE` skips authentication and uploads; `RESUTIIL_REMOTE` remains an alias.
|
|
24
|
+
- Experiments without synchronization receipts are conservatively uploaded once.
|
|
25
|
+
- `RESUTIL_TIMEZONE` accepts an IANA zone such as `Asia/Tokyo`.
|
|
26
|
+
|
|
27
|
+
## Validation
|
|
28
|
+
|
|
29
|
+
70 regression tests; Black, Flake8, mypy, and package-build checks.
|
|
30
|
+
Added CI for Linux and Windows on Python 3.10 and 3.12.
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
pip install --upgrade resutil==0.2.3
|
|
34
|
+
```
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Review fixes and compatibility
|
|
2
|
+
|
|
3
|
+
- Transfers validate experiment names and directories before any destructive operation.
|
|
4
|
+
Local symlinks and archive traversal are rejected. Downloads are staged before publication.
|
|
5
|
+
- Legacy dependency paths are normalized on read; new metadata stores experiment identifiers.
|
|
6
|
+
The coordinator visits cyclic graphs once and observes worker exceptions.
|
|
7
|
+
- Successful transfers save a content receipt in results/.resutil-sync.json for each destination.
|
|
8
|
+
Missing receipts and modified data are treated as unsynced, including final results after a checkpoint.
|
|
9
|
+
Old experiments without receipts are conservatively uploaded once.
|
|
10
|
+
This tracks local changes; edits made by another client to the remote archive are not detected.
|
|
11
|
+
- Explicit push can overwrite an existing remote experiment. Bulk push and the TUI skip
|
|
12
|
+
experiments whose local content matches the last successful transfer.
|
|
13
|
+
- rm requires confirmation unless --yes is passed. --local works without storage credentials.
|
|
14
|
+
- comment retains directory renaming. It updates local metadata and remote archive references.
|
|
15
|
+
Because the remote API has no dependency index, this reads remote archives before renaming.
|
|
16
|
+
The original remote target remains available until updated references have been saved.
|
|
17
|
+
Other clients' existing local copies keep their previous names and metadata.
|
|
18
|
+
- User exceptions, interrupts and SystemExit retain their exit semantics after best-effort
|
|
19
|
+
result upload. Successful computation with failed final upload raises an error.
|
|
20
|
+
Checkpoint failure returns False and does not abort computation.
|
|
21
|
+
- NO_REMOTE and DEBUG skip storage authentication and all uploads. Debug output is held
|
|
22
|
+
in a unique temporary directory and removed when the wrapped call finishes.
|
|
23
|
+
- Configuration paths resolve relative to the config file; the process cwd is unchanged.
|
|
24
|
+
- Metadata, configuration and text files use UTF-8. Comments are limited to 200 UTF-8 bytes.
|
|
25
|
+
- RESUTIL_TIMEZONE accepts an IANA zone, such as Asia/Tokyo. The default is the local zone.
|
|
26
|
+
started_at includes the UTC offset; the alphabetical identifier uses UTC elapsed minutes.
|
|
27
|
+
- Git snapshots include untracked files and deletion records; a first commit is optional.
|
|
28
|
+
- Google Drive operations are serialized, lists are paginated and project folders are scoped
|
|
29
|
+
to the configured parent. Existing archives are updated in place rather than deleted first.
|
|
30
|
+
- Server credentials are bound to their originating URL. Expiring tokens are refreshed on
|
|
31
|
+
requests; a 401 triggers one refresh/retry. URL path components are encoded.
|
|
32
|
+
- Login callbacks require an unpredictable state and ignore unrelated requests until timeout.
|
|
33
|
+
The server must preserve the state parameter from /auth/login to /auth/signin and /callback.
|
|
34
|
+
Apply server-login-state.patch in the resutil-server checkout and deploy the updated server
|
|
35
|
+
before using the new login command. Older servers fail safely rather than accepting an
|
|
36
|
+
unbound callback. Existing matching credentials can still be used and refreshed.
|
|
37
|
+
- Tests use temporary directories, in-memory storage, mocked Google clients and HTTP transports.
|
|
38
|
+
Real cloud storage is not contacted.
|
|
39
|
+
|
|
40
|
+
Related issues: #20, #28, #29, #31, #33, #34, #37, #39, #41, #45.
|
|
41
|
+
The architecture-only request #15 is intentionally outside this bug-fix scope.
|
|
42
|
+
Windows readline dependency (#36), keyless server authentication (#44) and the TUI (#27)
|
|
43
|
+
already exist; their surrounding error handling and documentation are covered here.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
--- a/src/resutil_server/routers/auth.py
|
|
2
|
+
+++ b/src/resutil_server/routers/auth.py
|
|
3
|
+
@@ -1,3 +1,6 @@
|
|
4
|
+
+import json
|
|
5
|
+
+from urllib.parse import urlencode
|
|
6
|
+
+
|
|
7
|
+
from fastapi import APIRouter, Query, Request
|
|
8
|
+
from fastapi.responses import HTMLResponse
|
|
9
|
+
|
|
10
|
+
@@ -14,17 +17,19 @@
|
|
11
|
+
@router.get("/login")
|
|
12
|
+
async def login(
|
|
13
|
+
request: Request,
|
|
14
|
+
- port: int = Query(description="CLI のローカルサーバーポート"),
|
|
15
|
+
+ port: int = Query(description="CLI のローカルサーバーポート", ge=1, le=65535),
|
|
16
|
+
+ state: str = Query(default="", max_length=256, pattern=r"^[A-Za-z0-9_-]*$"),
|
|
17
|
+
) -> dict:
|
|
18
|
+
"""CLI がブラウザで開く SSO ログインページの URL を返す。"""
|
|
19
|
+
signin_url = str(request.url_for("signin")).rstrip("/")
|
|
20
|
+
- login_url = f"{signin_url}?port={port}"
|
|
21
|
+
+ login_url = signin_url + "?" + urlencode({"port": port, "state": state})
|
|
22
|
+
return {"login_url": login_url, "api_key": settings.firebase_api_key}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@router.get("/signin", response_class=HTMLResponse)
|
|
26
|
+
async def signin(
|
|
27
|
+
- port: int = Query(description="CLI のローカルサーバーポート"),
|
|
28
|
+
+ port: int = Query(description="CLI のローカルサーバーポート", ge=1, le=65535),
|
|
29
|
+
+ state: str = Query(default="", max_length=256, pattern=r"^[A-Za-z0-9_-]*$"),
|
|
30
|
+
) -> str:
|
|
31
|
+
"""プロバイダ選択 → Firebase JS SDK で SSO → トークンを CLI に返す HTML ページ。"""
|
|
32
|
+
return f"""<!DOCTYPE html>
|
|
33
|
+
@@ -106,7 +111,8 @@
|
|
34
|
+
window.location.href =
|
|
35
|
+
"http://localhost:{port}/callback"
|
|
36
|
+
+ "?id_token=" + encodeURIComponent(idToken)
|
|
37
|
+
- + "&refresh_token=" + encodeURIComponent(refreshToken);
|
|
38
|
+
+ + "&refresh_token=" + encodeURIComponent(refreshToken)
|
|
39
|
+
+ + "&state=" + encodeURIComponent({json.dumps(state)});
|
|
40
|
+
}} catch (err) {{
|
|
41
|
+
status.className = "error";
|
|
42
|
+
status.textContent = "認証に失敗しました: " + err.message;
|
|
43
|
+
--- a/tests/test_auth.py
|
|
44
|
+
+++ b/tests/test_auth.py
|
|
45
|
+
@@ -34,3 +34,24 @@
|
|
46
|
+
assert resp.status_code == 200
|
|
47
|
+
assert "login('google')" in resp.text
|
|
48
|
+
assert "login('microsoft')" in resp.text
|
|
49
|
+
+
|
|
50
|
+
+
|
|
51
|
+
+def test_login_state_roundtrip():
|
|
52
|
+
+ client = TestClient(app)
|
|
53
|
+
+ state = "test_state-123"
|
|
54
|
+
+ resp = client.get("/auth/login", params={"port": 8085, "state": state})
|
|
55
|
+
+ assert resp.status_code == 200
|
|
56
|
+
+ assert "state=" + state in resp.json()["login_url"]
|
|
57
|
+
+ page = client.get(resp.json()["login_url"])
|
|
58
|
+
+ assert page.status_code == 200
|
|
59
|
+
+ assert 'encodeURIComponent("test_state-123")' in page.text
|
|
60
|
+
+
|
|
61
|
+
+
|
|
62
|
+
+def test_invalid_state_rejected():
|
|
63
|
+
+ client = TestClient(app)
|
|
64
|
+
+ assert client.get("/auth/signin", params={"port": 8085, "state": "<script>"}).status_code == 422
|
|
65
|
+
+
|
|
66
|
+
+
|
|
67
|
+
+def test_invalid_port_rejected():
|
|
68
|
+
+ client = TestClient(app)
|
|
69
|
+
+ assert client.get("/auth/login", params={"port": 0}).status_code == 422
|
|
@@ -54,11 +54,11 @@ resutil push/pull
|
|
|
54
54
|
resutil login --server-url https://resutil-server-586113954845.asia-northeast1.run.app
|
|
55
55
|
|
|
56
56
|
# Microsoft アカウントでログイン
|
|
57
|
-
resutil login --
|
|
57
|
+
resutil login --server-url https://resutil-server-586113954845.asia-northeast1.run.app
|
|
58
58
|
|
|
59
59
|
# resutil-conf.yaml に server_url が設定済みなら --server-url は省略可能
|
|
60
60
|
resutil login
|
|
61
|
-
resutil login
|
|
61
|
+
resutil login
|
|
62
62
|
```
|
|
63
63
|
|
|
64
64
|
### 2. プロジェクト初期化
|
|
@@ -122,11 +122,11 @@ storage_config:
|
|
|
122
122
|
```
|
|
123
123
|
|
|
124
124
|
- `id_token`: Firebase が発行する JWT(有効期限 1 時間)
|
|
125
|
-
- `refresh_token`:
|
|
125
|
+
- `refresh_token`: トークン更新用(API要求前と401応答時に自動更新)
|
|
126
126
|
- ファイルパーミッション: `600`
|
|
127
127
|
|
|
128
128
|
## 未実装・今後の課題
|
|
129
129
|
|
|
130
|
-
-
|
|
130
|
+
- **ログインstate対応**: 新しいCLIを利用する前に docs/server-login-state.patch をサーバーへ適用し、デプロイする。state未対応サーバーへの新規ログインは拒否される。
|
|
131
131
|
- **resutil-server 側のテスト**: サーバー側は 30 テスト通過済み(`uv run pytest -v`)
|
|
132
|
-
- **CLI 側のテスト**:
|
|
132
|
+
- **CLI 側のテスト**: 認証更新、接続先照合、URLエンコード、コールバックstateを回帰テストで検証。
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "resutil"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.3"
|
|
4
4
|
description = "Resutil is a utility to manage experimental result data obtained from Python projects. It also manages dependency such as codes and input data with result data. Data is synced to the cloud for team sharing and collaboration."
|
|
5
5
|
authors = [{ name = "nobkat", email = "katayama@rs.tus.ac.jp" }]
|
|
6
6
|
dependencies = [
|
|
@@ -16,6 +16,7 @@ dependencies = [
|
|
|
16
16
|
"pyreadline3>=3.5.4; platform_system == 'Windows'",
|
|
17
17
|
"textual>=8.2.1",
|
|
18
18
|
"httpx>=0.27.0",
|
|
19
|
+
"tzdata>=2024.1 ; sys_platform == 'win32'",
|
|
19
20
|
]
|
|
20
21
|
readme = "README.md"
|
|
21
22
|
requires-python = ">= 3.10"
|
|
@@ -32,12 +33,14 @@ dev = [
|
|
|
32
33
|
"black>=24.4.0",
|
|
33
34
|
"pytest-mock>=3.14.0",
|
|
34
35
|
"mypy>=1.10.0",
|
|
36
|
+
"types-pyyaml>=6.0.12.20261103",
|
|
35
37
|
]
|
|
36
38
|
|
|
37
39
|
[tool.hatch.metadata]
|
|
38
40
|
allow-direct-references = true
|
|
39
41
|
|
|
40
42
|
[tool.hatch.build.targets.wheel]
|
|
43
|
+
core-metadata-version = "2.4"
|
|
41
44
|
packages = ["src/resutil"]
|
|
42
45
|
|
|
43
46
|
[project.scripts]
|
|
@@ -46,3 +49,13 @@ resutil = 'resutil.cli:main'
|
|
|
46
49
|
[project.urls]
|
|
47
50
|
Homepage = "https://github.com/KatayamaLab/resutil"
|
|
48
51
|
Issues = "https://github.com/KatayamaLab/resutil/issues"
|
|
52
|
+
|
|
53
|
+
[tool.mypy]
|
|
54
|
+
ignore_missing_imports = true
|
|
55
|
+
check_untyped_defs = true
|
|
56
|
+
|
|
57
|
+
[tool.black]
|
|
58
|
+
target-version = ["py310"]
|
|
59
|
+
|
|
60
|
+
[tool.hatch.build.targets.sdist]
|
|
61
|
+
core-metadata-version = "2.4"
|