cellismo-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cellismo_cli-0.1.0/PKG-INFO +7 -0
- cellismo_cli-0.1.0/README.md +105 -0
- cellismo_cli-0.1.0/cellismo_cli/__init__.py +0 -0
- cellismo_cli-0.1.0/cellismo_cli/api_client.py +103 -0
- cellismo_cli-0.1.0/cellismo_cli/commands/__init__.py +0 -0
- cellismo_cli-0.1.0/cellismo_cli/commands/login.py +29 -0
- cellismo_cli-0.1.0/cellismo_cli/commands/upload.py +111 -0
- cellismo_cli-0.1.0/cellismo_cli/config.py +25 -0
- cellismo_cli-0.1.0/cellismo_cli/main.py +13 -0
- cellismo_cli-0.1.0/cellismo_cli/uploader.py +227 -0
- cellismo_cli-0.1.0/cellismo_cli.egg-info/PKG-INFO +7 -0
- cellismo_cli-0.1.0/cellismo_cli.egg-info/SOURCES.txt +21 -0
- cellismo_cli-0.1.0/cellismo_cli.egg-info/dependency_links.txt +1 -0
- cellismo_cli-0.1.0/cellismo_cli.egg-info/entry_points.txt +2 -0
- cellismo_cli-0.1.0/cellismo_cli.egg-info/requires.txt +2 -0
- cellismo_cli-0.1.0/cellismo_cli.egg-info/top_level.txt +1 -0
- cellismo_cli-0.1.0/pyproject.toml +25 -0
- cellismo_cli-0.1.0/setup.cfg +4 -0
- cellismo_cli-0.1.0/tests/test_api_client.py +41 -0
- cellismo_cli-0.1.0/tests/test_config.py +29 -0
- cellismo_cli-0.1.0/tests/test_login_command.py +29 -0
- cellismo_cli-0.1.0/tests/test_upload_command.py +43 -0
- cellismo_cli-0.1.0/tests/test_uploader.py +49 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# cellismo-cli
|
|
2
|
+
|
|
3
|
+
A command-line tool for uploading files to the Cellismo platform.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install cellismo-cli
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
### Login
|
|
14
|
+
|
|
15
|
+
Authenticate against your Cellismo server and save credentials locally:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
cellismo login
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
You will be prompted for:
|
|
22
|
+
- **Email**
|
|
23
|
+
- **Password**
|
|
24
|
+
|
|
25
|
+
Credentials are saved to `~/.cellismo/config.json`.
|
|
26
|
+
|
|
27
|
+
### Upload files
|
|
28
|
+
|
|
29
|
+
Upload one or more local files to a project:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
cellismo upload --project <project_id> file1.fastq.gz file2.fastq.gz
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Options:
|
|
36
|
+
- `--project` (required) — the project ID to upload files into
|
|
37
|
+
- `--threads` (optional) — number of parallel upload threads (default: auto, clamped to 10–30)
|
|
38
|
+
|
|
39
|
+
## Local Development
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
cd cellismo-cli
|
|
43
|
+
|
|
44
|
+
# Install in editable mode — makes the `cellismo` command available immediately
|
|
45
|
+
# and reflects code changes without reinstalling
|
|
46
|
+
pip install -e .
|
|
47
|
+
|
|
48
|
+
# Or build a distributable wheel/sdist
|
|
49
|
+
python -m build
|
|
50
|
+
pip install dist/cellismo_cli-*.whl
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Publishing to PyPI
|
|
54
|
+
|
|
55
|
+
### Prerequisites
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install build twine
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### 1. Update metadata
|
|
62
|
+
|
|
63
|
+
Before publishing, fill in author info in `pyproject.toml`:
|
|
64
|
+
|
|
65
|
+
```toml
|
|
66
|
+
[project]
|
|
67
|
+
authors = [{ name = "Your Name", email = "you@example.com" }]
|
|
68
|
+
license = { text = "MIT" }
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### 2. Build the distribution
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
cd cellismo-cli
|
|
75
|
+
python -m build
|
|
76
|
+
# Produces dist/cellismo_cli-0.1.0-py3-none-any.whl and dist/cellismo_cli-0.1.0.tar.gz
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### 3. (Optional) Test on TestPyPI first
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
twine upload --repository testpypi dist/*
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Install from TestPyPI to verify:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
pip install --index-url https://test.pypi.org/simple/ cellismo-cli
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### 4. Publish to PyPI
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
twine upload dist/*
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
You will be prompted for your PyPI username and password. Using an API token is recommended — generate one at [pypi.org](https://pypi.org) under Account Settings → API tokens.
|
|
98
|
+
|
|
99
|
+
After publishing, users can install with:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
pip install cellismo-cli
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
> **Note:** Check [pypi.org/project/cellismo-cli](https://pypi.org/project/cellismo-cli) to confirm the package name is available before publishing. For internal tools, consider a private PyPI registry (e.g., AWS CodeArtifact, Artifactory, or a self-hosted devpi).
|
|
File without changes
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ApiError(Exception):
|
|
5
|
+
def __init__(self, status: int, code: str, message: str) -> None:
|
|
6
|
+
super().__init__(message)
|
|
7
|
+
self.status = status
|
|
8
|
+
self.code = code
|
|
9
|
+
self.message = message
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ApiClient:
|
|
13
|
+
def __init__(self, base_url: str, token: str | None) -> None:
|
|
14
|
+
self._base = base_url.rstrip("/")
|
|
15
|
+
self._token = token
|
|
16
|
+
|
|
17
|
+
def _headers(self) -> dict[str, str]:
|
|
18
|
+
h: dict[str, str] = {"Content-Type": "application/json"}
|
|
19
|
+
if self._token:
|
|
20
|
+
h["Authorization"] = f"Bearer {self._token}"
|
|
21
|
+
return h
|
|
22
|
+
|
|
23
|
+
def _raise_for(self, r: requests.Response) -> None:
|
|
24
|
+
if r.ok:
|
|
25
|
+
return
|
|
26
|
+
try:
|
|
27
|
+
err = r.json().get("error", {})
|
|
28
|
+
if isinstance(err, dict):
|
|
29
|
+
code = err.get("code", "unknown")
|
|
30
|
+
message = err.get("message", r.text)
|
|
31
|
+
else:
|
|
32
|
+
code = "unknown"
|
|
33
|
+
message = r.text
|
|
34
|
+
except Exception:
|
|
35
|
+
code = "unknown"
|
|
36
|
+
message = r.text
|
|
37
|
+
raise ApiError(r.status_code, code, message)
|
|
38
|
+
|
|
39
|
+
def issue_token(self, email: str, password: str) -> str:
|
|
40
|
+
r = requests.post(f"{self._base}/api/cli/auth/token",
|
|
41
|
+
json={"email": email, "password": password},
|
|
42
|
+
headers=self._headers())
|
|
43
|
+
self._raise_for(r)
|
|
44
|
+
return r.json()["token"]
|
|
45
|
+
|
|
46
|
+
def register_file(self, *, project_id: str, filename: str, size_bytes: int) -> str:
|
|
47
|
+
r = requests.post(f"{self._base}/api/cli/files/register",
|
|
48
|
+
json={"project_id": project_id, "filename": filename,
|
|
49
|
+
"size_bytes": size_bytes},
|
|
50
|
+
headers=self._headers())
|
|
51
|
+
self._raise_for(r)
|
|
52
|
+
return r.json()["file_id"]
|
|
53
|
+
|
|
54
|
+
def upload_start(self, file_id: str, total_parts: int) -> str:
|
|
55
|
+
r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/start",
|
|
56
|
+
json={"total_parts": total_parts},
|
|
57
|
+
headers=self._headers())
|
|
58
|
+
self._raise_for(r)
|
|
59
|
+
return r.json()["upload_id"]
|
|
60
|
+
|
|
61
|
+
def get_presigned_urls(self, file_id: str, parts_info: list[dict], url_expire_time: int = 600) -> tuple[dict, float]:
|
|
62
|
+
r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/presigned-urls",
|
|
63
|
+
json={"parts_info": parts_info, "url_expire_time": url_expire_time},
|
|
64
|
+
headers=self._headers())
|
|
65
|
+
self._raise_for(r)
|
|
66
|
+
data = r.json()
|
|
67
|
+
return data["urls"], float(data["expires_at"])
|
|
68
|
+
|
|
69
|
+
def report_progress(self, file_id: str, parts_completed: int) -> bool:
|
|
70
|
+
"""Returns True if the server reports the upload was cancelled."""
|
|
71
|
+
r = requests.post(f"{self._base}/api/cli/files/{file_id}/progress",
|
|
72
|
+
json={"parts_completed": parts_completed},
|
|
73
|
+
headers=self._headers())
|
|
74
|
+
self._raise_for(r)
|
|
75
|
+
return bool(r.json().get("cancelled", False))
|
|
76
|
+
|
|
77
|
+
def complete_upload(self, file_id: str, parts: list[dict]) -> None:
|
|
78
|
+
r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/complete",
|
|
79
|
+
json={"parts": parts},
|
|
80
|
+
headers=self._headers())
|
|
81
|
+
self._raise_for(r)
|
|
82
|
+
|
|
83
|
+
def verify_upload(self, file_id: str) -> str:
|
|
84
|
+
"""Returns the S3 composite SHA-256 checksum for the completed upload."""
|
|
85
|
+
r = requests.get(f"{self._base}/api/cli/files/{file_id}/upload/verify",
|
|
86
|
+
headers=self._headers())
|
|
87
|
+
self._raise_for(r)
|
|
88
|
+
return r.json()["checksum_sha256"]
|
|
89
|
+
|
|
90
|
+
def abort_upload(self, file_id: str) -> None:
|
|
91
|
+
try:
|
|
92
|
+
r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/abort",
|
|
93
|
+
headers=self._headers())
|
|
94
|
+
self._raise_for(r)
|
|
95
|
+
except Exception:
|
|
96
|
+
pass # best-effort abort
|
|
97
|
+
|
|
98
|
+
def list_project_files(self, project_id: str) -> list[dict]:
|
|
99
|
+
r = requests.get(f"{self._base}/api/cli/projects/{project_id}/files",
|
|
100
|
+
headers=self._headers())
|
|
101
|
+
self._raise_for(r)
|
|
102
|
+
return r.json()["files"]
|
|
103
|
+
|
|
File without changes
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from cellismo_cli import config
|
|
4
|
+
from cellismo_cli.api_client import ApiClient, ApiError
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@click.command()
|
|
8
|
+
@click.option("--new", "force_new", is_flag=True, default=False, help="Force re-authentication even if already logged in.")
|
|
9
|
+
def login(force_new):
|
|
10
|
+
"""Authenticate with a Cellismo server and save credentials locally."""
|
|
11
|
+
if not force_new:
|
|
12
|
+
cfg = config.load()
|
|
13
|
+
if cfg and cfg.get("token") and cfg.get("base_url"):
|
|
14
|
+
click.echo("Already logged in. Use --new to authenticate again.")
|
|
15
|
+
return
|
|
16
|
+
|
|
17
|
+
# TODO: should be replaced with the real backend URL
|
|
18
|
+
base_url = "http://localhost:8000"
|
|
19
|
+
email = click.prompt("Email")
|
|
20
|
+
password = click.prompt("Password", hide_input=True)
|
|
21
|
+
|
|
22
|
+
client = ApiClient(base_url=base_url, token=None)
|
|
23
|
+
try:
|
|
24
|
+
token = client.issue_token(email, password)
|
|
25
|
+
except ApiError as e:
|
|
26
|
+
raise click.ClickException(e.message)
|
|
27
|
+
|
|
28
|
+
config.save(token, base_url)
|
|
29
|
+
click.echo(f"Logged in as {email}. Credentials saved to ~/.cellismo/config.json")
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import signal
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
from cellismo_cli import config
|
|
7
|
+
from cellismo_cli.api_client import ApiClient, ApiError
|
|
8
|
+
from cellismo_cli.uploader import upload_file, MAX_FILE_SIZE, UploadCancelledError
|
|
9
|
+
|
|
10
|
+
MIN_THREADS = 10
|
|
11
|
+
MAX_THREADS = 30
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.command()
|
|
15
|
+
@click.option("--project", required=True, help="Project ID to upload files into.")
|
|
16
|
+
@click.option("--threads", type=int, default=None, help="Number of upload threads.")
|
|
17
|
+
@click.argument("files", nargs=-1, required=True)
|
|
18
|
+
def upload(project, threads, files):
|
|
19
|
+
"""Upload files to the given project in Cellismo."""
|
|
20
|
+
cfg = config.require()
|
|
21
|
+
client = ApiClient(base_url=cfg["base_url"], token=cfg["token"])
|
|
22
|
+
|
|
23
|
+
if threads is None:
|
|
24
|
+
threads = max((os.cpu_count() or 4) * 3, MIN_THREADS)
|
|
25
|
+
threads = min(threads, MAX_THREADS)
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
existing_files = client.list_project_files(project)
|
|
29
|
+
except ApiError as e:
|
|
30
|
+
if e.status == 401:
|
|
31
|
+
raise click.ClickException(
|
|
32
|
+
"Session expired or invalid. Run `cellismo login --new` to create a new authentication."
|
|
33
|
+
)
|
|
34
|
+
raise click.ClickException(f"Failed to fetch project files: {e.message}")
|
|
35
|
+
|
|
36
|
+
existing_by_name: dict[str, str] = {f["filename"]: f["file_id"] for f in existing_files}
|
|
37
|
+
|
|
38
|
+
for file_path in files:
|
|
39
|
+
if not os.path.exists(file_path):
|
|
40
|
+
click.echo(f"✗ Skipping {file_path} — file not found locally")
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
file_size = os.path.getsize(file_path)
|
|
44
|
+
if file_size > MAX_FILE_SIZE:
|
|
45
|
+
click.echo(f"✗ Skipping {file_path} — file too large (max 48.8 TB)")
|
|
46
|
+
continue
|
|
47
|
+
|
|
48
|
+
filename = os.path.basename(file_path)
|
|
49
|
+
|
|
50
|
+
if filename in existing_by_name:
|
|
51
|
+
overwrite = click.confirm(
|
|
52
|
+
f" '{filename}' already exists in this project. Overwrite?", default=False
|
|
53
|
+
)
|
|
54
|
+
if overwrite:
|
|
55
|
+
# Delete the existing file
|
|
56
|
+
client.abort_upload(existing_by_name[filename])
|
|
57
|
+
else:
|
|
58
|
+
click.echo(f"✗ Skipping {filename} — not overwriting")
|
|
59
|
+
continue
|
|
60
|
+
|
|
61
|
+
try:
|
|
62
|
+
# Register the file information into database
|
|
63
|
+
file_id = client.register_file(
|
|
64
|
+
project_id=project, filename=filename, size_bytes=file_size
|
|
65
|
+
)
|
|
66
|
+
except ApiError as e:
|
|
67
|
+
if e.status == 403:
|
|
68
|
+
click.echo(f"✗ Skipping {filename} — you don't have access to project {project}")
|
|
69
|
+
elif e.status == 401:
|
|
70
|
+
raise click.ClickException(
|
|
71
|
+
"Session expired or invalid. Run `cellismo login --new` to create a new authentication."
|
|
72
|
+
)
|
|
73
|
+
else:
|
|
74
|
+
click.echo(f"✗ Skipping {filename} — {e.message}")
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
click.echo(f"\nUploading {filename} (file ID: {file_id})")
|
|
78
|
+
|
|
79
|
+
def on_progress(parts_done: int, total_parts: int) -> bool:
|
|
80
|
+
try:
|
|
81
|
+
return client.report_progress(file_id, parts_done)
|
|
82
|
+
except Exception:
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
success = upload_file(
|
|
87
|
+
file_path=file_path,
|
|
88
|
+
file_id=file_id,
|
|
89
|
+
api_client=client,
|
|
90
|
+
num_threads=threads,
|
|
91
|
+
on_progress=on_progress,
|
|
92
|
+
)
|
|
93
|
+
if success:
|
|
94
|
+
click.echo(f"✓ {filename} uploaded successfully")
|
|
95
|
+
else:
|
|
96
|
+
click.echo(f"✗ {filename} upload failed")
|
|
97
|
+
except UploadCancelledError:
|
|
98
|
+
client.abort_upload(file_id)
|
|
99
|
+
click.echo(f"\n✗ {filename} upload cancelled from the web UI")
|
|
100
|
+
except KeyboardInterrupt:
|
|
101
|
+
# to make pressing any more ctrl+c do nothing
|
|
102
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
103
|
+
click.echo(f"\n✗ Upload interrupted — aborting {filename}")
|
|
104
|
+
try:
|
|
105
|
+
client.abort_upload(file_id)
|
|
106
|
+
except Exception:
|
|
107
|
+
pass
|
|
108
|
+
raise SystemExit(1)
|
|
109
|
+
except Exception as e:
|
|
110
|
+
click.echo(f"✗ {filename} upload failed: {e}")
|
|
111
|
+
client.abort_upload(file_id)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
_CONFIG_PATH = Path.home() / ".cellismo" / "config.json"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load() -> dict | None:
|
|
8
|
+
if not _CONFIG_PATH.exists():
|
|
9
|
+
return None
|
|
10
|
+
try:
|
|
11
|
+
return json.loads(_CONFIG_PATH.read_text())
|
|
12
|
+
except (json.JSONDecodeError, OSError):
|
|
13
|
+
return None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def save(token: str, base_url: str) -> None:
|
|
17
|
+
_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
18
|
+
_CONFIG_PATH.write_text(json.dumps({"token": token, "base_url": base_url}))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def require() -> dict:
|
|
22
|
+
cfg = load()
|
|
23
|
+
if cfg is None:
|
|
24
|
+
raise SystemExit("Not logged in. Run `cellismo login` first.")
|
|
25
|
+
return cfg
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from cellismo_cli.commands.login import login
|
|
4
|
+
from cellismo_cli.commands.upload import upload
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@click.group()
|
|
8
|
+
def cli():
|
|
9
|
+
"""Cellismo CLI — upload files to your projects."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
cli.add_command(login)
|
|
13
|
+
cli.add_command(upload)
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import hashlib
|
|
3
|
+
import math
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
7
|
+
from threading import Event, Thread
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from cellismo_cli.api_client import ApiError
|
|
12
|
+
|
|
13
|
+
class UploadCancelledError(Exception):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
CANCEL_POLL_INTERVAL = 2 # seconds between background cancellation checks
|
|
18
|
+
UPLOAD_CHUNK_SIZE = 256 * 1024 # 256 KiB — abort is checked between chunks
|
|
19
|
+
|
|
20
|
+
PART_SIZE_MIN = 5 * 1024 * 1024 # 5 MiB
|
|
21
|
+
PART_SIZE_MAX = 5 * 1024 * 1024 * 1024 # 5 GiB
|
|
22
|
+
PART_COUNT_MAX = 10_000
|
|
23
|
+
MAX_FILE_SIZE = int(48.8 * 1024**4)
|
|
24
|
+
MAX_RETRY = 10
|
|
25
|
+
URL_EXPIRE_TIME = 600
|
|
26
|
+
URL_EXPIRE_BUFFER = 30 # seconds before expiry to refresh URLs
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def get_part_num_size(file_size: int) -> tuple[int, int]:
|
|
30
|
+
minimum = math.ceil(file_size / PART_COUNT_MAX)
|
|
31
|
+
part_size = max(minimum, PART_SIZE_MIN)
|
|
32
|
+
part_size = min(part_size, PART_SIZE_MAX)
|
|
33
|
+
number_of_parts = math.ceil(file_size / part_size)
|
|
34
|
+
return number_of_parts, part_size
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def compute_checksums(file_path: str, part_size: int, number_of_parts: int) -> dict[int, tuple[bytes, str]]:
|
|
38
|
+
checksums: dict[int, tuple[bytes, str]] = {}
|
|
39
|
+
with open(file_path, "rb") as f:
|
|
40
|
+
for pn in range(1, number_of_parts + 1):
|
|
41
|
+
f.seek((pn - 1) * part_size)
|
|
42
|
+
data = f.read(part_size)
|
|
43
|
+
raw = hashlib.sha256(data).digest()
|
|
44
|
+
checksums[pn] = (raw, base64.b64encode(raw).decode())
|
|
45
|
+
return checksums
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _upload_part(
|
|
49
|
+
part_number: int,
|
|
50
|
+
file_path: str,
|
|
51
|
+
part_size: int,
|
|
52
|
+
presigned_url: str,
|
|
53
|
+
checksum_b64: str,
|
|
54
|
+
urls_expire_at: float,
|
|
55
|
+
abort_event: Event,
|
|
56
|
+
url_expire_event: Event,
|
|
57
|
+
) -> dict:
|
|
58
|
+
if abort_event.is_set():
|
|
59
|
+
return {"PartNumber": part_number, "Failed": True}
|
|
60
|
+
if url_expire_event.is_set() or time.time() >= urls_expire_at - URL_EXPIRE_BUFFER:
|
|
61
|
+
url_expire_event.set()
|
|
62
|
+
return {"Expired": True, "PartNumber": part_number}
|
|
63
|
+
try:
|
|
64
|
+
with open(file_path, "rb") as f:
|
|
65
|
+
f.seek((part_number - 1) * part_size)
|
|
66
|
+
data = f.read(part_size)
|
|
67
|
+
|
|
68
|
+
r = requests.put(
|
|
69
|
+
presigned_url,
|
|
70
|
+
data=data,
|
|
71
|
+
headers={
|
|
72
|
+
"x-amz-checksum-sha256": checksum_b64,
|
|
73
|
+
"x-amz-sdk-checksum-algorithm": "SHA256",
|
|
74
|
+
},
|
|
75
|
+
)
|
|
76
|
+
if abort_event.is_set():
|
|
77
|
+
return {"PartNumber": part_number, "Failed": True}
|
|
78
|
+
if r.status_code == 403:
|
|
79
|
+
url_expire_event.set()
|
|
80
|
+
return {"Expired": True, "PartNumber": part_number}
|
|
81
|
+
if not r.ok:
|
|
82
|
+
r.raise_for_status()
|
|
83
|
+
except Exception as e:
|
|
84
|
+
print(f"\n [part {part_number} error] {type(e).__name__}: {e}")
|
|
85
|
+
return {"PartNumber": part_number, "Failed": True}
|
|
86
|
+
return {"PartNumber": part_number, "ETag": r.headers["ETag"], "ChecksumSHA256": checksum_b64}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def upload_file(
|
|
90
|
+
file_path: str,
|
|
91
|
+
file_id: str,
|
|
92
|
+
api_client,
|
|
93
|
+
num_threads: int,
|
|
94
|
+
on_progress=None,
|
|
95
|
+
) -> bool:
|
|
96
|
+
"""Upload a single file. Returns True on success, False on failure."""
|
|
97
|
+
file_size = os.path.getsize(file_path)
|
|
98
|
+
if file_size > MAX_FILE_SIZE:
|
|
99
|
+
print(f" ✗ Skipping {file_path} — file too large (max 48.8 TB)")
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
number_of_parts, part_size = get_part_num_size(file_size)
|
|
103
|
+
try:
|
|
104
|
+
api_client.upload_start(file_id, total_parts=number_of_parts)
|
|
105
|
+
except ApiError as e:
|
|
106
|
+
if e.code == "cancelled":
|
|
107
|
+
raise UploadCancelledError()
|
|
108
|
+
raise
|
|
109
|
+
checksums = compute_checksums(file_path, part_size, number_of_parts)
|
|
110
|
+
|
|
111
|
+
# check whether file upload is cancelled on the frontend before going into the while loop
|
|
112
|
+
try:
|
|
113
|
+
if api_client.report_progress(file_id, 0):
|
|
114
|
+
raise UploadCancelledError()
|
|
115
|
+
except ApiError as e:
|
|
116
|
+
if e.code == "cancelled":
|
|
117
|
+
raise UploadCancelledError()
|
|
118
|
+
raise
|
|
119
|
+
|
|
120
|
+
unsorted: dict[int, dict] = {}
|
|
121
|
+
remaining = set(range(1, number_of_parts + 1))
|
|
122
|
+
completed: set[int] = set()
|
|
123
|
+
failed_counts: dict[int, int] = {}
|
|
124
|
+
num_attempt = 1
|
|
125
|
+
|
|
126
|
+
print(" Preparing...", end="", flush=True)
|
|
127
|
+
abort_event = Event()
|
|
128
|
+
stop_poll = Event()
|
|
129
|
+
|
|
130
|
+
def _poll_cancel():
|
|
131
|
+
while not stop_poll.wait(CANCEL_POLL_INTERVAL):
|
|
132
|
+
try:
|
|
133
|
+
cancelled = api_client.report_progress(file_id, len(completed))
|
|
134
|
+
except Exception:
|
|
135
|
+
continue
|
|
136
|
+
if cancelled:
|
|
137
|
+
abort_event.set()
|
|
138
|
+
break
|
|
139
|
+
|
|
140
|
+
poll_thread = Thread(target=_poll_cancel, daemon=True)
|
|
141
|
+
poll_thread.start()
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
while remaining:
|
|
145
|
+
url_expire_event = Event()
|
|
146
|
+
|
|
147
|
+
parts_info = [
|
|
148
|
+
{"part_number": pn, "checksum_sha256": checksums[pn][1]}
|
|
149
|
+
for pn in sorted(remaining)
|
|
150
|
+
]
|
|
151
|
+
try:
|
|
152
|
+
urls_dict, expires_at = api_client.get_presigned_urls(
|
|
153
|
+
file_id, parts_info, url_expire_time=URL_EXPIRE_TIME * num_attempt
|
|
154
|
+
)
|
|
155
|
+
except ApiError as e:
|
|
156
|
+
if e.code == "cancelled":
|
|
157
|
+
raise UploadCancelledError()
|
|
158
|
+
raise
|
|
159
|
+
urls = {int(k): v for k, v in urls_dict.items()}
|
|
160
|
+
|
|
161
|
+
pool = ThreadPoolExecutor(max_workers=num_threads)
|
|
162
|
+
try:
|
|
163
|
+
futures = [
|
|
164
|
+
pool.submit(
|
|
165
|
+
_upload_part, pn, file_path, part_size,
|
|
166
|
+
urls[pn], checksums[pn][1], expires_at, abort_event, url_expire_event,
|
|
167
|
+
)
|
|
168
|
+
for pn in sorted(remaining)
|
|
169
|
+
]
|
|
170
|
+
for future in as_completed(futures):
|
|
171
|
+
if abort_event.is_set():
|
|
172
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
173
|
+
raise UploadCancelledError()
|
|
174
|
+
|
|
175
|
+
result = future.result()
|
|
176
|
+
if result.get("Failed"):
|
|
177
|
+
pn = result["PartNumber"]
|
|
178
|
+
failed_counts[pn] = failed_counts.get(pn, 0) + 1
|
|
179
|
+
if max(failed_counts.values()) >= MAX_RETRY:
|
|
180
|
+
abort_event.set()
|
|
181
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
182
|
+
api_client.abort_upload(file_id)
|
|
183
|
+
return False
|
|
184
|
+
elif result.get("Expired"):
|
|
185
|
+
continue
|
|
186
|
+
else:
|
|
187
|
+
pn = result["PartNumber"]
|
|
188
|
+
unsorted[pn] = result
|
|
189
|
+
remaining.discard(pn)
|
|
190
|
+
completed.add(pn)
|
|
191
|
+
pct = len(completed) * 100 // number_of_parts
|
|
192
|
+
if on_progress:
|
|
193
|
+
cancelled = on_progress(len(completed), number_of_parts)
|
|
194
|
+
if cancelled:
|
|
195
|
+
abort_event.set()
|
|
196
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
197
|
+
raise UploadCancelledError()
|
|
198
|
+
filled = pct * 50 // 100
|
|
199
|
+
bar = "█" * filled + "░" * (50 - filled)
|
|
200
|
+
print(f"\r |{bar}| {pct}%", end="", flush=True)
|
|
201
|
+
except KeyboardInterrupt:
|
|
202
|
+
abort_event.set()
|
|
203
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
204
|
+
raise
|
|
205
|
+
else:
|
|
206
|
+
pool.shutdown(wait=True)
|
|
207
|
+
|
|
208
|
+
if abort_event.is_set():
|
|
209
|
+
raise UploadCancelledError()
|
|
210
|
+
|
|
211
|
+
num_attempt += 1
|
|
212
|
+
finally:
|
|
213
|
+
stop_poll.set()
|
|
214
|
+
poll_thread.join()
|
|
215
|
+
|
|
216
|
+
parts = [unsorted[pn] for pn in range(1, number_of_parts + 1)]
|
|
217
|
+
api_client.complete_upload(file_id, parts)
|
|
218
|
+
|
|
219
|
+
part_raw_digests = b"".join(checksums[pn][0] for pn in range(1, number_of_parts + 1))
|
|
220
|
+
composite = base64.b64encode(hashlib.sha256(part_raw_digests).digest()).decode()
|
|
221
|
+
local_checksum = f"{composite}-{number_of_parts}"
|
|
222
|
+
s3_checksum = api_client.verify_upload(file_id)
|
|
223
|
+
if local_checksum != s3_checksum:
|
|
224
|
+
raise RuntimeError(f"Checksum mismatch: local={local_checksum} s3={s3_checksum}")
|
|
225
|
+
print(f"\n ✓ Integrity OK: {local_checksum}")
|
|
226
|
+
|
|
227
|
+
return True
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
cellismo_cli/__init__.py
|
|
4
|
+
cellismo_cli/api_client.py
|
|
5
|
+
cellismo_cli/config.py
|
|
6
|
+
cellismo_cli/main.py
|
|
7
|
+
cellismo_cli/uploader.py
|
|
8
|
+
cellismo_cli.egg-info/PKG-INFO
|
|
9
|
+
cellismo_cli.egg-info/SOURCES.txt
|
|
10
|
+
cellismo_cli.egg-info/dependency_links.txt
|
|
11
|
+
cellismo_cli.egg-info/entry_points.txt
|
|
12
|
+
cellismo_cli.egg-info/requires.txt
|
|
13
|
+
cellismo_cli.egg-info/top_level.txt
|
|
14
|
+
cellismo_cli/commands/__init__.py
|
|
15
|
+
cellismo_cli/commands/login.py
|
|
16
|
+
cellismo_cli/commands/upload.py
|
|
17
|
+
tests/test_api_client.py
|
|
18
|
+
tests/test_config.py
|
|
19
|
+
tests/test_login_command.py
|
|
20
|
+
tests/test_upload_command.py
|
|
21
|
+
tests/test_uploader.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cellismo_cli
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "cellismo-cli"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Cellismo file upload CLI"
|
|
5
|
+
requires-python = ">=3.12"
|
|
6
|
+
dependencies = [
|
|
7
|
+
"click>=8.1",
|
|
8
|
+
"requests>=2.32",
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
[project.scripts]
|
|
12
|
+
cellismo = "cellismo_cli.main:cli"
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["setuptools>=75"]
|
|
16
|
+
build-backend = "setuptools.build_meta"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.packages.find]
|
|
19
|
+
where = ["."]
|
|
20
|
+
include = ["cellismo_cli*"]
|
|
21
|
+
|
|
22
|
+
[tool.pytest.ini_options]
|
|
23
|
+
asyncio_mode = "auto"
|
|
24
|
+
testpaths = ["tests"]
|
|
25
|
+
addopts = "-ra"
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
import responses as resp_lib
|
|
3
|
+
from cellismo_cli.api_client import ApiClient, ApiError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@resp_lib.activate
|
|
7
|
+
def test_issue_token_success():
|
|
8
|
+
resp_lib.add(resp_lib.POST, "http://backend/api/cli/auth/token",
|
|
9
|
+
json={"token": "raw-tok-123"}, status=200)
|
|
10
|
+
client = ApiClient(base_url="http://backend", token=None)
|
|
11
|
+
token = client.issue_token("user@x.com", "pass")
|
|
12
|
+
assert token == "raw-tok-123"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@resp_lib.activate
|
|
16
|
+
def test_issue_token_401_raises():
|
|
17
|
+
resp_lib.add(resp_lib.POST, "http://backend/api/cli/auth/token",
|
|
18
|
+
json={"error": {"code": "invalid_credentials", "message": "bad"}}, status=401)
|
|
19
|
+
client = ApiClient(base_url="http://backend", token=None)
|
|
20
|
+
with pytest.raises(ApiError) as exc_info:
|
|
21
|
+
client.issue_token("user@x.com", "wrong")
|
|
22
|
+
assert exc_info.value.status == 401
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@resp_lib.activate
|
|
26
|
+
def test_register_file_success():
|
|
27
|
+
resp_lib.add(resp_lib.POST, "http://backend/api/cli/files/register",
|
|
28
|
+
json={"file_id": "fid-abc"}, status=200)
|
|
29
|
+
client = ApiClient(base_url="http://backend", token="tok")
|
|
30
|
+
fid = client.register_file(project_id="proj1", filename="r.gz", size_bytes=1000)
|
|
31
|
+
assert fid == "fid-abc"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@resp_lib.activate
|
|
35
|
+
def test_register_file_403_raises():
|
|
36
|
+
resp_lib.add(resp_lib.POST, "http://backend/api/cli/files/register",
|
|
37
|
+
json={"error": {"code": "forbidden", "message": "no access"}}, status=403)
|
|
38
|
+
client = ApiClient(base_url="http://backend", token="tok")
|
|
39
|
+
with pytest.raises(ApiError) as exc_info:
|
|
40
|
+
client.register_file(project_id="p", filename="f", size_bytes=1)
|
|
41
|
+
assert exc_info.value.status == 403
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from unittest.mock import patch
|
|
3
|
+
from cellismo_cli import config
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_load_returns_none_when_file_missing(tmp_path):
|
|
7
|
+
with patch.object(config, "_CONFIG_PATH", tmp_path / "config.json"):
|
|
8
|
+
assert config.load() is None
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_save_and_load_roundtrip(tmp_path):
|
|
12
|
+
with patch.object(config, "_CONFIG_PATH", tmp_path / ".cellismo" / "config.json"):
|
|
13
|
+
config.save("mytoken", "http://localhost:8000")
|
|
14
|
+
cfg = config.load()
|
|
15
|
+
assert cfg == {"token": "mytoken", "base_url": "http://localhost:8000"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_require_exits_when_no_config(tmp_path):
|
|
19
|
+
with patch.object(config, "_CONFIG_PATH", tmp_path / "config.json"):
|
|
20
|
+
with pytest.raises(SystemExit):
|
|
21
|
+
config.require()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_require_returns_config_when_present(tmp_path):
|
|
25
|
+
p = tmp_path / ".cellismo" / "config.json"
|
|
26
|
+
with patch.object(config, "_CONFIG_PATH", p):
|
|
27
|
+
config.save("tok", "http://x")
|
|
28
|
+
cfg = config.require()
|
|
29
|
+
assert cfg["token"] == "tok"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from click.testing import CliRunner
|
|
3
|
+
from unittest.mock import patch, MagicMock
|
|
4
|
+
from cellismo_cli.main import cli
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_login_saves_config_on_success():
|
|
8
|
+
runner = CliRunner()
|
|
9
|
+
with patch("cellismo_cli.commands.login.ApiClient") as MockClient, \
|
|
10
|
+
patch("cellismo_cli.commands.login.config") as mock_cfg:
|
|
11
|
+
mock_instance = MagicMock()
|
|
12
|
+
mock_instance.issue_token.return_value = "raw-tok-abc"
|
|
13
|
+
MockClient.return_value = mock_instance
|
|
14
|
+
result = runner.invoke(cli, ["login"], input="http://localhost:8000\nuser@x.com\nmypass\n")
|
|
15
|
+
assert result.exit_code == 0
|
|
16
|
+
mock_cfg.save.assert_called_once_with("raw-tok-abc", "http://localhost:8000")
|
|
17
|
+
assert "Logged in" in result.output
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_login_prints_error_on_bad_credentials():
|
|
21
|
+
runner = CliRunner()
|
|
22
|
+
with patch("cellismo_cli.commands.login.ApiClient") as MockClient:
|
|
23
|
+
from cellismo_cli.api_client import ApiError
|
|
24
|
+
mock_instance = MagicMock()
|
|
25
|
+
mock_instance.issue_token.side_effect = ApiError(401, "invalid_credentials", "Invalid email or password.")
|
|
26
|
+
MockClient.return_value = mock_instance
|
|
27
|
+
result = runner.invoke(cli, ["login"], input="http://localhost:8000\nuser@x.com\nbad\n")
|
|
28
|
+
assert result.exit_code != 0
|
|
29
|
+
assert "Invalid" in result.output or "invalid" in result.output
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
import pytest
|
|
4
|
+
from click.testing import CliRunner
|
|
5
|
+
from unittest.mock import patch, MagicMock
|
|
6
|
+
from cellismo_cli.main import cli
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_upload_exits_if_not_logged_in(tmp_path):
|
|
10
|
+
runner = CliRunner()
|
|
11
|
+
with patch("cellismo_cli.commands.upload.config") as mock_cfg:
|
|
12
|
+
mock_cfg.require.side_effect = SystemExit("Not logged in. Run `cellismo login` first.")
|
|
13
|
+
f = tmp_path / "file.gz"
|
|
14
|
+
f.write_bytes(b"x")
|
|
15
|
+
result = runner.invoke(cli, ["upload", "--project", "proj1", str(f)])
|
|
16
|
+
assert result.exit_code != 0
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_upload_skips_missing_file(tmp_path):
|
|
20
|
+
runner = CliRunner()
|
|
21
|
+
with patch("cellismo_cli.commands.upload.config") as mock_cfg, \
|
|
22
|
+
patch("cellismo_cli.commands.upload.ApiClient"):
|
|
23
|
+
mock_cfg.require.return_value = {"token": "tok", "base_url": "http://x"}
|
|
24
|
+
result = runner.invoke(cli, ["upload", "--project", "proj1",
|
|
25
|
+
str(tmp_path / "nonexistent.gz")])
|
|
26
|
+
assert "Skipping" in result.output
|
|
27
|
+
assert "not found" in result.output
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_upload_skips_on_403(tmp_path):
|
|
31
|
+
runner = CliRunner()
|
|
32
|
+
f = tmp_path / "reads.gz"
|
|
33
|
+
f.write_bytes(b"data")
|
|
34
|
+
with patch("cellismo_cli.commands.upload.config") as mock_cfg, \
|
|
35
|
+
patch("cellismo_cli.commands.upload.ApiClient") as MockClient:
|
|
36
|
+
mock_cfg.require.return_value = {"token": "tok", "base_url": "http://x"}
|
|
37
|
+
from cellismo_cli.api_client import ApiError
|
|
38
|
+
mock_instance = MagicMock()
|
|
39
|
+
mock_instance.register_file.side_effect = ApiError(403, "forbidden", "no access")
|
|
40
|
+
MockClient.return_value = mock_instance
|
|
41
|
+
result = runner.invoke(cli, ["upload", "--project", "proj1", str(f)])
|
|
42
|
+
assert "Skipping" in result.output
|
|
43
|
+
assert "access" in result.output.lower()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import base64
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
from cellismo_cli.uploader import get_part_num_size, compute_checksums, PART_SIZE_MIN
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_get_part_num_size_small_file():
|
|
9
|
+
num, size = get_part_num_size(1024)
|
|
10
|
+
assert size == PART_SIZE_MIN
|
|
11
|
+
assert num == 1
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_get_part_num_size_large_file():
|
|
15
|
+
file_size = 100 * 1024 * 1024 # 100 MB
|
|
16
|
+
num, size = get_part_num_size(file_size)
|
|
17
|
+
assert num * size >= file_size
|
|
18
|
+
assert num <= 10_000
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_compute_checksums_single_part():
|
|
22
|
+
data = b"hello world"
|
|
23
|
+
with tempfile.NamedTemporaryFile(delete=False) as f:
|
|
24
|
+
f.write(data)
|
|
25
|
+
path = f.name
|
|
26
|
+
try:
|
|
27
|
+
checksums = compute_checksums(path, part_size=len(data) + 1, number_of_parts=1)
|
|
28
|
+
raw, b64 = checksums[1]
|
|
29
|
+
expected_raw = hashlib.sha256(data).digest()
|
|
30
|
+
assert raw == expected_raw
|
|
31
|
+
assert b64 == base64.b64encode(expected_raw).decode()
|
|
32
|
+
finally:
|
|
33
|
+
os.unlink(path)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_compute_checksums_two_parts():
|
|
37
|
+
data = b"a" * 10 + b"b" * 10
|
|
38
|
+
with tempfile.NamedTemporaryFile(delete=False) as f:
|
|
39
|
+
f.write(data)
|
|
40
|
+
path = f.name
|
|
41
|
+
try:
|
|
42
|
+
checksums = compute_checksums(path, part_size=10, number_of_parts=2)
|
|
43
|
+
assert len(checksums) == 2
|
|
44
|
+
r1, _ = checksums[1]
|
|
45
|
+
r2, _ = checksums[2]
|
|
46
|
+
assert r1 == hashlib.sha256(b"a" * 10).digest()
|
|
47
|
+
assert r2 == hashlib.sha256(b"b" * 10).digest()
|
|
48
|
+
finally:
|
|
49
|
+
os.unlink(path)
|