cellismo-cli 0.1.0__py3-none-any.whl

cellismo_cli/api_client.py

@@ -0,0 +1,103 @@
+import requests
+
+
+class ApiError(Exception):
+    def __init__(self, status: int, code: str, message: str) -> None:
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.message = message
+
+
+class ApiClient:
+    def __init__(self, base_url: str, token: str | None) -> None:
+        self._base = base_url.rstrip("/")
+        self._token = token
+
+    def _headers(self) -> dict[str, str]:
+        h: dict[str, str] = {"Content-Type": "application/json"}
+        if self._token:
+            h["Authorization"] = f"Bearer {self._token}"
+        return h
+
+    def _raise_for(self, r: requests.Response) -> None:
+        if r.ok:
+            return
+        try:
+            err = r.json().get("error", {})
+            if isinstance(err, dict):
+                code = err.get("code", "unknown")
+                message = err.get("message", r.text)
+            else:
+                code = "unknown"
+                message = r.text
+        except Exception:
+            code = "unknown"
+            message = r.text
+        raise ApiError(r.status_code, code, message)
+
+    def issue_token(self, email: str, password: str) -> str:
+        r = requests.post(f"{self._base}/api/cli/auth/token",
+                          json={"email": email, "password": password},
+                          headers=self._headers())
+        self._raise_for(r)
+        return r.json()["token"]
+
+    def register_file(self, *, project_id: str, filename: str, size_bytes: int) -> str:
+        r = requests.post(f"{self._base}/api/cli/files/register",
+                          json={"project_id": project_id, "filename": filename,
+                                "size_bytes": size_bytes},
+                          headers=self._headers())
+        self._raise_for(r)
+        return r.json()["file_id"]
+
+    def upload_start(self, file_id: str, total_parts: int) -> str:
+        r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/start",
+                          json={"total_parts": total_parts},
+                          headers=self._headers())
+        self._raise_for(r)
+        return r.json()["upload_id"]
+
+    def get_presigned_urls(self, file_id: str, parts_info: list[dict], url_expire_time: int = 600) -> tuple[dict, float]:
+        r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/presigned-urls",
+                          json={"parts_info": parts_info, "url_expire_time": url_expire_time},
+                          headers=self._headers())
+        self._raise_for(r)
+        data = r.json()
+        return data["urls"], float(data["expires_at"])
+
+    def report_progress(self, file_id: str, parts_completed: int) -> bool:
+        """Returns True if the server reports the upload was cancelled."""
+        r = requests.post(f"{self._base}/api/cli/files/{file_id}/progress",
+                          json={"parts_completed": parts_completed},
+                          headers=self._headers())
+        self._raise_for(r)
+        return bool(r.json().get("cancelled", False))
+
+    def complete_upload(self, file_id: str, parts: list[dict]) -> None:
+        r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/complete",
+                          json={"parts": parts},
+                          headers=self._headers())
+        self._raise_for(r)
+
+    def verify_upload(self, file_id: str) -> str:
+        """Returns the S3 composite SHA-256 checksum for the completed upload."""
+        r = requests.get(f"{self._base}/api/cli/files/{file_id}/upload/verify",
+                         headers=self._headers())
+        self._raise_for(r)
+        return r.json()["checksum_sha256"]
+
+    def abort_upload(self, file_id: str) -> None:
+        try:
+            r = requests.post(f"{self._base}/api/cli/files/{file_id}/upload/abort",
+                              headers=self._headers())
+            self._raise_for(r)
+        except Exception:
+            pass  # best-effort abort
+
+    def list_project_files(self, project_id: str) -> list[dict]:
+        r = requests.get(f"{self._base}/api/cli/projects/{project_id}/files",
+                         headers=self._headers())
+        self._raise_for(r)
+        return r.json()["files"]
+

cellismo_cli/commands/login.py

@@ -0,0 +1,29 @@
+import click
+
+from cellismo_cli import config
+from cellismo_cli.api_client import ApiClient, ApiError
+
+
+@click.command()
+@click.option("--new", "force_new", is_flag=True, default=False, help="Force re-authentication even if already logged in.")
+def login(force_new):
+    """Authenticate with a Cellismo server and save credentials locally."""
+    if not force_new:
+        cfg = config.load()
+        if cfg and cfg.get("token") and cfg.get("base_url"):
+            click.echo("Already logged in. Use --new to authenticate again.")
+            return
+
+    # TODO: should be replaced with the real backend URL
+    base_url = "http://localhost:8000"
+    email = click.prompt("Email")
+    password = click.prompt("Password", hide_input=True)
+
+    client = ApiClient(base_url=base_url, token=None)
+    try:
+        token = client.issue_token(email, password)
+    except ApiError as e:
+        raise click.ClickException(e.message)
+
+    config.save(token, base_url)
+    click.echo(f"Logged in as {email}. Credentials saved to ~/.cellismo/config.json")

cellismo_cli/commands/upload.py

@@ -0,0 +1,111 @@
+import os
+import signal
+
+import click
+
+from cellismo_cli import config
+from cellismo_cli.api_client import ApiClient, ApiError
+from cellismo_cli.uploader import upload_file, MAX_FILE_SIZE, UploadCancelledError
+
+MIN_THREADS = 10
+MAX_THREADS = 30
+
+
+@click.command()
+@click.option("--project", required=True, help="Project ID to upload files into.")
+@click.option("--threads", type=int, default=None, help="Number of upload threads.")
+@click.argument("files", nargs=-1, required=True)
+def upload(project, threads, files):
+    """Upload files to the given project in Cellismo."""
+    cfg = config.require()
+    client = ApiClient(base_url=cfg["base_url"], token=cfg["token"])
+
+    if threads is None:
+        threads = max((os.cpu_count() or 4) * 3, MIN_THREADS)
+        threads = min(threads, MAX_THREADS)
+
+    try:
+        existing_files = client.list_project_files(project)
+    except ApiError as e:
+        if e.status == 401:
+            raise click.ClickException(
+                "Session expired or invalid. Run `cellismo login --new` to create a new authentication."
+            )
+        raise click.ClickException(f"Failed to fetch project files: {e.message}")
+
+    existing_by_name: dict[str, str] = {f["filename"]: f["file_id"] for f in existing_files}
+
+    for file_path in files:
+        if not os.path.exists(file_path):
+            click.echo(f"✗ Skipping {file_path} — file not found locally")
+            continue
+
+        file_size = os.path.getsize(file_path)
+        if file_size > MAX_FILE_SIZE:
+            click.echo(f"✗ Skipping {file_path} — file too large (max 48.8 TB)")
+            continue
+
+        filename = os.path.basename(file_path)
+
+        if filename in existing_by_name:
+            overwrite = click.confirm(
+                f"  '{filename}' already exists in this project. Overwrite?", default=False
+            )
+            if overwrite:
+                # Delete the existing file
+                client.abort_upload(existing_by_name[filename])
+            else:
+                click.echo(f"✗ Skipping {filename} — not overwriting")
+                continue
+    
+        try:
+            # Register the file information into database
+            file_id = client.register_file(
+                project_id=project, filename=filename, size_bytes=file_size
+            )
+        except ApiError as e:
+            if e.status == 403:
+                click.echo(f"✗ Skipping {filename} — you don't have access to project {project}")
+            elif e.status == 401:
+                raise click.ClickException(
+                    "Session expired or invalid. Run `cellismo login --new` to create a new authentication."
+                )
+            else:
+                click.echo(f"✗ Skipping {filename} — {e.message}")
+            continue
+
+        click.echo(f"\nUploading {filename} (file ID: {file_id})")
+
+        def on_progress(parts_done: int, total_parts: int) -> bool:
+            try:
+                return client.report_progress(file_id, parts_done)
+            except Exception:
+                return False
+
+        try:
+            success = upload_file(
+                file_path=file_path,
+                file_id=file_id,
+                api_client=client,
+                num_threads=threads,
+                on_progress=on_progress,
+            )
+            if success:
+                click.echo(f"✓ {filename} uploaded successfully")
+            else:
+                click.echo(f"✗ {filename} upload failed")
+        except UploadCancelledError:
+            client.abort_upload(file_id)
+            click.echo(f"\n✗ {filename} upload cancelled from the web UI")
+        except KeyboardInterrupt:
+            # to make pressing any more ctrl+c do nothing
+            signal.signal(signal.SIGINT, signal.SIG_IGN)
+            click.echo(f"\n✗ Upload interrupted — aborting {filename}")
+            try:
+                client.abort_upload(file_id)
+            except Exception:
+                pass
+            raise SystemExit(1)
+        except Exception as e:
+            click.echo(f"✗ {filename} upload failed: {e}")
+            client.abort_upload(file_id)

cellismo_cli/config.py

@@ -0,0 +1,25 @@
+import json
+from pathlib import Path
+
+_CONFIG_PATH = Path.home() / ".cellismo" / "config.json"
+
+
+def load() -> dict | None:
+    if not _CONFIG_PATH.exists():
+        return None
+    try:
+        return json.loads(_CONFIG_PATH.read_text())
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def save(token: str, base_url: str) -> None:
+    _CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    _CONFIG_PATH.write_text(json.dumps({"token": token, "base_url": base_url}))
+
+
+def require() -> dict:
+    cfg = load()
+    if cfg is None:
+        raise SystemExit("Not logged in. Run `cellismo login` first.")
+    return cfg

cellismo_cli/main.py

@@ -0,0 +1,13 @@
+import click
+
+from cellismo_cli.commands.login import login
+from cellismo_cli.commands.upload import upload
+
+
+@click.group()
+def cli():
+    """Cellismo CLI — upload files to your projects."""
+
+
+cli.add_command(login)
+cli.add_command(upload)

cellismo_cli/uploader.py

@@ -0,0 +1,227 @@
+import base64
+import hashlib
+import math
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Event, Thread
+
+import requests
+
+from cellismo_cli.api_client import ApiError
+
+class UploadCancelledError(Exception):
+    pass
+
+
+CANCEL_POLL_INTERVAL = 2  # seconds between background cancellation checks
+UPLOAD_CHUNK_SIZE = 256 * 1024  # 256 KiB — abort is checked between chunks
+
+PART_SIZE_MIN = 5 * 1024 * 1024          # 5 MiB
+PART_SIZE_MAX = 5 * 1024 * 1024 * 1024   # 5 GiB
+PART_COUNT_MAX = 10_000
+MAX_FILE_SIZE = int(48.8 * 1024**4)
+MAX_RETRY = 10
+URL_EXPIRE_TIME = 600
+URL_EXPIRE_BUFFER = 30  # seconds before expiry to refresh URLs
+
+
+def get_part_num_size(file_size: int) -> tuple[int, int]:
+    minimum = math.ceil(file_size / PART_COUNT_MAX)
+    part_size = max(minimum, PART_SIZE_MIN)
+    part_size = min(part_size, PART_SIZE_MAX)
+    number_of_parts = math.ceil(file_size / part_size)
+    return number_of_parts, part_size
+
+
+def compute_checksums(file_path: str, part_size: int, number_of_parts: int) -> dict[int, tuple[bytes, str]]:
+    checksums: dict[int, tuple[bytes, str]] = {}
+    with open(file_path, "rb") as f:
+        for pn in range(1, number_of_parts + 1):
+            f.seek((pn - 1) * part_size)
+            data = f.read(part_size)
+            raw = hashlib.sha256(data).digest()
+            checksums[pn] = (raw, base64.b64encode(raw).decode())
+    return checksums
+
+
+def _upload_part(
+    part_number: int,
+    file_path: str,
+    part_size: int,
+    presigned_url: str,
+    checksum_b64: str,
+    urls_expire_at: float,
+    abort_event: Event,
+    url_expire_event: Event,
+) -> dict:
+    if abort_event.is_set():
+        return {"PartNumber": part_number, "Failed": True}
+    if url_expire_event.is_set() or time.time() >= urls_expire_at - URL_EXPIRE_BUFFER:
+        url_expire_event.set()
+        return {"Expired": True, "PartNumber": part_number}
+    try:
+        with open(file_path, "rb") as f:
+            f.seek((part_number - 1) * part_size)
+            data = f.read(part_size)
+
+        r = requests.put(
+            presigned_url,
+            data=data,
+            headers={
+                "x-amz-checksum-sha256": checksum_b64,
+                "x-amz-sdk-checksum-algorithm": "SHA256",
+            },
+        )
+        if abort_event.is_set():
+            return {"PartNumber": part_number, "Failed": True}
+        if r.status_code == 403:
+            url_expire_event.set()
+            return {"Expired": True, "PartNumber": part_number}
+        if not r.ok:
+            r.raise_for_status()
+    except Exception as e:
+        print(f"\n  [part {part_number} error] {type(e).__name__}: {e}")
+        return {"PartNumber": part_number, "Failed": True}
+    return {"PartNumber": part_number, "ETag": r.headers["ETag"], "ChecksumSHA256": checksum_b64}
+
+
+def upload_file(
+    file_path: str,
+    file_id: str,
+    api_client,
+    num_threads: int,
+    on_progress=None,
+) -> bool:
+    """Upload a single file. Returns True on success, False on failure."""
+    file_size = os.path.getsize(file_path)
+    if file_size > MAX_FILE_SIZE:
+        print(f"  ✗ Skipping {file_path} — file too large (max 48.8 TB)")
+        return False
+
+    number_of_parts, part_size = get_part_num_size(file_size)
+    try:
+        api_client.upload_start(file_id, total_parts=number_of_parts)
+    except ApiError as e:
+        if e.code == "cancelled":
+            raise UploadCancelledError()
+        raise
+    checksums = compute_checksums(file_path, part_size, number_of_parts)
+
+    # check whether file upload is cancelled on the frontend before going into the while loop
+    try:
+        if api_client.report_progress(file_id, 0):
+            raise UploadCancelledError()
+    except ApiError as e:
+        if e.code == "cancelled":
+            raise UploadCancelledError()
+        raise
+
+    unsorted: dict[int, dict] = {}
+    remaining = set(range(1, number_of_parts + 1))
+    completed: set[int] = set()
+    failed_counts: dict[int, int] = {}
+    num_attempt = 1
+
+    print("  Preparing...", end="", flush=True)
+    abort_event = Event()
+    stop_poll = Event()
+
+    def _poll_cancel():
+        while not stop_poll.wait(CANCEL_POLL_INTERVAL):
+            try:
+                cancelled = api_client.report_progress(file_id, len(completed))
+            except Exception:
+                continue
+            if cancelled:
+                abort_event.set()
+                break
+
+    poll_thread = Thread(target=_poll_cancel, daemon=True)
+    poll_thread.start()
+
+    try:
+        while remaining:
+            url_expire_event = Event()
+
+            parts_info = [
+                {"part_number": pn, "checksum_sha256": checksums[pn][1]}
+                for pn in sorted(remaining)
+            ]
+            try:
+                urls_dict, expires_at = api_client.get_presigned_urls(
+                    file_id, parts_info, url_expire_time=URL_EXPIRE_TIME * num_attempt
+                )
+            except ApiError as e:
+                if e.code == "cancelled":
+                    raise UploadCancelledError()
+                raise
+            urls = {int(k): v for k, v in urls_dict.items()}
+
+            pool = ThreadPoolExecutor(max_workers=num_threads)
+            try:
+                futures = [
+                    pool.submit(
+                        _upload_part, pn, file_path, part_size,
+                        urls[pn], checksums[pn][1], expires_at, abort_event, url_expire_event,
+                    )
+                    for pn in sorted(remaining)
+                ]
+                for future in as_completed(futures):
+                    if abort_event.is_set():
+                        pool.shutdown(wait=False, cancel_futures=True)
+                        raise UploadCancelledError()
+
+                    result = future.result()
+                    if result.get("Failed"):
+                        pn = result["PartNumber"]
+                        failed_counts[pn] = failed_counts.get(pn, 0) + 1
+                        if max(failed_counts.values()) >= MAX_RETRY:
+                            abort_event.set()
+                            pool.shutdown(wait=False, cancel_futures=True)
+                            api_client.abort_upload(file_id)
+                            return False
+                    elif result.get("Expired"):
+                        continue
+                    else:
+                        pn = result["PartNumber"]
+                        unsorted[pn] = result
+                        remaining.discard(pn)
+                        completed.add(pn)
+                        pct = len(completed) * 100 // number_of_parts
+                        if on_progress:
+                            cancelled = on_progress(len(completed), number_of_parts)
+                            if cancelled:
+                                abort_event.set()
+                                pool.shutdown(wait=False, cancel_futures=True)
+                                raise UploadCancelledError()
+                        filled = pct * 50 // 100
+                        bar = "█" * filled + "░" * (50 - filled)
+                        print(f"\r  |{bar}| {pct}%", end="", flush=True)
+            except KeyboardInterrupt:
+                abort_event.set()
+                pool.shutdown(wait=False, cancel_futures=True)
+                raise
+            else:
+                pool.shutdown(wait=True)
+
+            if abort_event.is_set():
+                raise UploadCancelledError()
+
+            num_attempt += 1
+    finally:
+        stop_poll.set()
+        poll_thread.join()
+
+    parts = [unsorted[pn] for pn in range(1, number_of_parts + 1)]
+    api_client.complete_upload(file_id, parts)
+
+    part_raw_digests = b"".join(checksums[pn][0] for pn in range(1, number_of_parts + 1))
+    composite = base64.b64encode(hashlib.sha256(part_raw_digests).digest()).decode()
+    local_checksum = f"{composite}-{number_of_parts}"
+    s3_checksum = api_client.verify_upload(file_id)
+    if local_checksum != s3_checksum:
+        raise RuntimeError(f"Checksum mismatch: local={local_checksum} s3={s3_checksum}")
+    print(f"\n  ✓ Integrity OK: {local_checksum}")
+
+    return True

cellismo_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: cellismo-cli
+Version: 0.1.0
+Summary: Cellismo file upload CLI
+Requires-Python: >=3.12
+Requires-Dist: click>=8.1
+Requires-Dist: requests>=2.32

cellismo_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+cellismo_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cellismo_cli/api_client.py,sha256=cL2F2bAUzFArbi6PrMF8hWreJfj8wmsgaaTdkzIlPTY,4205
+cellismo_cli/config.py,sha256=lGyswZzzqwif-Vn5H3jOhTec107xFaO0lBqiunFWR_0,637
+cellismo_cli/main.py,sha256=YhqFrGe5wX4qu6_5JQhbylwbPxkgn0hUOIhopG2iMYA,243
+cellismo_cli/uploader.py,sha256=S6KfnHLRASLeHW3g3RMoptLDiQRRLoWWkTCKoKt2aSs,8187
+cellismo_cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cellismo_cli/commands/login.py,sha256=lNLcv4x91PDDgABjdec3_oUQmogbz7MNlWHz8Ek1AsM,1047
+cellismo_cli/commands/upload.py,sha256=pZgxt6AT40OQsrexWmkGk1YLQ8DdMv3wwQIA34T1aX8,4154
+cellismo_cli-0.1.0.dist-info/METADATA,sha256=UisU2y8EO_ySaWclu-yNX2AxSVkgExUmv9Og3e_40oc,170
+cellismo_cli-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+cellismo_cli-0.1.0.dist-info/entry_points.txt,sha256=RxCGc2xpV1KW83nmjkHr-GIsJWoteA86yUTfV8iUPsg,51
+cellismo_cli-0.1.0.dist-info/top_level.txt,sha256=xAudVMyMor2VwwrGShFmilPtCtiRp0MN1dsxn74G1GE,13
+cellismo_cli-0.1.0.dist-info/RECORD,,

cellismo_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cellismo_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cellismo = cellismo_cli.main:cli

cellismo_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+cellismo_cli