streamlift-worker 1.1.0__tar.gz

streamlift_worker-1.1.0/MANIFEST.in

@@ -0,0 +1,17 @@
+# Only include what belongs in the published source distribution.
+# Everything not listed here is excluded — the rest of the monorepo never touches PyPI.
+
+include pyproject.toml
+include README.md
+include MANIFEST.in
+
+recursive-include streamlift_worker *.py
+
+# Explicitly exclude everything else (belt-and-suspenders)
+exclude .env
+exclude .env.example
+exclude .gitignore
+prune dist
+prune build
+prune .github
+prune downloads

streamlift_worker-1.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: streamlift-worker
+Version: 1.1.0
+Summary: StreamLift distributed download worker — runs in Google Colab
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: requests==2.32.3
+Requires-Dist: psutil==6.1.1
+Provides-Extra: dev
+Requires-Dist: ruff==0.9.10; extra == "dev"
+Requires-Dist: pytest==8.4.1; extra == "dev"
+Requires-Dist: build==1.2.2; extra == "dev"
+Requires-Dist: twine==6.1.0; extra == "dev"
+
+# streamlift-worker
+
+Distributed download worker for [StreamLift](https://github.com/shubhra92/StreamLift).  
+Designed to run in **Google Colab** — paste the one-liner from the dashboard and go.
+
+## What it does
+
+- Registers itself with the StreamLift backend
+- Polls for download tasks (HTTP URLs or magnet links)
+- Downloads files and streams them directly to **Mega** (zero local disk) or saves them **locally**
+- Reports live progress back to the backend
+- Sends heartbeats with CPU / RAM / network metrics
+
+## Install & run (Colab cell)
+
+The StreamLift UI generates this for you with all credentials pre-filled:
+
+```python
+!pip install -q requests psutil
+!pip install -q git+https://github.com/shubhra92/megapy.git
+!pip install -q git+https://github.com/shubhra92/streamlift-worker.git
+
+!streamlift-worker \
+  --worker-id     "your-worker-uuid" \
+  --auth-token    "your-auth-token" \
+  --api-url       "https://your-app.com" \
+  --compute-type  "medium" \
+  --location      "mega" \
+  --mega-email    "you@example.com" \
+  --mega-password "yourpassword"
+```
+
+## CLI flags
+
+| Flag | Required | Description |
+|------|----------|-------------|
+| `--worker-id` | ✅ | Worker UUID from the StreamLift dashboard |
+| `--auth-token` | ✅ | Auth token for this worker |
+| `--api-url` | ✅ | StreamLift backend base URL |
+| `--compute-type` | ✅ | Resource profile: `low` \| `medium` \| `high` |
+| `--location` | ✅ | Where to store files: `local` \| `mega` |
+| `--mega-email` | ⚠️ | Required when `--location=mega` |
+| `--mega-password` | ⚠️ | Required when `--location=mega` |
+
+## Package structure
+
+```
+streamlift_worker/
+├── __init__.py       # version
+├── __main__.py       # CLI entry point & arg parsing
+├── config.py         # WorkerConfig dataclass
+├── worker.py         # main loop, registration, heartbeat thread
+├── downloader.py     # HTTP + torrent download handlers
+├── mega.py           # Mega upload helpers (stream & file)
+├── api.py            # all HTTP calls to the backend
+├── metrics.py        # CPU / RAM / network metrics
+└── logger.py         # in-process log queue
+```

streamlift_worker-1.1.0/PUBLISHING.md

@@ -0,0 +1,104 @@
+# Publishing Guide — streamlift-worker
+
+## One-time PyPI Trusted Publishing setup
+
+Do this once. You never need to store an API token anywhere.
+
+1. Log in to [pypi.org](https://pypi.org)
+2. Go to **Your projects → streamlift-worker → Manage → Publishing**  
+   (If the project doesn't exist yet, go to **Account settings → Publishing** and add a pending publisher)
+3. Click **Add a new publisher** and fill in:
+
+   | Field | Value |
+   |-------|-------|
+   | PyPI project name | `streamlift-worker` |
+   | Owner | your GitHub username |
+   | Repository | `StreamLift` (your private repo name) |
+   | Workflow filename | `publish-python-worker.yml` |
+   | Environment name | `pypi` |
+
+4. On GitHub, go to **Settings → Environments → New environment** and name it `pypi`.  
+   No secrets needed — OIDC handles authentication automatically.
+
+---
+
+## How to release a new version
+
+```bash
+# 1. Make sure your changes are committed and on main
+git checkout main
+git pull
+
+# 2. Tag the release — this is what drives the version number
+git tag v1.2.0
+
+# 3. Push the tag — this triggers the GitHub Actions workflow
+git push origin v1.2.0
+```
+
+That's it. GitHub Actions will:
+- Check out the repo with full history
+- Build the wheel + sdist from `python-worker/` only
+- Publish to PyPI via OIDC (no token needed)
+
+The package will be live at `https://pypi.org/project/streamlift-worker/1.2.0/`  
+within ~2 minutes of pushing the tag.
+
+---
+
+## Version numbering
+
+The version is read directly from the Git tag by `setuptools-scm`:
+
+| Git tag | PyPI version |
+|---------|-------------|
+| `v1.0.0` | `1.0.0` |
+| `v1.2.3` | `1.2.3` |
+| `v2.0.0-rc1` | `2.0.0rc1` |
+
+**Do not set `version =` in `pyproject.toml`** — it's `dynamic` and will be wrong.
+
+---
+
+## Test locally before releasing
+
+```bash
+cd python-worker
+
+# Create a clean virtual environment
+python -m venv .venv
+source .venv/bin/activate
+
+# Install in editable mode with dev tools
+pip install -e ".[dev]"
+
+# Run a lint check
+ruff check streamlift_worker/
+
+# Build the package locally (output goes to dist/)
+python -m build
+
+# Inspect what's inside the wheel — make sure only streamlift_worker/ is there
+unzip -l dist/streamlift_worker-*.whl
+
+# Optional: do a dry-run upload to Test PyPI first
+pip install twine
+twine upload --repository testpypi dist/*
+# Then test install from Test PyPI:
+pip install --index-url https://test.pypi.org/simple/ streamlift-worker
+```
+
+---
+
+## What's public vs private
+
+| What | Visibility |
+|------|-----------|
+| Your GitHub repo (`StreamLift/`) | **Private** — only you |
+| `express-backend/`, `next-frontend/`, etc. | **Private** — never leaves GitHub |
+| `python-worker/streamlift_worker/*.py` | **Public** — included in the PyPI wheel |
+| `python-worker/README.md`, `pyproject.toml` | **Public** — included in the sdist |
+| `.env`, credentials, other configs | **Private** — excluded by `MANIFEST.in` |
+
+> PyPI publishes only the built distribution files (`.whl` + `.tar.gz`).  
+> It has no access to your GitHub repository at all.

streamlift_worker-1.1.0/README.md

@@ -0,0 +1,58 @@
+# streamlift-worker
+
+Distributed download worker for [StreamLift](https://github.com/shubhra92/StreamLift).  
+Designed to run in **Google Colab** — paste the one-liner from the dashboard and go.
+
+## What it does
+
+- Registers itself with the StreamLift backend
+- Polls for download tasks (HTTP URLs or magnet links)
+- Downloads files and streams them directly to **Mega** (zero local disk) or saves them **locally**
+- Reports live progress back to the backend
+- Sends heartbeats with CPU / RAM / network metrics
+
+## Install & run (Colab cell)
+
+The StreamLift UI generates this for you with all credentials pre-filled:
+
+```python
+!pip install -q requests psutil
+!pip install -q git+https://github.com/shubhra92/megapy.git
+!pip install -q git+https://github.com/shubhra92/streamlift-worker.git
+
+!streamlift-worker \
+  --worker-id     "your-worker-uuid" \
+  --auth-token    "your-auth-token" \
+  --api-url       "https://your-app.com" \
+  --compute-type  "medium" \
+  --location      "mega" \
+  --mega-email    "you@example.com" \
+  --mega-password "yourpassword"
+```
+
+## CLI flags
+
+| Flag | Required | Description |
+|------|----------|-------------|
+| `--worker-id` | ✅ | Worker UUID from the StreamLift dashboard |
+| `--auth-token` | ✅ | Auth token for this worker |
+| `--api-url` | ✅ | StreamLift backend base URL |
+| `--compute-type` | ✅ | Resource profile: `low` \| `medium` \| `high` |
+| `--location` | ✅ | Where to store files: `local` \| `mega` |
+| `--mega-email` | ⚠️ | Required when `--location=mega` |
+| `--mega-password` | ⚠️ | Required when `--location=mega` |
+
+## Package structure
+
+```
+streamlift_worker/
+├── __init__.py       # version
+├── __main__.py       # CLI entry point & arg parsing
+├── config.py         # WorkerConfig dataclass
+├── worker.py         # main loop, registration, heartbeat thread
+├── downloader.py     # HTTP + torrent download handlers
+├── mega.py           # Mega upload helpers (stream & file)
+├── api.py            # all HTTP calls to the backend
+├── metrics.py        # CPU / RAM / network metrics
+└── logger.py         # in-process log queue
+```

streamlift_worker-1.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=68", "setuptools-scm>=8", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "streamlift-worker"
+# Version is derived automatically from the Git tag (e.g. v1.1.0 → 1.1.0).
+# Do NOT set version here manually — setuptools-scm handles it.
+dynamic = ["version"]
+description = "StreamLift distributed download worker — runs in Google Colab"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "requests==2.32.3",
+    "psutil==6.1.1",
+    # mega.py fork with upload_stream support:
+    # pip install git+https://github.com/shubhra92/megapy.git
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff==0.9.10",
+    "pytest==8.4.1",
+    "build==1.2.2",
+    "twine==6.1.0",
+]
+
+[project.scripts]
+streamlift-worker = "streamlift_worker.__main__:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["streamlift_worker*"]
+
+# setuptools-scm: read version from the nearest Git tag
+# root ".." because pyproject.toml lives in python-worker/ but .git is at repo root
+[tool.setuptools_scm]
+root = ".."
+write_to = "streamlift_worker/_version.py"
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

streamlift_worker-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

streamlift_worker-1.1.0/streamlift_worker/__init__.py

@@ -0,0 +1,8 @@
+"""StreamLift Worker — distributed download agent for Google Colab."""
+
+try:
+    # Populated by setuptools-scm at build time from the Git tag.
+    from streamlift_worker._version import version as __version__
+except ImportError:
+    # Running from source without a build (e.g. direct git clone).
+    __version__ = "0.0.0.dev0"

streamlift_worker-1.1.0/streamlift_worker/__main__.py

@@ -0,0 +1,72 @@
+"""
+CLI entry point.
+
+Usage (from PyPI / GitHub):
+    streamlift-worker \\
+        --worker-id      "uuid" \\
+        --auth-token     "token" \\
+        --api-url        "https://your-app.com" \\
+        --compute-type   "medium" \\
+        --location       "mega" \\
+        --mega-email     "you@example.com" \\
+        --mega-password  "secret"
+
+Or run as a module:
+    python -m streamlift_worker ...same flags...
+"""
+
+import argparse
+import sys
+
+from streamlift_worker.config import WorkerConfig
+from streamlift_worker.worker import run
+
+
+def _parse_args() -> WorkerConfig:
+    p = argparse.ArgumentParser(
+        prog="streamlift-worker",
+        description="StreamLift distributed download worker",
+    )
+
+    p.add_argument("--worker-id",      required=True,  help="Worker UUID from StreamLift dashboard")
+    p.add_argument("--auth-token",     required=True,  help="Auth token for this worker")
+    p.add_argument("--api-url",        required=True,  help="StreamLift backend base URL (e.g. https://app.streamlift.io)")
+    p.add_argument("--compute-type",   required=True,  choices=["low", "medium", "high"],
+                   help="Resource profile: low | medium | high")
+    p.add_argument("--location",       required=True,  choices=["local", "mega"],
+                   dest="download_location", help="Where to store downloaded files: local | mega")
+    p.add_argument("--mega-email",     default="",     help="Mega account email (required when --location=mega)")
+    p.add_argument("--mega-password",  default="",     help="Mega account password (required when --location=mega)")
+    p.add_argument("--version",        action="version", version=f"%(prog)s {_get_version()}")
+
+    args = p.parse_args()
+
+    if args.download_location == "mega" and not (args.mega_email and args.mega_password):
+        p.error("--mega-email and --mega-password are required when --location=mega")
+
+    return WorkerConfig(
+        worker_id=         args.worker_id,
+        auth_token=        args.auth_token,
+        api_base_url=      args.api_url.rstrip("/"),
+        compute_type=      args.compute_type,
+        download_location= args.download_location,
+        mega_email=        args.mega_email,
+        mega_password=     args.mega_password,
+    )
+
+
+def _get_version() -> str:
+    try:
+        from streamlift_worker import __version__
+        return __version__
+    except Exception:
+        return "unknown"
+
+
+def main() -> None:
+    config = _parse_args()
+    run(config)
+
+
+if __name__ == "__main__":
+    main()

streamlift_worker-1.1.0/streamlift_worker/api.py

@@ -0,0 +1,148 @@
+"""
+HTTP communication with the StreamLift backend.
+All outbound calls live here — the rest of the package never calls requests directly.
+"""
+
+import sys
+import threading
+import time
+from typing import Any, Optional
+
+import requests
+
+from streamlift_worker import logger
+from streamlift_worker.config import MAX_RETRIES, RETRY_DELAY, WorkerConfig
+
+# Shared stop event — set by the auth-failure handler so every loop exits cleanly
+stop_event = threading.Event()
+
+
+def _post(config: WorkerConfig, endpoint: str, data: dict[str, Any]) -> Optional[dict]:
+    url     = f"{config.api_base_url}{endpoint}"
+    headers = {"Content-Type": "application/json"}
+
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            r = requests.post(url, json=data, headers=headers, timeout=30)
+
+            if r.status_code == 401:
+                logger.log("error", "Authentication failed — invalid worker ID or auth token. Stopping.")
+                stop_event.set()
+                sys.exit(1)
+
+            if r.status_code >= 500:
+                raise requests.RequestException(f"Server error {r.status_code}")
+
+            return r.json()
+
+        except requests.RequestException as e:
+            logger.log("warning", f"POST {endpoint} failed (attempt {attempt}/{MAX_RETRIES}): {e}")
+            if attempt < MAX_RETRIES:
+                time.sleep(RETRY_DELAY * attempt)
+
+    logger.log("error", f"POST {endpoint} gave up after {MAX_RETRIES} attempts")
+    return None
+
+
+def _get(config: WorkerConfig, endpoint: str, params: dict[str, Any] | None = None) -> Optional[dict]:
+    url = f"{config.api_base_url}{endpoint}"
+    try:
+        r = requests.get(url, params=params, timeout=15)
+        if r.status_code == 200:
+            return r.json()
+    except requests.RequestException as e:
+        logger.log("warning", f"GET {endpoint} failed: {e}")
+    return None
+
+
+# ── Worker lifecycle ──────────────────────────────────────────────────────────
+
+def register(config: WorkerConfig, ip_address: str) -> bool:
+    result = _post(config, "/api/worker/register", {
+        "workerId":  config.worker_id,
+        "authToken": config.auth_token,
+        "ipAddress": ip_address,
+        "version":   config.worker_version,
+    })
+    if result and result.get("success"):
+        logger.log("info", f"Registered successfully. Public IP: {ip_address}")
+        return True
+    logger.log("error", f"Registration failed. Response: {result}")
+    return False
+
+
+def heartbeat(config: WorkerConfig, current_task: Optional[dict], metrics: dict) -> Optional[dict]:
+    payload: dict[str, Any] = {
+        "workerId":  config.worker_id,
+        "authToken": config.auth_token,
+        "metrics":   metrics,
+    }
+    if current_task:
+        payload["currentTask"] = current_task
+    return _post(config, "/api/worker/heartbeat", payload)
+
+
+def flush_logs(config: WorkerConfig) -> None:
+    if logger.pending_count() == 0:
+        return
+    batch = logger.pop_batch(10)
+    result = _post(config, "/api/worker/logs", {
+        "workerId":  config.worker_id,
+        "authToken": config.auth_token,
+        "logs":      batch,
+    })
+    # If the call failed, put the batch back at the front so we retry next time
+    if not (result and result.get("success")):
+        from streamlift_worker.logger import _queue
+        _queue[:0] = batch
+
+
+def report_progress(
+    config: WorkerConfig,
+    download_id: str,
+    downloaded: int,
+    total: int,
+    status: str,
+    file_name: str = "",
+    error_msg:  str = "",
+) -> None:
+    pct = round(downloaded / total * 100, 2) if total > 0 else 0
+    payload: dict[str, Any] = {
+        "workerId":   config.worker_id,
+        "authToken":  config.auth_token,
+        "downloadId": download_id,
+        "progress": {
+            "downloadedBytes": downloaded,
+            "totalBytes":      total,
+            "percent":         pct,
+            "status":          status,
+        },
+    }
+    if error_msg:
+        payload["progress"]["errorMessage"] = error_msg
+    _post(config, "/api/worker/download-progress", payload)
+
+
+# ── Mega session persistence ──────────────────────────────────────────────────
+
+def save_mega_session(config: WorkerConfig, session_id: str) -> None:
+    try:
+        _post(config, "/api/worker/mega-session", {
+            "workerId":    config.worker_id,
+            "authToken":   config.auth_token,
+            "email":       config.mega_email,
+            "sessionData": session_id,
+        })
+        logger.log("info", "Mega session saved to backend")
+    except Exception as e:
+        logger.log("warning", f"Could not save Mega session: {e}")
+
+
+def load_mega_session(config: WorkerConfig) -> Optional[str]:
+    data = _get(config, "/api/worker/mega-session", {
+        "workerId":  config.worker_id,
+        "authToken": config.auth_token,
+    })
+    if data and data.get("success") and data.get("sessionData"):
+        return data["sessionData"]
+    return None

streamlift_worker-1.1.0/streamlift_worker/config.py

@@ -0,0 +1,38 @@
+"""
+Worker configuration — populated entirely from CLI args.
+No .env file needed; the user just runs the one-liner from the UI.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+
+ComputeType      = Literal["low", "medium", "high"]
+DownloadLocation = Literal["local", "mega"]
+
+COMPUTE_CONFIG = {
+    "low":    {"max_cpu_pct": 25,  "chunk_size": 512  * 1024},   # 512 KB
+    "medium": {"max_cpu_pct": 50,  "chunk_size": 1024 * 1024},   # 1 MB
+    "high":   {"max_cpu_pct": 100, "chunk_size": 2048 * 1024},   # 2 MB
+}
+
+POLL_INTERVAL      = 10   # seconds between main-loop polls
+MAX_RETRIES        = 3
+RETRY_DELAY        = 5    # seconds, multiplied by attempt number
+MAX_LOG_QUEUE      = 50
+HEARTBEAT_INTERVAL = 8    # background heartbeat thread interval (seconds)
+
+
+@dataclass
+class WorkerConfig:
+    worker_id:         str
+    auth_token:        str
+    api_base_url:      str
+    compute_type:      ComputeType
+    download_location: DownloadLocation
+    mega_email:        str = ""
+    mega_password:     str = ""
+    worker_version:    str = "1.1.0"
+
+    @property
+    def chunk_size(self) -> int:
+        return COMPUTE_CONFIG[self.compute_type]["chunk_size"]