sideloader 2.0.0__py3-none-any.whl

sideloader/server.py

@@ -0,0 +1,379 @@
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import threading
+import time
+
+import requests
+
+from sideloader.jsonbin_connector import JSONBinConnector
+
+JSONBIN_TOKEN = os.environ["JSONBIN_TOKEN"]
+PYPI_TOKEN = os.environ["PYPI_TOKEN"]
+MAX_PACKAGE_SIZE = 92 * 1024 * 1024  # 95 MB
+
+LAST_BINS: dict[str, str | None] = {}
+
+PYPROJECT_TEMPLATE = """
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "{package_name}"
+version = "1.0.0"
+description = "Sideloaded package"
+requires-python = ">=3.8"
+authors = [
+  {{name = "Null Void" }}
+]
+
+[tool.setuptools.data-files]
+"share/{package_name}" = ["{package_name}"]
+"""
+
+jsonbin_connector = JSONBinConnector()
+
+
+def package_build(directory: Path) -> bool:
+    result = subprocess.run(
+        ["python3", "-m", "build", "--wheel"], cwd=str(directory), check=True
+    )
+    return result.returncode == 0
+
+
+def twine_upload(directory: Path):
+    result = subprocess.run(
+        [
+            "twine",
+            "upload",
+            "dist/*",
+            "-u",
+            "__token__",
+            "-p",
+            PYPI_TOKEN,
+        ],
+        cwd=str(directory),
+        check=True,
+    )
+    return result.returncode == 0
+
+
+def download_file(bin_id: str, url: str):
+    try:
+        # Send a HTTP request to the server.
+        response = requests.get(url, stream=True)
+    except Exception as e:
+        jsonbin_connector.update_bin(
+            bin_id,
+            {
+                "status": "REJECTED",
+                "details": f"Failed to download file: [{e.__class__.__name__}]: {e}",
+            },
+        )
+        return
+    if not response.ok:
+        jsonbin_connector.update_bin(
+            bin_id,
+            {
+                "status": "REJECTED",
+                "details": f"URL returned code {response.status_code}: {response.reason}",
+            },
+        )
+        return
+    # Total size in bytes.
+    total_size = int(response.headers.get("content-length", 1))
+    try:
+        content_disposition = response.headers.get("Content-Disposition", "")
+        filename = None
+
+        # Try to extract filename from Content-Disposition header
+        # Format can be: attachment; filename="file.ext"; filename*=utf-8''encoded%20name.ext
+        if content_disposition:
+            import re
+            from urllib.parse import unquote
+
+            # First try filename*= (RFC 5987 encoded, preferred)
+            match = re.search(
+                r"filename\*=(?:utf-8''|UTF-8'')([^;]+)", content_disposition
+            )
+            if match:
+                filename = unquote(match.group(1).strip())
+            else:
+                # Fall back to filename= (may be quoted)
+                match = re.search(
+                    r'filename=(?:"([^"]+)"|([^;\s]+))', content_disposition
+                )
+                if match:
+                    filename = match.group(1) or match.group(2)
+                    filename = filename.strip('"').strip()
+
+        if not filename:
+            raise ValueError("No filename found in Content-Disposition")
+    except Exception:
+        filename = response.url.removesuffix("/").split("/")[-1]
+        # URL decode and clean up the filename
+        from urllib.parse import unquote
+
+        filename = unquote(filename)
+
+    # Initialize variables to track progress.
+    downloaded = 0
+    chunk_size = 1024 * 1024  # Size of each chunk in bytes.
+    last_progress = 0
+    filename_root = filename.split(".")[0]
+    package_name = f"sideload_{filename_root}_bin_{bin_id}"
+    # replace all non-alphanumeric characters with an underscore
+    package_name = "".join(c if c.isalnum() else "_" for c in package_name)
+    parts: list[Path] = []
+
+    def make_part_name():
+        return f"{package_name}_p{len(parts)}"
+
+    def make_new_part():
+        part_name = make_part_name()
+        part_directory = Path(temp_dir) / package_name / part_name
+        part_directory.mkdir(parents=True, exist_ok=False)
+        part_path = part_directory / part_name
+        parts.append(part_path)
+        return open(part_path, "wb")
+
+    # Open a local file for writing in binary mode.
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # temp_dir = "./dumptmp3"  # only for debugging
+        os.mkdir(os.path.join(temp_dir, package_name))
+        jsonbin_connector.update_bin(
+            bin_id,
+            {"status": "DOWNLOADING", "progress": 0},
+        )
+        current_part_fp = make_new_part()
+        try:
+            current_chunk_size = 0
+            for data in response.iter_content(chunk_size=chunk_size):
+                if current_chunk_size + len(data) > MAX_PACKAGE_SIZE:
+                    current_part_fp.close()
+                    current_part_fp = make_new_part()
+                    current_chunk_size = 0
+                current_part_fp.write(data)
+                downloaded += len(data)
+                current_chunk_size += len(data)
+                if total_size < downloaded:
+                    total_size = downloaded
+                    progress = 99
+                else:
+                    progress = int((downloaded / total_size) * 100)
+                if progress != last_progress:
+                    jsonbin_connector.update_bin(bin_id, {"progress": progress})
+                    last_progress = progress
+        finally:
+            current_part_fp.close()
+        jsonbin_connector.update_bin(bin_id, {"progress": 100, "status": "DOWNLOADED"})
+        for part_idx, path_part in enumerate(parts):
+            with open(
+                path_part.parent / "pyproject.toml",
+                "w",
+                encoding="utf-8",
+            ) as pyproject_file:
+                pyproject_file.write(
+                    PYPROJECT_TEMPLATE.format(package_name=path_part.name)
+                )
+
+            jsonbin_connector.update_bin(
+                bin_id,
+                {
+                    "status": "BUILDING",
+                    "details": f"Building package part {part_idx}/{len(parts)}.",
+                },
+            )
+            if not package_build(path_part.parent):
+                jsonbin_connector.update_bin(
+                    bin_id,
+                    {
+                        "status": "BULDING",
+                        "details": f"Failed to build package part {part_idx}/{len(parts)}.",
+                    },
+                )
+                return
+            jsonbin_connector.update_bin(
+                bin_id,
+                {
+                    "status": "UPLOADING",
+                    "details": f"Uploading package part {part_idx}/{len(parts)}.",
+                },
+            )
+            if not twine_upload(path_part.parent):
+                jsonbin_connector.update_bin(
+                    bin_id,
+                    {
+                        "status": "FAILED",
+                        "details": f"Failed to upload package part {part_idx}/{len(parts)}.",
+                    },
+                )
+                return
+        jsonbin_connector.update_bin(
+            bin_id,
+            {
+                "status": "UPLOADED",
+                "packages_names": [path_part.name for path_part in parts],
+                "filename": filename,
+                "file_size": total_size,
+                "total_packages": len(parts),
+            },
+        )
+
+
+def process_bin(bin_id: str):
+    url = f"https://api.jsonbin.io/v3/b/{bin_id}"
+    bin_data = requests.get(url, headers={"X-Master-Key": JSONBIN_TOKEN}).json()
+    bin_record = bin_data["record"]
+    if bin_record["status"] == "CREATED":
+        print("Processing bin:", bin_id)
+        download_file(bin_id, bin_record["url"])
+    elif bin_record["status"] != "UPLOADED":
+        jsonbin_connector.update_bin(
+            bin_id, {"status": "FAILED", "details": "Server interruption"}
+        )
+    else:
+        print("Bin already processed:", bin_id)
+
+
+def watch_collection(collection_id: str):
+    print("Watching collection:", collection_id)
+    while True:
+        collection_data = jsonbin_connector.get_collection_bins(
+            collection_id, LAST_BINS.get(collection_id)
+        )
+        last_bin: str | None = None
+        for bin_data in collection_data:
+            bin_id = bin_data["record"]
+            process_bin(bin_id)
+            last_bin = bin_id
+        if last_bin is not None:
+            LAST_BINS[collection_id] = last_bin
+        time.sleep(3)
+
+
+# Statuses that indicate a bin can be cleaned up
+CLEANUP_STATUSES = {"UPLOADED", "FAILED", "REJECTED"}
+# Max age for bins in CREATED status (considered dead/stale) - 24 hours
+MAX_CREATED_AGE_SECONDS = 24 * 60 * 60
+
+
+def cleanup_collection(collection_id: str) -> tuple[int, int]:
+    """
+    Clean up finished and dead bins from a collection.
+    Handles pagination to process ALL bins.
+
+    Returns:
+        Tuple of (deleted_count, error_count)
+    """
+    deleted = 0
+    errors = 0
+    current_time = time.time()
+    last_bin_id: str | None = None
+
+    try:
+        while True:
+            # Get bins in the collection (paginated)
+            collection_data = jsonbin_connector.get_collection_bins(
+                collection_id, last_bin_id
+            )
+
+            if not collection_data:
+                break  # No more bins
+
+            for bin_data in collection_data:
+                bin_id = bin_data["record"]
+                last_bin_id = bin_id  # Track for pagination
+
+                try:
+                    bin_record = jsonbin_connector.get_bin(bin_id)
+                    status = bin_record.get("status", "UNKNOWN")
+                    created_at = bin_record.get("created_at", 0)
+
+                    should_delete = False
+                    reason = ""
+
+                    # Delete finished/failed bins
+                    if status in CLEANUP_STATUSES:
+                        should_delete = True
+                        reason = f"status={status}"
+                    # Delete stale CREATED bins (stuck/dead requests)
+                    elif status == "CREATED" and created_at > 0:
+                        age = current_time - created_at
+                        if age > MAX_CREATED_AGE_SECONDS:
+                            should_delete = True
+                            reason = f"stale CREATED (age={age / 3600:.1f}h)"
+
+                    if should_delete:
+                        print(f"  Deleting bin {bin_id}: {reason}")
+                        jsonbin_connector.delete_bin(bin_id)
+                        deleted += 1
+
+                except Exception as e:
+                    print(f"  Error processing bin {bin_id}: {e}")
+                    errors += 1
+
+    except Exception as e:
+        print(f"  Error fetching collection bins: {e}")
+        errors += 1
+
+    return deleted, errors
+
+
+def cleanup_all_collections():
+    """Clean up all sideload collections on startup."""
+    print("🧹 Cleaning up old bins...")
+    total_deleted = 0
+    total_errors = 0
+
+    collections = jsonbin_connector.get_collections()
+    for collection in collections:
+        if collection["collectionMeta"]["name"].startswith("sideload_"):
+            collection_id = collection["record"]
+            collection_name = collection["collectionMeta"]["name"]
+            print(f"  Cleaning collection: {collection_name} ({collection_id})")
+
+            deleted, errors = cleanup_collection(collection_id)
+            total_deleted += deleted
+            total_errors += errors
+
+    print(f"✅ Cleanup complete: {total_deleted} bins deleted, {total_errors} errors")
+    return total_deleted, total_errors
+
+
+def server_main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Sideload server")
+    parser.add_argument(
+        "--request-id",
+        type=str,
+        default=None,
+        help="Process a single request by JSONBin ID and exit",
+    )
+    args = parser.parse_args()
+
+    if args.request_id:
+        # Single-request mode: process one request and exit
+        print(f"🎯 Processing single request: {args.request_id}")
+        process_bin(args.request_id)
+        print(f"✅ Finished processing request: {args.request_id}")
+        return
+
+    # Polling mode: clean up and watch all collections
+    cleanup_all_collections()
+
+    print("🚀 Starting sideload server...")
+    collections = jsonbin_connector.get_collections()
+    for collection in collections:
+        if collection["collectionMeta"]["name"].startswith("sideload_"):
+            collection_id = collection["record"]
+            collection_name = collection["collectionMeta"]["name"]
+            print(f"  Watching collection: {collection_name} ({collection_id})")
+            threading.Thread(target=watch_collection, args=(collection_id,)).start()
+
+
+if __name__ == "__main__":
+    server_main()

sideloader-2.0.0.dist-info/METADATA

@@ -0,0 +1,140 @@
+Metadata-Version: 2.3
+Name: sideloader
+Version: 2.0.0
+Summary: Download large files via PyPI packages
+Author: Sygmei
+Author-email: Sygmei <3835355+Sygmei@users.noreply.github.com>
+Requires-Dist: build>=1.3.0
+Requires-Dist: twine>=6.2.0
+Requires-Dist: wheel>=0.45.1
+Requires-Dist: rich>=13.0.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pip>=25.2
+Requires-Dist: playwright>=1.55.0
+Requires-Dist: pyotp>=2.9.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# 🚀 Sideload
+
+Download large files via PyPI packages! Sideload automatically splits large files into PyPI-compliant chunks and allows you to download them through a beautiful CLI interface.
+
+## Features
+
+- ✨ **Beautiful CLI** powered by Rich with progress bars and status updates
+- 📦 **Automatic file splitting** into 99MB PyPI-compliant chunks
+- 🔄 **Automatic reassembly** of downloaded parts
+- 🌐 **JSONBin integration** for request tracking
+- 🛡️ **Error handling** with detailed progress monitoring
+
+## Installation
+
+```bash
+git clone <repository-url>
+cd Sideload
+uv install
+```
+
+## Usage
+
+### Environment Variables
+
+Set up your credentials:
+
+```bash
+export JSONBIN_TOKEN="your_jsonbin_token"
+export SIDELOAD_COLLECTION_ID="your_collection_id"
+```
+
+### Download a File
+
+```bash
+# Basic usage
+uv run sideload download https://example.com/largefile.zip
+
+# Specify output directory
+uv run sideload download https://example.com/largefile.zip --output ./downloads/
+
+# Override credentials
+uv run sideload download https://example.com/largefile.zip --token YOUR_TOKEN --collection YOUR_COLLECTION
+```
+
+### How it Works
+
+1. **Submit Request**: The CLI creates a new request in your JSONBin collection
+2. **Monitor Progress**: Real-time progress monitoring with beautiful progress bars
+3. **Download Packages**: Automatically downloads all PyPI packages containing file parts
+4. **Reassemble**: Extracts and concatenates parts to rebuild the original file
+
+### CLI Interface
+
+The CLI provides:
+
+- 🌈 **Colorful output** with status indicators
+- 📊 **Progress bars** for downloads and processing
+- 📈 **Real-time monitoring** of server-side processing
+- ✅ **Success/error reporting** with detailed information
+- 📋 **Summary tables** showing download statistics
+
+### Example Output
+
+```
+🚀 SIDELOAD
+Download large files via PyPI packages
+
+🌐 Requesting download for: https://example.com/largefile.zip
+✅ Created sideload request: abc123def456
+
+📡 Monitoring Progress
+📥 Downloading... (45%) ████████████████▌                 
+🔨 Building packages...
+📤 Uploading part 1/3...
+
+📊 Download Summary
+┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃ Property          ┃ Value                           ┃
+┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ Original Filename │ largefile.zip                   │
+│ File Size         │ 250,123,456 bytes               │
+│ Total Packages    │ 3                               │
+│ Status            │ ✅ UPLOADED                     │
+└───────────────────┴─────────────────────────────────┘
+
+📦 Downloading Packages
+📦 Downloading package 1/3... ████████████████████████████
+
+🔧 Reassembling File
+🔗 Assembling part 1/3... ████████████████████████████
+
+✨ Complete
+🎉 File successfully downloaded to: largefile.zip
+📊 File size: 250,123,456 bytes
+```
+
+## Development
+
+### Server Setup
+
+The server component handles file processing and PyPI uploads:
+
+```bash
+# Set environment variables
+export JSONBIN_TOKEN="your_token"
+export PYPI_TOKEN="your_pypi_token"
+
+# Run the server
+uv run python src/sideload/main.py
+```
+
+### Project Structure
+
+```
+src/sideload/
+├── __init__.py     # Package initialization
+├── main.py         # Server component
+└── cli.py          # CLI client
+```
+
+## License
+
+[Your License Here]

sideloader-2.0.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+sideloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sideloader/cli.py,sha256=2rLvz2N1BsEZOtCoXopIUAl5mIeFgerOujuu48uETPA,37994
+sideloader/jsonbin_connector.py,sha256=rgaynojjVT3zK-me_IGssWMDt43w98IOnH8yLBX_HOk,10014
+sideloader/scripts/cleanup_pypi.py,sha256=YPzYVsutFVsRoCi4qG6vZo8x5YiLzpAfJ7xKxZiZWgI,15264
+sideloader/server.py,sha256=qUh4YUSKyGCzlzx_I5e2T8P-PmFFpr93sx8qx6yUfNU,12753
+sideloader-2.0.0.dist-info/WHEEL,sha256=Jb20R3Ili4n9P1fcwuLup21eQ5r9WXhs4_qy7VTrgPI,79
+sideloader-2.0.0.dist-info/entry_points.txt,sha256=XIfbsCo-UMQvQwAuocZo7d7o9YMJ6yNy1P2sK5OeGA0,106
+sideloader-2.0.0.dist-info/METADATA,sha256=o8S-kWlUipN5n6iKw_Vj62gJagMQFeMyeytiDVB584c,4311
+sideloader-2.0.0.dist-info/RECORD,,

sideloader-2.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.8.15
+Root-Is-Purelib: true
+Tag: py3-none-any

sideloader-2.0.0.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+sideloader = sideloader.cli:cli_main
+sideloader-server = sideloader.server:server_main
+