noaa-jetstream 0.1.2__tar.gz

noaa_jetstream-0.1.2/LICENSE.md

@@ -0,0 +1 @@
+Software code created by U.S. Government employees is not subject to copyright in the United States (17 U.S.C. §105). The United States/Department of Commerce reserve all rights to seek and obtain copyright protection in countries other than the United States for Software authored in its entirety by the Department of Commerce. To this end, the Department of Commerce hereby grants to Recipient a royalty-free, nonexclusive license to use, copy, and create derivative works of the Software outside of the United States.

noaa_jetstream-0.1.2/MANIFEST.in

@@ -0,0 +1,19 @@
+# Include documentation
+include README.md
+include LICENSE.md
+include requirements.txt
+
+# Include static web files from package
+recursive-include jetstream/static *.html *.css *.js
+
+# Include logs directory structure (empty)
+recursive-include logs .gitkeep
+
+# Exclude unnecessary files
+global-exclude __pycache__
+global-exclude *.py[co]
+global-exclude .DS_Store
+global-exclude *.swp
+global-exclude .gitignore
+global-exclude *.db
+global-exclude *.log

noaa_jetstream-0.1.2/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: noaa-jetstream
+Version: 0.1.2
+Summary: JetStream: Cloud Data Manager - A comprehensive tool for managing local-to-cloud uploads with queue management, statistics, and folder analysis
+Home-page: https://github.com/MichaelAkridge-NOAA/jetstream
+Author: Michael Akridge
+Author-email: 
+License: MIT
+Project-URL: Homepage, https://github.com/MichaelAkridge-NOAA/jetstream
+Project-URL: Documentation, https://github.com/MichaelAkridge-NOAA/jetstream#readme
+Project-URL: Repository, https://github.com/MichaelAkridge-NOAA/jetstream
+Project-URL: Bug Tracker, https://github.com/MichaelAkridge-NOAA/jetstream/issues
+Keywords: cloud,upload,gcs,google-cloud,data-management,queue,noaa
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: fastapi>=0.109.0
+Requires-Dist: fastapi-cli>=0.0.20
+Requires-Dist: uvicorn[standard]>=0.27.0
+Requires-Dist: python-multipart>=0.0.6
+Requires-Dist: sqlalchemy>=2.0.25
+Requires-Dist: alembic>=1.13.0
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: pydantic-settings>=2.1.0
+Requires-Dist: aiofiles>=23.2.0
+Requires-Dist: python-dotenv>=1.0.0
+Provides-Extra: cloud
+Requires-Dist: google-cloud-storage>=2.14.0; extra == "cloud"
+Requires-Dist: google-auth>=2.27.0; extra == "cloud"
+Requires-Dist: google-auth-oauthlib>=1.2.0; extra == "cloud"
+Provides-Extra: shortcuts
+Requires-Dist: pyshortcuts>=1.9.0; extra == "shortcuts"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: noaa-jetstream[cloud,dev,shortcuts]; extra == "all"
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# NOAA JetStream — Cloud Data Management Transfer System
+<img align="right" src="./docs/jetstream_logo_400px.png" alt="jetstream" width="250">
+
+A comprehensive web-based application for managing Google Cloud Storage uploads with features including job queuing, real-time analytics, cloud bucket analysis, and batch processing capabilities.
+
+### Features
+
+- **Upload Management**
+- **Analytics & Monitoring**
+- **Cloud Bucket Analysis**
+- **File Filtering**
+- **Web Dashboard**
+
+## Screenshots
+
+| Dashboard | Upload Jobs | Analytics |
+|-----------|-------------|----------|
+| ![Home](./docs/screenshot_home.png) | ![Uploads](./docs/screenshot_uploads.png) | ![Analytics](./docs/screenshot_analytics.png) |
+
+### Prerequisites
+
+- **Python 3.9+**
+- **Google Cloud SDK** (includes gsutil) — for cloud upload features
+- **Permissions** to target GCS buckets
+
+**OR** use Docker (see [Docker Installation](#docker-installation) below)
+
+---
+
+### Installation
+
+#### Option 1: Install from PyPI (Recommended)
+
+```bash
+# Install core package
+pip install noaa-jetstream
+
+# Or install with Google Cloud support
+pip install noaa-jetstream[cloud]
+
+# Or install with desktop shortcuts
+pip install noaa-jetstream[shortcuts]
+
+# Or install everything
+pip install noaa-jetstream[all]
+```
+
+#### Option 2: Install from Source (Development)
+
+```bash
+# Clone the repository
+git clone https://github.com/MichaelAkridge-NOAA/jetstream.git
+cd jetstream
+
+# Install in development mode
+pip install -e .
+
+# Or with all optional features
+pip install -e ".[all]"
+```
+---
+
+### Google Cloud Setup (Optional)
+
+Required only for cloud upload features:
+
+```bash
+# Install Google Cloud SDK
+# Download from: https://cloud.google.com/sdk/docs/install
+
+# Authenticate
+gcloud auth login --no-launch-browser
+gcloud auth application-default login --no-launch-browser
+
+# Verify access (optional)
+gsutil ls
+gcloud auth list
+```
+
+---
+
+## Starting the Application
+
+### If Installed via pip
+
+```bash
+# Start the server (opens browser automatically)
+jetstream
+
+# With custom options
+jetstream --port 9000
+jetstream --host 127.0.0.1 --port 8080
+jetstream --no-browser
+jetstream --log-level debug
+```
+
+### If Running from Source
+
+```bash
+# Using the CLI
+python main.py
+
+# Or with the diagnostic startup script
+python start.py
+
+# Or directly with uvicorn
+python -m uvicorn jetstream.main:app --reload
+```
+
+The application will start on **http://localhost:8000** and automatically open in your default browser.
+
+### Desktop Shortcuts
+
+Create desktop and Start Menu shortcuts for easy access:
+
+```bash
+# Install with shortcuts support
+pip install noaa-jetstream[shortcuts]
+
+# Create shortcuts
+jetstream-create-shortcuts
+
+# Remove shortcuts
+jetstream-remove-shortcuts
+```
+
+### Troubleshooting Startup Issues
+
+**If the server appears to start but you can't connect:**
+
+1. **Run diagnostics:**
+   ```bash
+   python diagnose.py
+   ```
+   
+2. **Run with debug logging:**
+   ```bash
+   jetstream --log-level debug
+   # or from source:
+   python -m uvicorn jetstream.main:app --reload --log-level debug
+   ```
+---
+
+## Troubleshooting
+
+**Cannot connect to GCS:**
+- Verify authentication: `gcloud auth list`
+- Check bucket permissions
+- Ensure Application Default Credentials are set
+
+**Jobs stuck in queue:**
+- Check queue status in dashboard
+- Verify no jobs are blocking the queue
+- Restart the application if needed
+
+**Database errors:**
+- Delete `jetstream.db` to reset (loses history)
+- Check file permissions in application directory
+
+**API not responding:**
+- Check if port 8000 is already in use
+- View logs in terminal for error messages
+- Ensure all dependencies are installed
+
+----------
+#### Disclaimer
+This repository is a scientific product and is not official communication of the National Oceanic and Atmospheric Administration, or the United States Department of Commerce. All NOAA GitHub project content is provided on an 'as is' basis and the user assumes responsibility for its use. Any claims against the Department of Commerce or Department of Commerce bureaus stemming from the use of this GitHub project will be governed by all applicable Federal law. Any reference to specific commercial products, processes, or services by service mark, trademark, manufacturer, or otherwise, does not constitute or imply their endorsement, recommendation or favoring by the Department of Commerce. The Department of Commerce seal and logo, or the seal and logo of a DOC bureau, shall not be used in any manner to imply endorsement of any commercial product or activity by DOC or the United States Government.
+
+## License
+See the [LICENSE.md](./LICENSE.md) for details

noaa_jetstream-0.1.2/README.md

@@ -0,0 +1,170 @@
+# NOAA JetStream — Cloud Data Management Transfer System
+<img align="right" src="./docs/jetstream_logo_400px.png" alt="jetstream" width="250">
+
+A comprehensive web-based application for managing Google Cloud Storage uploads with features including job queuing, real-time analytics, cloud bucket analysis, and batch processing capabilities.
+
+### Features
+
+- **Upload Management**
+- **Analytics & Monitoring**
+- **Cloud Bucket Analysis**
+- **File Filtering**
+- **Web Dashboard**
+
+## Screenshots
+
+| Dashboard | Upload Jobs | Analytics |
+|-----------|-------------|----------|
+| ![Home](./docs/screenshot_home.png) | ![Uploads](./docs/screenshot_uploads.png) | ![Analytics](./docs/screenshot_analytics.png) |
+
+### Prerequisites
+
+- **Python 3.9+**
+- **Google Cloud SDK** (includes gsutil) — for cloud upload features
+- **Permissions** to target GCS buckets
+
+**OR** use Docker (see [Docker Installation](#docker-installation) below)
+
+---
+
+### Installation
+
+#### Option 1: Install from PyPI (Recommended)
+
+```bash
+# Install core package
+pip install noaa-jetstream
+
+# Or install with Google Cloud support
+pip install noaa-jetstream[cloud]
+
+# Or install with desktop shortcuts
+pip install noaa-jetstream[shortcuts]
+
+# Or install everything
+pip install noaa-jetstream[all]
+```
+
+#### Option 2: Install from Source (Development)
+
+```bash
+# Clone the repository
+git clone https://github.com/MichaelAkridge-NOAA/jetstream.git
+cd jetstream
+
+# Install in development mode
+pip install -e .
+
+# Or with all optional features
+pip install -e ".[all]"
+```
+---
+
+### Google Cloud Setup (Optional)
+
+Required only for cloud upload features:
+
+```bash
+# Install Google Cloud SDK
+# Download from: https://cloud.google.com/sdk/docs/install
+
+# Authenticate
+gcloud auth login --no-launch-browser
+gcloud auth application-default login --no-launch-browser
+
+# Verify access (optional)
+gsutil ls
+gcloud auth list
+```
+
+---
+
+## Starting the Application
+
+### If Installed via pip
+
+```bash
+# Start the server (opens browser automatically)
+jetstream
+
+# With custom options
+jetstream --port 9000
+jetstream --host 127.0.0.1 --port 8080
+jetstream --no-browser
+jetstream --log-level debug
+```
+
+### If Running from Source
+
+```bash
+# Using the CLI
+python main.py
+
+# Or with the diagnostic startup script
+python start.py
+
+# Or directly with uvicorn
+python -m uvicorn jetstream.main:app --reload
+```
+
+The application will start on **http://localhost:8000** and automatically open in your default browser.
+
+### Desktop Shortcuts
+
+Create desktop and Start Menu shortcuts for easy access:
+
+```bash
+# Install with shortcuts support
+pip install noaa-jetstream[shortcuts]
+
+# Create shortcuts
+jetstream-create-shortcuts
+
+# Remove shortcuts
+jetstream-remove-shortcuts
+```
+
+### Troubleshooting Startup Issues
+
+**If the server appears to start but you can't connect:**
+
+1. **Run diagnostics:**
+   ```bash
+   python diagnose.py
+   ```
+   
+2. **Run with debug logging:**
+   ```bash
+   jetstream --log-level debug
+   # or from source:
+   python -m uvicorn jetstream.main:app --reload --log-level debug
+   ```
+---
+
+## Troubleshooting
+
+**Cannot connect to GCS:**
+- Verify authentication: `gcloud auth list`
+- Check bucket permissions
+- Ensure Application Default Credentials are set
+
+**Jobs stuck in queue:**
+- Check queue status in dashboard
+- Verify no jobs are blocking the queue
+- Restart the application if needed
+
+**Database errors:**
+- Delete `jetstream.db` to reset (loses history)
+- Check file permissions in application directory
+
+**API not responding:**
+- Check if port 8000 is already in use
+- View logs in terminal for error messages
+- Ensure all dependencies are installed
+
+----------
+#### Disclaimer
+This repository is a scientific product and is not official communication of the National Oceanic and Atmospheric Administration, or the United States Department of Commerce. All NOAA GitHub project content is provided on an 'as is' basis and the user assumes responsibility for its use. Any claims against the Department of Commerce or Department of Commerce bureaus stemming from the use of this GitHub project will be governed by all applicable Federal law. Any reference to specific commercial products, processes, or services by service mark, trademark, manufacturer, or otherwise, does not constitute or imply their endorsement, recommendation or favoring by the Department of Commerce. The Department of Commerce seal and logo, or the seal and logo of a DOC bureau, shall not be used in any manner to imply endorsement of any commercial product or activity by DOC or the United States Government.
+
+## License
+See the [LICENSE.md](./LICENSE.md) for details

noaa_jetstream-0.1.2/jetstream/__init__.py

@@ -0,0 +1,22 @@
+"""
+NOAA JetStream - Cloud Data Manager
+A comprehensive tool for managing local-to-cloud uploads with queue management,
+statistics, and folder analysis.
+"""
+
+__version__ = "0.1.2"
+__author__ = "Michael Akridge"
+__description__ = "JetStream: Cloud Data Manager for local-to-cloud uploads"
+
+# Lazy import to avoid triggering server initialization on package import
+def _get_app():
+    from jetstream.main import app
+    return app
+
+# For backwards compatibility
+def __getattr__(name):
+    if name == "app":
+        return _get_app()
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+__all__ = ["__version__"]

noaa_jetstream-0.1.2/jetstream/cli.py

@@ -0,0 +1,92 @@
+"""
+Command-line interface for NOAA JetStream
+"""
+import sys
+import argparse
+import uvicorn
+import webbrowser
+import threading
+import time
+import os
+
+
+def open_browser(url, delay=2.0):
+    """Open browser after a delay to ensure server is ready"""
+    def _open():
+        time.sleep(delay)
+        print(f"\n🌐 Opening browser: {url}")
+        webbrowser.open(url)
+    
+    thread = threading.Thread(target=_open, daemon=True)
+    thread.start()
+
+
+def main():
+    """Start the JetStream server"""
+    parser = argparse.ArgumentParser(
+        description="NOAA JetStream - Cloud Data Manager"
+    )
+    parser.add_argument(
+        "--host",
+        default="0.0.0.0",
+        help="Host to bind to (default: 0.0.0.0)"
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=8000,
+        help="Port to bind to (default: 8000)"
+    )
+    parser.add_argument(
+        "--reload",
+        action="store_true",
+        help="Enable auto-reload for development"
+    )
+    parser.add_argument(
+        "--no-browser",
+        action="store_true",
+        help="Don't automatically open browser"
+    )
+    parser.add_argument(
+        "--log-level",
+        default="info",
+        choices=["debug", "info", "warning", "error"],
+        help="Log level (default: info)"
+    )
+    
+    args = parser.parse_args()
+    
+    print()
+    print("=" * 60)
+    print("  🚀 NOAA JetStream - Cloud Data Manager")
+    print("  Local-to-Cloud Upload Management System")
+    print("=" * 60)
+    print()
+    print(f"📋 Configuration:")
+    print(f"   Host: {args.host}")
+    print(f"   Port: {args.port}")
+    print(f"   Log Level: {args.log_level}")
+    print()
+    print(f"🌐 Web Interface: http://localhost:{args.port}")
+    print(f"📚 API Documentation: http://localhost:{args.port}/docs")
+    print()
+    print("Press Ctrl+C to stop the server")
+    print()
+    
+    # Open browser automatically unless disabled
+    if not args.no_browser:
+        url = f"http://localhost:{args.port}"
+        open_browser(url, delay=2.0)
+    
+    # Start server
+    uvicorn.run(
+        "jetstream.main:app",
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+        log_level=args.log_level,
+    )
+
+
+if __name__ == "__main__":
+    main()

noaa_jetstream-0.1.2/jetstream/config.py

@@ -0,0 +1,55 @@
+"""Configuration settings for JetStream API."""
+
+from pydantic_settings import BaseSettings
+from typing import Optional
+import os
+
+class Settings(BaseSettings):
+    """Application settings."""
+    
+    # Server settings
+    HOST: str = "0.0.0.0"
+    PORT: int = 8000
+    DEBUG: bool = False
+    AUTO_OPEN_BROWSER: bool = True  # Auto-open browser on startup
+    
+    # Database
+    DATABASE_URL: str = "sqlite:///./jetstream.db"
+    
+    # Google Cloud Storage
+    GCS_PROJECT_ID: Optional[str] = None
+    GCS_CREDENTIALS_PATH: Optional[str] = None
+    GCS_BUCKET_NAME: str = "nmfs_odp_pifsc"  # Default bucket name
+    
+    # Scanner Performance Settings (Optimized for very large folders)
+    MAX_FILES_FOR_DETAILED_SCAN: int = 1000   # Switch to folder-only after this
+    MAX_SUBFOLDERS_FOR_DETAILED_SCAN: int = 20  # Switch to folder-only after this
+    SCAN_TIMEOUT_SECONDS: int = 60  # Max time for a scan (1 minute)
+    ENABLE_FAST_SCAN: bool = True  # Use os.scandir instead of os.walk
+    
+    # File filtering (from jetstream.py)
+    EXCLUDE_FOLDERS: list = [
+        "_archive", "_YEAR", "ISLAND", "SITE-ID", "SITE_PHOTOS", 
+        "Corrected", "corrected", "uncorrected", "MISC", "DARK", 
+        "Products", "Thumbs.db", ".DS_Store", "__pycache__"
+    ]
+    
+    EXCLUDE_PATTERNS: list = [
+        r'.*\.tmp$',
+        r'.*\.bak$',
+        r'.*~$',
+        r'.*\.pyc$',
+    ]
+    
+    INCLUDE_PATTERNS: list = [
+        r'^.*(?<!\.JPG)$',
+        r'^.*\.(jpg|jpeg|png|tiff|tif|raw|cr2|nef|arw|dng)$',
+        r'^.*\.(mp4|mov|avi|mkv)$',
+        r'^.*\.(txt|csv|json|xml|log)$'
+    ]
+    
+    class Config:
+        env_file = ".env"
+        case_sensitive = True
+
+settings = Settings()