rms-cloud-tasks 0.0.1__tar.gz

rms_cloud_tasks-0.0.1/.coveragerc

@@ -0,0 +1,5 @@
+[run]
+branch = false
+omit =
+    tests/*
+    azure.py

rms_cloud_tasks-0.0.1/.cursor/rules/python-rules.mdc

@@ -0,0 +1,16 @@
+---
+description: General Python rules
+globs: 
+---
+You are an expert systems-level Python programmer. All of the code that you write is carefully crafted for readability, ease of understanding, and ease of debugging.
+
+These rules must be followed:
+- All function and method definitions must use appropriate type annotations.
+- All classes, methods, and functions must have docstrings.
+- All docstrings must follow PEP257 and be in Google style. Use the word "Parameters" instead of "Arguments". If any docstrings are in a different format, convert them.
+- Liberally use comments to explain the logic behind the code if it is not self-explanatory.
+- Write all tests in pytest format. If any tests are in unittest format, convert them to pytest.
+- Whenever making any modifications, also update or create new tests and update all documentation including docstrings and the README file.
+- Never use mocks when writing the main code unless explicitly told to. It is ok to use mocks for tests.
+
+

rms_cloud_tasks-0.0.1/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length: 200
+exclude =
+    **/azure.py
+extend-ignore =
+    # F722 syntax error in forward annotation - due to Pydantic
+    F722

rms_cloud_tasks-0.0.1/.github/workflows/publish_to_pypi.yml

@@ -0,0 +1,33 @@
+name: Publish to PyPI
+run-name: Publish to PyPI triggered by ${{ github.ref_type }} ${{ github.ref_name }}
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  upload_pypi:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -r requirements.txt
+
+      - name: Build
+        run: |
+          python3 -m pip install --upgrade build && python3 -m build
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

rms_cloud_tasks-0.0.1/.github/workflows/publish_to_test_pypi.yml

@@ -0,0 +1,35 @@
+name: Publish to Test PyPI
+run-name: Publish to Test PyPI triggered by ${{ github.ref_type }} ${{ github.ref_name }}
+
+on:
+  workflow_dispatch:
+
+jobs:
+  upload_pypi:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -r requirements.txt
+
+      - name: Build
+        run: |
+          python3 -m pip install --upgrade build && python3 -m build
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/
+          verify-metadata: false

rms_cloud_tasks-0.0.1/.github/workflows/run-tests.yml

@@ -0,0 +1,79 @@
+name: Run Tests
+run-name: Run Tests triggered by ${{ github.ref_type }} ${{ github.ref_name }} or ${{ github.triggering_actor }}
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+  schedule:
+    - cron: "21 11 * * 0"
+
+jobs:
+  flake8:
+    name: Lint cloud_tasks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -r requirements.txt
+
+      - name: Flake8
+        run: |
+          flake8 src tests
+
+      # - name: Mypy
+      #   run: |
+      #     mypy src tests
+
+  test:
+    name: Test cloud_tasks
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+      fail-fast: false
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -r requirements.txt
+
+      - name: Test with coverage
+        run: |
+          pytest --cov
+
+      - name: Print coverage report
+        run: |
+          coverage report -m
+
+      - name: Upload coverage report to codecov
+        uses: codecov/codecov-action@v5
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true

rms_cloud_tasks-0.0.1/.gitignore

@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+**/_version.py
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+*venv*/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

rms_cloud_tasks-0.0.1/.readthedocs.yaml

@@ -0,0 +1,32 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "19"
+    # rust: "1.64"
+    # golang: "1.19"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: requirements.txt

rms_cloud_tasks-0.0.1/.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File with Arguments",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "args": "${command:pickArgs}"
+        }
+    ]
+}

rms_cloud_tasks-0.0.1/.vscode/settings.json

@@ -0,0 +1,14 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true
+    },
+    "python.analysis.diagnosticSeverityOverrides": {
+        "reportInvalidTypeForm": "none"
+    },
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

rms_cloud_tasks-0.0.1/CONTRIBUTING.md

@@ -0,0 +1,112 @@
+# Information for Potential Contributors
+
+First off, thanks for taking the time to contribute!
+
+This software is maintained by the [Ring-Moon Systems Node](https://pds-rings.seti.org) of NASA's [Planetary Data System](https://pds.nasa.gov). All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them; please read the relevant section before making your contribution.
+
+> If you like the project, but just don't have time to contribute, there are other easy ways to support the project and show your appreciation!
+> - Star the project on GitHub
+> - Post about it on social media
+> - Refer to this project in your project's README
+> - Mention the project at conferences and workshops and tell your friends/colleagues
+> - Cite the project in your papers and posters
+
+
+## Table of Contents
+
+- [Code of Conduct](#code-of-conduct)
+- [I Have a Question](#i-have-a-question)
+- [I Want to Report a Bug](#i-want-to-report-a-bug)
+- [I Want to Suggest an Enhancement](#i-want-to-suggest-an-enhancement)
+- [I Want To Contribute Code](#i-want-to-contribute-code)
+
+
+## Code of Conduct
+
+This project and everyone participating in it are governed by the
+[Code of Conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code. Please report unacceptable behavior
+to <matt@seti.org>.
+
+
+## I Have a Question
+
+> Please read the available documentation!
+
+Before asking a question, you can search for existing [issues](https://github.com/SETI/rms-cloud-tasks/issues) that might help you. If you find a suitable issue and still need clarification, you can write your question in that issue.
+
+If you can't find an appropriate issue and still want to ask a question, we recommend the following:
+
+- Open an [issue](https://github.com/SETI/rms-cloud-tasks/issues/new).
+- Provide as much context and detail as you can.
+- Provide project and platform versions (operating system, Python version, etc.), depending on what seems relevant.
+
+We will try to answer your question as soon as possible.
+
+
+## I Want to Report a Bug
+
+### Before Submitting a Bug Report
+
+A good bug report shouldn't leave others needing to chase you for more information. Therefore, we ask you to investigate carefully and collect all appropriate information in advance.
+
+- Make sure that you are using the latest version of this software and its supporting packages.
+- Make sure that you have read the documentation.
+- Determine that your bug really is a bug and not an error in your code or a misunderstanding in how to use our software.
+- To see if other users have experienced (and potentially solved) the same issue you're having, check if there is an existing issue for your bug or error in the [bug tracker](https://github.com/SETI/rms-cloud-tasks/issues).
+- Collect information about the bug:
+  - Stack trace (Traceback)
+  - OS and version (Windows/Linux/macOS), processor (x86/ARM/M1), Python version
+  - Detailed information on how to reproduce the bug, including function parameters, command line arguments, and input given/output received.
+
+### How Do I Submit a Good Bug Report?
+
+> You must never report security-related issues, vulnerabilities, or bugs that include sensitive information to the issue tracker or elsewhere in public. Instead, sensitive bugs must be sent by email to <matt@seti.org>.
+
+We use GitHub Issues to track bugs and errors. If you run into an issue with the project:
+
+- Open an [issue](https://github.com/SETI/rms-cloud-tasks/issues/new) with a **clear and descriptive title**. Please label the issue as `A-Bug` with no other labels.
+- Explain the **behavior you would expect** and the **behavior observed**.
+- Provide as much context as possible and describe the **detailed steps** that someone can follow to reproduce the problem. This usually includes providing your code; for a good bug report you should isolate the problem and create a reduced test case.
+- Provide other information collected in the previous section, such as the operating system and language version.
+
+Once it's filed:
+
+- The project team will label the issue accordingly.
+- A team member will try to reproduce the issue with your provided steps. If there are no steps given and no obvious way to reproduce the issue, the team will ask you for clarification.
+- If the team is able to reproduce the issue, it will be appropriately labeled and either assigned to a team member to fix, or left unassigned to be [implemented by someone else](#i-want-to-contribute-code).
+
+
+## I Want to Suggest an Enhancement
+
+This section guides you through submitting an enhancement, **including completely new features and minor improvements to existing functionality**.
+
+### Before Submitting an Enhancement
+
+- Make sure that you are using the latest version of this software and its supporting packages.
+- Make sure that you have read the documentation to see if the desired functionality is already provided.
+- Perform a [search](https://github.com/SETI/rms-cloud-tasks/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
+- Find out whether your idea fits within the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library.
+
+### How Do I Submit a Good Enhancement Suggestion?
+
+We use GitHub Issues to track enhancement requests. If you want to suggest an enhancement:
+
+- Open an [issue](https://github.com/SETI/rms-cloud-tasks/issues/new) with a **clear and descriptive title**. Please label the issue as `A-Enhancement` with no other labels.
+- Provide a **detailed** description of the suggested enhancement.
+- **Explain why this enhancement would be useful** to most users.
+
+
+## I Want To Contribute Code
+
+> ### Legal Notice
+> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content, and that the content you contribute may be provided under the project license.
+
+We welcome all code contributions, including bug fixes, new features, and improvements to documentation.
+
+- All suggested changes must be submitted using GitHub's "Pull Request" functionality.
+- All code changes must include appropriate new or updated tests to verify the changes made.
+- Existing documentation, including function- and file-level docstrings, must be updated as necessary, and new features fully described.
+- Code style must conform to that of the existing code; for Python this is generally a variant of PEP8 and PEP257.
+
+All submissions will be reviewed in detail by a project team member and changes may be suggested. Once the reviewer approves the changes, they will be merged into the main project branch and made a permanent part of the software. Your efforts to improve the software are greatly appreciated!

rms_cloud_tasks-0.0.1/DESIGN.md

@@ -0,0 +1,85 @@
+# Multi-Cloud Task Processing System Design Document
+
+## System Overview
+
+A scalable, multi-cloud task processing system that distributes independent tasks across compute instances, optimizing for cost and reliability with minimal overhead.
+
+## Core Components
+
+### 1. Task Queue Manager
+- Uses native queue services (AWS SQS, GCP Pub/Sub, Azure Service Bus)
+- Handles visibility timeout/locking to prevent duplicate processing
+- Automatically requeues failed tasks
+
+### 2. Instance Orchestrator
+- Monitors queue depth
+- Provisions instances based on workload
+- Terminates instances when no longer needed
+- Selects optimal instance types based on cost
+
+### 3. Worker Module
+- Downloads processing code from GitHub on startup
+- Polls for available tasks
+- Processes task input (JSON)
+- Handles graceful exit for spot instances
+
+## Architectural Decisions
+
+### Queue Management
+- **Task Visibility**: Use queue-native visibility timeout mechanisms to "lock" tasks during processing
+- **Completion Handling**: Workers delete/acknowledge tasks only after successful processing
+- **Failure Handling**: Failed tasks automatically return to queue after visibility timeout
+
+### Instance Management
+- **Instance Selection**: System will analyze requirements (CPU, memory, disk) and select the lowest-cost instance type that meets requirements
+- **Scaling Logic**: Scale up when queue depth exceeds threshold, scale down when queue depletes
+- **Spot Instance Handling**: Implement handlers for spot termination notices to exit gracefully
+
+### Worker Implementation
+- **Initialization**: Pull worker code from GitHub on startup
+- **Processing Loop**: Poll queue → Process → Mark complete → Repeat
+- **Termination**: Worker self-terminates when queue is empty for a specified duration
+- **Spot Instance Awareness**: Periodic checks for imminent termination
+
+## Implementation Considerations
+
+### Platform Abstraction
+- Implement thin abstraction layer over cloud provider APIs
+- Standardize instance provisioning/termination interfaces
+- Standardize queue operations across providers
+
+### Cost Optimization
+- System will calculate and display estimated costs before execution
+- Only use the specified maximum number of instances
+- Terminate instances as soon as tasks complete
+
+### Deployment Flow
+1. User configures task inputs and execution parameters
+2. System initializes queue with all task inputs
+3. Orchestrator provisions initial instances
+4. Workers process tasks until queue is empty
+5. Instances self-terminate when no work remains
+
+## Operational Considerations
+
+### Monitoring
+- Implement basic job progress tracking (tasks queued/completed)
+- Expose logs from worker instances
+- Track instance lifecycle events
+
+### Security
+- Use user-provided credentials for all cloud operations
+- Store credentials securely during execution
+
+### Error Handling
+- Gracefully handle cloud API failures
+- Report batch-level errors to user
+
+## Next Steps for Implementation
+
+1. Define cloud provider abstraction interfaces
+2. Implement queue adapters for each platform
+3. Develop instance selection algorithm
+4. Create worker initialization and processing logic
+5. Build orchestration controller
+6. Implement basic monitoring and reporting