pyantz 0.1.0__tar.gz

pyantz-0.1.0/.github/workflows/bandit.yml

@@ -0,0 +1,21 @@
+name: Bandit Security
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Bandit
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout the Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Run Bandit
+        env:
+          EXCLUDE_FILES: tests/*
+        run: bash <(curl -s https://raw.githubusercontent.com/CICDToolbox/bandit/master/pipeline.sh)

pyantz-0.1.0/.github/workflows/coverage_badge.yml

@@ -0,0 +1,34 @@
+on:
+  push:
+    branches:
+      - master
+name: CI
+
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+        matrix:
+            python-version:
+            - "3.11"
+            - "3.12"
+            - "3.13"
+    name: Python ${{ matrix.python-version }} on coverage
+    steps:
+        - uses: actions/checkout@v4
+
+        - name: Install uv and set the python version
+          uses: astral-sh/setup-uv@v5
+          with:
+            python-version: ${{ matrix.python-version }}
+
+        - name: Install the project
+          run: uv sync --all-extras --dev
+
+        - name: Run ruff lint
+          # For example, using `pytest`
+          run: uv run pytest tests --cov=src
+
+        - name: Run coverage-badge on own test coverage
+          run: uvx run coverage-badge

pyantz-0.1.0/.github/workflows/mypy.yml

@@ -0,0 +1,28 @@
+name: Check Types
+
+on: [push]
+
+jobs:
+  build:
+    name: continuous-integration
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+          - "3.12"
+          - "3.13"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run mypy type checks
+        run: uv run mypy src

pyantz-0.1.0/.github/workflows/pylint.yml

@@ -0,0 +1,31 @@
+name: Lint
+
+on: [push]
+
+jobs:
+  build:
+    name: continuous-integration
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+          - "3.12"
+          - "3.13"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run ruff lint
+        run: uv run ruff check src
+
+      - name: Run pylint
+        run: uv run pylint src

pyantz-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Test
+
+on: [push]
+
+jobs:
+  build:
+    name: continuous-integration
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+          - "3.12"
+          - "3.13"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run pytes
+        # For example, using `pytest`
+        run: uv run pytest tests

pyantz-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+.pytest*
+.venv*
+**__pycache__**
+.coverage*
+.mypy_cache
+.vscode
+.idea
+*.egg-info
+*.egg
+*.egg-info/
+*.egg-info/*
+*.egg-info/**/*
+.ruff_cache
+dist
+.hypothesis
+dist/*

pyantz-0.1.0/.python-version

@@ -0,0 +1 @@
+>=3.10

pyantz-0.1.0/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2025
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyantz-0.1.0/Makefile

@@ -0,0 +1,23 @@
+test:
+	uv run pytest -n 16 --cov=src
+
+format:
+	uv run isort src
+	uv run ruff check src --fix
+	uv run ruff format src
+	uv run isort tests
+	uv run ruff check tests --fix
+	uv run ruff format tests
+
+lint:
+	uv run ruff check src
+	uv run pylint src
+
+type:
+	uv run mypy src
+
+build:
+	uv build
+
+sync:
+	uv sync --all-extras

pyantz-0.1.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: pyantz
+Version: 0.1.0
+Summary: Add your description here
+Author-email: igreen1 <dev@iangreen.dev>
+License-File: LICENSE.txt
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: rich>=13.9.4
+Requires-Dist: sqlalchemy>=2.0.38
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.2.3; extra == 'pandas'
+Provides-Extra: pyarrow
+Requires-Dist: pandas>=2.2.3; extra == 'pyarrow'
+Requires-Dist: pyarrow>=19.0.1; extra == 'pyarrow'
+Provides-Extra: slurm
+Requires-Dist: requests>=2.32.3; extra == 'slurm'
+Description-Content-Type: text/markdown
+
+# PyAntz
+
+Job runner and scheduling
+
+Setup configurations to chain together different jobs quickly and with minimal code. The goal here
+is to help those who need to quickly setup a local data pipeline for small projects. This spawned
+off of a school project
+
+
+## Dev Notes
+
+TODO
+-> slurm
+    on failure of submission, retry with different node
+    batch submit
+    individual submit
+-> windows hpc
+    on failure of submission, retyr with different node
+    batch submit
+-> GUI to create the jobs
+-> check the arg types of the functions in the job
+-> use pyarrow
+--> support using dask, pandas or pyarrow as the backend
+
+slurm
+
+
+- edit yaml
+edit fields (multiple!) set values
+---> accepts two lists. the first is a list of str (paths to edit) and the second it values
+---> must be of the same length, checked at runtime :(
+
+- edit json
+---> same as edit yaml above
+- edit csv
+---> accepts column names, list of values
+---> only one column at a time
+
+- convert csv to parquet
+---> read csv and output to parquet
+- convert excel to parquet
+accepts path to excel and optionally the sheet name
+---> read into memory and output to parquet
+- convert hdf5 to parquet
+accepts path and keys and reads into pandas
+outputs to parquet
+
+convert parquet to csv
+convert parquet to excel
+convert hdf5 to excel
+
+# DB
+CRUD postgres
+CRUD mysql
+CRUD minio
+CRUD sql server
+
+# Plot
+- scatter x/y with color options
+- hist
+- stacked hist
+
+# Aggregation
+- merge parquets
+- concatenate parquets
+- filter parquet
+- count parquet
+- sum parquet
+- min parquet
+- max parquet
+- xth percentile
+- groupby and agg
+
+# splitters
+- split template into N pipelines
+take one example pipeline and produce a PIPELINE ID variable for each in range
+- generate simple case matrix
+for one variable being modified, create pipelines for range of values for the variable (only integers)
+-

pyantz-0.1.0/README.md

@@ -0,0 +1,79 @@
+# PyAntz
+
+Job runner and scheduling
+
+Setup configurations to chain together different jobs quickly and with minimal code. The goal here
+is to help those who need to quickly setup a local data pipeline for small projects. This spawned
+off of a school project
+
+
+## Dev Notes
+
+TODO
+-> slurm
+    on failure of submission, retry with different node
+    batch submit
+    individual submit
+-> windows hpc
+    on failure of submission, retyr with different node
+    batch submit
+-> GUI to create the jobs
+-> check the arg types of the functions in the job
+-> use pyarrow
+--> support using dask, pandas or pyarrow as the backend
+
+slurm
+
+
+- edit yaml
+edit fields (multiple!) set values
+---> accepts two lists. the first is a list of str (paths to edit) and the second it values
+---> must be of the same length, checked at runtime :(
+
+- edit json
+---> same as edit yaml above
+- edit csv
+---> accepts column names, list of values
+---> only one column at a time
+
+- convert csv to parquet
+---> read csv and output to parquet
+- convert excel to parquet
+accepts path to excel and optionally the sheet name
+---> read into memory and output to parquet
+- convert hdf5 to parquet
+accepts path and keys and reads into pandas
+outputs to parquet
+
+convert parquet to csv
+convert parquet to excel
+convert hdf5 to excel
+
+# DB
+CRUD postgres
+CRUD mysql
+CRUD minio
+CRUD sql server
+
+# Plot
+- scatter x/y with color options
+- hist
+- stacked hist
+
+# Aggregation
+- merge parquets
+- concatenate parquets
+- filter parquet
+- count parquet
+- sum parquet
+- min parquet
+- max parquet
+- xth percentile
+- groupby and agg
+
+# splitters
+- split template into N pipelines
+take one example pipeline and produce a PIPELINE ID variable for each in range
+- generate simple case matrix
+for one variable being modified, create pipelines for range of values for the variable (only integers)
+-

pyantz-0.1.0/docs/examples/simple_runner.json

@@ -0,0 +1,38 @@
+{
+    "submitter_config" {
+        "local"
+    },
+    "config": {
+        "type": "pipeline",
+        "jobs": [
+            {
+                "type": "submitter_job",
+                "function": "pyantz.jobs.branch.create_pipelines_from_matrix.create_pipelines_from_matrix",
+                "parameters": {
+                    "pipeline_config_template": {
+                        "type": "pipeline", 
+                        "jobs": [
+                            {
+                                "type": "mutable_job",
+                                "function": "pyantz.jobs.variables.assign_environment_variable.assign_environment_variable",
+                                "parameters": {
+                                    "environmental_variables": {
+                                        "my_account": "%{my_account_from_csv}"
+                                    }
+                                }
+                            },
+                            {
+                                "type": "simple_job",
+                                "function": "pyantz.jobs.dispatch.run_command.run_command",
+                                "parameters": {
+                                    "cmd": ["sbatch", "--wckey=%{wckey}", "myscript", "$env_var", "%{arg1}", "arg2"]
+                                }
+                            }
+                        ]
+                    }
+                }
+                
+            }
+        ]
+    }
+}

pyantz-0.1.0/pyproject.toml

@@ -0,0 +1,54 @@
+[project]
+name = "pyantz"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "igreen1", email = "dev@iangreen.dev" }
+]
+requires-python = ">=3.11"
+dependencies = [
+    "pydantic>=2.10.6",
+    "rich>=13.9.4",
+    "sqlalchemy>=2.0.38",
+]
+
+[project.optional-dependencies]
+pyarrow = [
+    "pyarrow>=19.0.1",
+    "pandas>=2.2.3",
+]
+pandas = [
+    "pandas>=2.2.3",
+]
+slurm = [
+    "requests>=2.32.3",
+]
+
+[project.scripts]
+pyantz = "pyantz:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.isort]
+profile = "black"
+
+[tool.pylint.main]
+disable = "W0401,W0614"
+
+[dependency-groups]
+dev = [
+    "bandit>=1.8.3",
+    "hypothesis>=6.127.6",
+    "isort>=6.0.1",
+    "mypy>=1.15.0",
+    "pandas-stubs>=2.2.3.241126",
+    "pyarrow-stubs>=17.17",
+    "pylint>=3.3.4",
+    "pytest>=8.3.5",
+    "pytest-cov>=6.0.0",
+    "pytest-xdist>=3.6.1",
+    "ruff>=0.9.9",
+]

pyantz-0.1.0/src/pyantz/infrastructure/__init__.py

@@ -0,0 +1,4 @@
+"""
+Infrastructure provides the backbones of this runner. This module contains the code
+    to run all the various components to setup and execute the jobs in the configuration
+"""