FlowerPower 0.3.1__tar.gz

flowerpower-0.3.1/.gitignore

@@ -0,0 +1,13 @@
+# python generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# venv
+.venv
+.env
+.vscode
+docker/config/.bash_history

flowerpower-0.3.1/.python-version

@@ -0,0 +1 @@
+3.12.2

flowerpower-0.3.1/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.3
+Name: FlowerPower
+Version: 0.3.1
+Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
+Author-email: "Volker L." <ligno.blades@gmail.com>
+Keywords: apscheduler,dask,hamilton,pipeline,ray,scheduler,workflow
+Requires-Python: >=3.11
+Requires-Dist: fsspec>=2024.5.0
+Requires-Dist: munch>=4.0.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: rich>=13.7.1
+Requires-Dist: sf-hamilton-sdk>=0.5.2
+Requires-Dist: sf-hamilton[visualization]>=1.69.0
+Requires-Dist: typer>=0.12.3
+Provides-Extra: dask
+Requires-Dist: dask[complete]>=2024.7.1; extra == 'dask'
+Provides-Extra: mongodb
+Requires-Dist: pymongo>=4.7.2; extra == 'mongodb'
+Provides-Extra: mqtt
+Requires-Dist: paho-mqtt>=2.1.0; extra == 'mqtt'
+Provides-Extra: ray
+Requires-Dist: ray>=2.34.0; extra == 'ray'
+Provides-Extra: redis
+Requires-Dist: redis>=5.0.4; extra == 'redis'
+Provides-Extra: scheduler
+Requires-Dist: aiosqlite>=0.20.0; extra == 'scheduler'
+Requires-Dist: apscheduler>=4.0.0a5; extra == 'scheduler'
+Requires-Dist: asyncpg>=0.29.0; extra == 'scheduler'
+Requires-Dist: greenlet>=3.0.3; extra == 'scheduler'
+Requires-Dist: sqlalchemy>=2.0.30; extra == 'scheduler'
+Provides-Extra: ui
+Requires-Dist: sf-hamilton-ui>=0.0.11; extra == 'ui'
+Description-Content-Type: text/markdown
+
+# FlowerPower
+
+![Bild](./image.png)
+
+FlowerPower is a simple workflow framework based on the fantastic [Hamilton](https://github.com/DAGWorks-Inc/hamilton) and [Advanced Python Scheduler - APScheduler](https://github.com/agronholm/apscheduler)
+
+## Installation
+
+```shell
+pip install "flowerpower" git+https://github.com/legout/flowerpower
+# with scheduler
+pip install "flowerpower[scheduler]" git+https://github.com/legout/flowerpower
+# with mqtt event broker
+pip install "flowerpower[scheduler,mqtt]" git+https://github.com/legout/flowerpower
+# with redis event broker
+pip install "flowerpower[scheduler,redis]" git+https://github.com/legout/flowerpower
+```
+
+## Usage
+
+### 0) Optional: Dev Services
+```shell
+curl -O https://raw.githubusercontent.com/legout/flowerpower/main/docker/Dockerfile
+curl -O https://raw.githubusercontent.com/legout/flowerpower/main/docker/docker-compose.yml
+
+# Hamilton UI, which allows you to track and visualize your pipelines
+docker-compose up hamilton_ui -d 
+# jupyterlab and code-server
+docker-compose up jupytercode -d 
+# s3 compatible object storage
+docker-compose up minio -d 
+#  mosquitto mqtt broker if you want to use mqtt as the event broker
+docker-compose up mqtt -d 
+# valkey (OSS redis) if you want to use redis as the event broker
+docker-compose up redis -d 
+# mongodb if you want to use mongodb as the data store
+docker-compose up mongodb -d 
+ # postgres db if you want to use postgres as data store and/or event broker. This db is also used for hamilton ui
+docker-compose up postgres -d
+
+```
+
+
+### a) Initialze a new flowerpower project
+```shell
+mkdir new-project
+cd new-project
+python -m flowerpower.cli init 
+```
+This adds basic config files `conf/pipelines.yml`, `conf/scheduler.yml` and `conf/tracker.yml`
+
+### b) Add a new pipeline
+```shell
+python -m flowerpower.cli add-pipeline my_flow
+```
+A new file `pipelines/my_flow.py` is created and the relevant entries are added to the config files.
+
+### c) Setup the new pipeline
+Edit `pipelines/my_flow.py` and add the pipeline functions.
+
+FlowerPower uses [Hamilton](https://github.com/DAGWorks-Inc/hamilton) that converts your pipeline functions into nodes and then creates a [Directed Acyclic Graph (DAG)](https://en.wikipedia.org/wiki/Directed_acyclic_graph).
+
+It is therefore mandatory to write your pipeline files according to the Hamilton paradigm. You can read more about this in the Hamilton documentaion chapter [Function, Nodes and DataFlow](https://hamilton.dagworks.io/en/latest/concepts/node/)
+
+Optinally edit the config files `conf/pipelines.yml`, `conf/scheduler.yml` and `conf/tracker.yml`
+
+### d) Run or Scheduler the new pipeline
+```shell
+python -m flowerpower.cli run-pipeline my_flow
+# or schedule with a 30 seconds interval
+python -m flowerpower.cli schedule-pipeline my_flow interval --interval-params seconds=30 --auto-start
+```
+
+
+
+
+

flowerpower-0.3.1/README.md

@@ -0,0 +1,77 @@
+# FlowerPower
+
+![Bild](./image.png)
+
+FlowerPower is a simple workflow framework based on the fantastic [Hamilton](https://github.com/DAGWorks-Inc/hamilton) and [Advanced Python Scheduler - APScheduler](https://github.com/agronholm/apscheduler)
+
+## Installation
+
+```shell
+pip install "flowerpower" git+https://github.com/legout/flowerpower
+# with scheduler
+pip install "flowerpower[scheduler]" git+https://github.com/legout/flowerpower
+# with mqtt event broker
+pip install "flowerpower[scheduler,mqtt]" git+https://github.com/legout/flowerpower
+# with redis event broker
+pip install "flowerpower[scheduler,redis]" git+https://github.com/legout/flowerpower
+```
+
+## Usage
+
+### 0) Optional: Dev Services
+```shell
+curl -O https://raw.githubusercontent.com/legout/flowerpower/main/docker/Dockerfile
+curl -O https://raw.githubusercontent.com/legout/flowerpower/main/docker/docker-compose.yml
+
+# Hamilton UI, which allows you to track and visualize your pipelines
+docker-compose up hamilton_ui -d 
+# jupyterlab and code-server
+docker-compose up jupytercode -d 
+# s3 compatible object storage
+docker-compose up minio -d 
+#  mosquitto mqtt broker if you want to use mqtt as the event broker
+docker-compose up mqtt -d 
+# valkey (OSS redis) if you want to use redis as the event broker
+docker-compose up redis -d 
+# mongodb if you want to use mongodb as the data store
+docker-compose up mongodb -d 
+ # postgres db if you want to use postgres as data store and/or event broker. This db is also used for hamilton ui
+docker-compose up postgres -d
+
+```
+
+
+### a) Initialze a new flowerpower project
+```shell
+mkdir new-project
+cd new-project
+python -m flowerpower.cli init 
+```
+This adds basic config files `conf/pipelines.yml`, `conf/scheduler.yml` and `conf/tracker.yml`
+
+### b) Add a new pipeline
+```shell
+python -m flowerpower.cli add-pipeline my_flow
+```
+A new file `pipelines/my_flow.py` is created and the relevant entries are added to the config files.
+
+### c) Setup the new pipeline
+Edit `pipelines/my_flow.py` and add the pipeline functions.
+
+FlowerPower uses [Hamilton](https://github.com/DAGWorks-Inc/hamilton) that converts your pipeline functions into nodes and then creates a [Directed Acyclic Graph (DAG)](https://en.wikipedia.org/wiki/Directed_acyclic_graph).
+
+It is therefore mandatory to write your pipeline files according to the Hamilton paradigm. You can read more about this in the Hamilton documentaion chapter [Function, Nodes and DataFlow](https://hamilton.dagworks.io/en/latest/concepts/node/)
+
+Optinally edit the config files `conf/pipelines.yml`, `conf/scheduler.yml` and `conf/tracker.yml`
+
+### d) Run or Scheduler the new pipeline
+```shell
+python -m flowerpower.cli run-pipeline my_flow
+# or schedule with a 30 seconds interval
+python -m flowerpower.cli schedule-pipeline my_flow interval --interval-params seconds=30 --auto-start
+```
+
+
+
+
+

flowerpower-0.3.1/docker/Caddyfile.bak

@@ -0,0 +1,45 @@
+{
+  http_port 80
+}
+
+:80 {
+  route /jupytercode/* {
+    uri strip_prefix /jupytercode
+    reverse_proxy jupytercode:8888
+  }
+
+  route /minio/* {
+    uri strip_prefix /minio
+    reverse_proxy minio:9001
+  }
+
+  route /mqtt/* {
+    uri strip_prefix /mqtt
+    reverse_proxy mqtt:1883
+  }
+
+  route /redis/* {
+    uri strip_prefix /redis
+    reverse_proxy redis:6379
+  }
+
+  route /mongodb/* {
+    uri strip_prefix /mongodb
+    reverse_proxy mongodb:27017
+  }
+
+  route /nodered/* {
+    uri strip_prefix /nodered
+    reverse_proxy nodered:1880
+  }
+
+  route /postgres/* {
+    uri strip_prefix /postgres
+    reverse_proxy postgres:5432
+  }
+
+  route /hamilton_ui/* {
+    uri strip_prefix /hamilton_ui
+    reverse_proxy hamilton_ui:8242
+  }
+}

flowerpower-0.3.1/docker/Dockerfile

@@ -0,0 +1,69 @@
+FROM quay.io/jupyter/base-notebook
+
+ARG CODE_VERSION=4.89.1
+ARG PLATFORM=arm64
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+# Install all OS dependencies for a fully functional Server
+RUN apt-get update --yes && \
+    apt-get install --yes --no-install-recommends \
+    # Common useful utilities
+    curl \
+    git \
+    nano-tiny \
+    tzdata \
+    unzip \
+    # git-over-ssh
+    openssh-client \
+    # Enable clipboard on Linux host systems
+    xclip && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create alternative for nano -> nano-tiny
+RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10
+
+# Install code-server
+
+RUN curl -fOL https://github.com/coder/code-server/releases/download/v$CODE_VERSION/code-server_${CODE_VERSION}_${PLATFORM}.deb \
+    && dpkg -i code-server_${CODE_VERSION}_${PLATFORM}.deb \
+    && rm -f code-server_${CODE_VERSION}_${PLATFORM}.deb
+
+# Switch back to jovyan to avoid accidental container runs as root
+USER ${NB_UID}
+
+#conda install jupyterlab -y && \
+RUN conda install jupyter-server-proxy jupyter-vscode-proxy  -y && \
+    conda clean --tarballs --index-cache --packages --yes && \
+    find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \
+    find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
+    conda clean --force-pkgs-dirs --all --yes
+#  && \
+#sudo echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
+#echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
+
+COPY assets/GitHub.copilot-1.199.0.vsix GitHub.copilot-1.199.0.vsix
+COPY assets/GitHub.copilot-chat-0.15.2024043005.vsix GitHub.copilot-chat-0.15.2024043005.vsix
+
+RUN code-server --install-extension ms-toolsai.jupyter && \
+    code-server --install-extension ms-python.python && \
+    code-server --install-extension mtxr.sqltools && \
+    code-server --install-extension bungcip.better-toml && \
+    code-server --install-extension ms-python.black-formatter && \
+    code-server --install-extension ms-python.isort && \
+    code-server --install-extension redhat.vscode-yaml && \
+    code-server --install-extension charliermarsh.ruff && \
+    code-server --install-extension esbenp.prettier-vscode && \
+    code-server --install-extension mtxr.sqltools-driver-sqlite && \
+    code-server --install-extension mtxr.sqltools-driver-pg && \
+    code-server --install-extension GitHub.copilot-1.199.0.vsix && \
+    code-server --install-extension GitHub.copilot-chat-0.15.2024043005.vsix && \
+    rm -r GitHub.copilot-*.vsix && \
+    rm -r /home/jovyan/.local/share/code-server/CachedExtensionVSIXs/
+
+
+#EXPOSE 8888
+#EXPOSE 3000
+#ENTRYPOINT  ["jupyter", "lab", "--ip=0.0.0.0", "--allow-root", "--no-browser", "--NotebookApp.token=''"]

flowerpower-0.3.1/docker/conf/Caddyfile

@@ -0,0 +1,37 @@
+
+
+jupytercode.localhost {
+    reverse_proxy jupytercode:8888
+}
+
+minio.localhost {
+    reverse_proxy minio:9001
+}
+
+minio_api.localhost {
+    reverse_proxy minio:9000
+}
+
+mqtt.localhost {
+    reverse_proxy jupytercode:1883
+}
+
+redis.localhost {
+    reverse_proxy minio:6379
+}
+
+mongodb.localhost {
+    reverse_proxy mqtt:27017
+}
+
+nodered.localhost {
+    reverse_proxy mqtt:1880
+}
+
+hamilton_ui.localhost {
+    reverse_proxy hamilton_ui:8242
+}
+
+postgres.localhost {
+    reverse_proxy hamilton_ui:5432
+}

flowerpower-0.3.1/docker/conf/mosquitto.conf

@@ -0,0 +1,2 @@
+listener 1883
+allow_anonymous true

flowerpower-0.3.1/docker/conf/nginx.conf

@@ -0,0 +1,62 @@
+events { worker_connections 1024; }
+
+http {
+    server {
+        listen 80;
+
+        location /jupytercode/ {
+            proxy_pass http://jupytercode:8888/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /minio/ {
+            proxy_pass http://minio:9001/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /mqtt/ {
+            proxy_pass http://mqtt:1883/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+
+        location /redis/ {
+            proxy_pass http://redis:6379/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /mongodb/ {
+            proxy_pass http://mongodb:27017/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /nodered/ {
+            proxy_pass http://nodered:1880/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /postgres/ {
+            proxy_pass http://postgres:5432/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /backend/ {
+            proxy_pass http://backend:8241/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /hamilton_ui/ {
+            proxy_pass http://hamilton_ui:8242/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+    }
+}

flowerpower-0.3.1/docker/docker-compose.yml

@@ -0,0 +1,126 @@
+version: "2"
+services:
+  # jupytercode:
+  #   build: 
+  #     context: .
+  #     args:
+  #       - CODE_VERSION=4.89.1
+  #       - PLATFORM=arm64
+  #   image: jupytercode:latest
+  #   user: root
+  #   environment:
+  #     - GRANT_SUDO=yes
+  #   ports:
+  #     - 8888:8888
+  #   volumes:
+  #     - jupytercode:/home/jovyan
+  #
+  minio:
+    image: minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: minio1234
+    volumes:
+      - minio_data:/data
+    command: server /data --console-address ":9001"
+  
+  mqtt:
+    image: eclipse-mosquitto
+    volumes:
+      - ./conf/mosquitto.conf:/mosquitto/config/mosquitto.conf
+    ports:
+      - 1883:1883
+    command: mosquitto -c /mosquitto/config/mosquitto.conf
+
+  redis:
+    image: valkey/valkey
+    ports:
+      - 6379:6379
+  
+  mongodb:
+    image: mongo
+    ports:
+      - 27017:27017
+
+  # nodered:
+  #   image: nodered/node-red
+  #   volumes:
+  #     - nodered:/data
+  #   ports: 
+  #     - 1880:1880
+  #
+  postgres:
+    image: postgres:12
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=flowerpower
+      - POSTGRES_USER=edge
+      - POSTGRES_PASSWORD=edge
+    ports:
+      - 5432:5432
+    restart: always
+
+  backend:
+    image: dagworks/ui-backend:latest
+    entrypoint: ["/bin/bash", "-c", "cd /code/server && ls && ./entrypoint.sh"]
+    ports:
+      - 8241:8241
+    environment:
+      - DB_HOST=postgres
+      - DB_PORT=5432
+      - DB_NAME=flowerpower
+      - DB_USER=edge
+      - DB_PASSWORD=edge # TODO: Change this to a secret
+      - HAMILTON_BLOB_STORE=local
+      - HAMILTON_ENV=local # local env
+      - HAMILTON_LOCAL_BLOB_DIR=/data/blobs # TODO -- set this up to be a better one
+      - DJANGO_SECRET_KEY=do_not_use_in_production
+      - HAMILTON_TELEMETRY_ENABLED=false #${HAMILTON_TELEMETRY_ENABLED-true}
+      - HAMILTON_AUTH_MODE=permissive
+    depends_on:
+      - postgres
+    volumes:
+      - backend_data:/data/
+
+  hamilton_ui:
+    image: dagworks/ui-frontend:latest
+    ports:
+      - 8242:8242
+    environment:
+      - NODE_ENV=development
+      - REACT_APP_AUTH_MODE=local
+      - REACT_APP_USE_POSTHOG=false
+    depends_on:
+      - backend
+
+  # nginx:
+  #   image: nginx:latest
+  #   volumes:
+  #     - ./nginx.conf:/etc/nginx/nginx.conf
+  #   ports:
+  #     - 80:80
+  #     - 443:443
+  #   depends_on:
+  #     - hamilton_ui
+
+  # caddy:
+  #   image: caddy:latest
+  #   volumes:
+  #     - .conf/Caddyfile:/etc/caddy/Caddyfile
+  #   ports:
+  #     - 80:80
+  #     - 443:443
+  #   depends_on:
+  #     - hamilton_ui
+
+volumes:
+  jupytercode:
+  minio_data:
+  minio_config:
+  nodered:
+  postgres_data:
+  backend_data:

flowerpower-0.3.1/examples/hello-world3/README.md

@@ -0,0 +1,6 @@
+# HELLO-WORLD3
+
+**created with FlowerPower**
+
+*2024-08-11 18:30:13*
+

flowerpower-0.3.1/examples/hello-world3/conf/pipeline.yml

@@ -0,0 +1,35 @@
+# ---------------- Pipelines Configuration ----------------- #
+
+# ------------------------ Example ------------------------- #
+#
+# path: pipelines
+#
+# ## pipeline parameter
+#
+# params:
+#   flow1:                      ## pipeline name
+#       step1:                  ## step name
+#         param1_1: 123         ## step parameters
+#         param1_2: abc
+#       step2:
+#         param2_1: true
+#
+# ## run configuration
+#
+# run:
+#   prod: # environment name
+#     flow1:
+#       inputs:                 ## input parameters
+#       final_vars: [step2]     ## final output vars
+#       with_tracker: true      ## whether to track the run
+#
+#   dev:
+#     flow1:
+#       inputs:
+#       final_vars: [step2]
+#       with_tracker: false
+#
+# ---------------------------------------------------------- #
+
+run: 
+params: 

flowerpower-0.3.1/examples/hello-world3/conf/scheduler.yml

@@ -0,0 +1,83 @@
+# ---------------- Scheduler Configuration ----------------- #
+
+# ------------------------ Example ------------------------- #
+#
+# ##  data store configuration
+#
+# ### postgres
+# data_store:
+#   type: sqlalchemy
+#   url: postgresql+asyncpg://edge:edge@postgres/flowerpower
+#
+# ### sqlite
+# data_store:
+#   type: sqlalchemy
+#   url: sqlite+aiosqlite:///flowerpower.db
+#
+# ### memory
+# data_store:
+#   type: memory
+#
+# ### mongodb
+# data_store:
+#   type: mongodb
+#   url: mongodb://localhost:27017/scheduler
+#
+# ## event broker configuration
+#
+# ### postgres
+# event_broker:
+#   type: asyncpg
+#   url: postgresql+asyncpg://edge:edge@postgres/flowerpower
+#
+# ### mqtt
+# event_broker:
+#   type: mqtt
+#   host: localhost
+#   port: 1883
+#   username: edge
+#   password: edge
+
+# ### redis
+# event_broker:
+#   type: redis
+#   host: localhost
+#   port: 6379
+
+# ### local
+# event_broker:
+#   type: local # or memory
+#
+# ## pipeline schedule configuration
+#
+# pipeline:
+#   my_flow:
+#     type: cron                ## options: interval, calendarinterval, date
+#     start_time:
+#     end_time:
+#     ## optional cron arguments
+#     crontab: * * * * *
+#     year:
+#     month:
+#     week:
+#     day:
+#     days_of_week:
+#     hour:
+#     minute:
+#     second:
+#     timezone:
+#     ## optional interval arguments
+#     weeks:
+#     days:
+#     hours:
+#     minutes:
+#     seconds:
+#     microseconds:
+#
+# ---------------------------------------------------------- #
+
+data_path:
+  type: memory
+event_broker:
+  type: local
+pipeline: