flowtask 5.11.9__tar.gz → 5.11.10__tar.gz

flowtask-5.11.10/.gitignore

@@ -0,0 +1,210 @@
+# Worktrees
+.worktrees/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+cache/*
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+*.egg-info
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Config Stuff
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# configuration directories
+etc/
+env/
+settings/
+run.py
+
+## examples
+examples/
+samples/
+flowtask/exceptions.c
+*.c
+*.cpp
+flowtask/parsers/*.c
+flowtask/utils/*.cpp
+flowtask/utils/*.c
+
+# perf data
+memory_usage.csv
+perf.data
+perf.data.old
+
+# Visual Studio Code artifacts
+.cache/*
+.vscode/*
+.history/
+
+# ViM swap files
+.*.swp
+google/polestar-analytics-378119.json
+app.py
+resources/*.py
+*.code-workspace
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Yolo
+*.pt
+
+# Other docs:
+docs/*
+
+# Gecko Driver
+bin/geckodriver
+bin/*.gz
+bin/*.zip
+
+# Images:
+docs/images/*.png
+docs/images/*.jpg
+docs/images/*.jpeg
+data/gensim/information.json
+stderr.txt
+data/gensim/word2vec-google-news-300/__init__.py
+data/gensim/word2vec-google-news-300/word2vec-google-news-300.gz
+google/navigator-data.json
+tasks/programs/test/files/*
+static/nextstop/*
+# Jinja
+templates/.compiled
+templates/fontlist-v390.json
+
+# AI Models Cache (Hugging Face, Transformers, etc.)
+model_cache/
+.cache/huggingface/
+models--*/
+.huggingface/
+
+# UV:
+uv.lock
+
+# Statics
+static/
+
+# code worktrees:
+.claude/worktrees/

flowtask-5.11.10/Makefile

@@ -0,0 +1,191 @@
+# FlowTask Makefile
+# This Makefile provides a set of commands to manage the FlowTask project.
+
+.PHONY: venv install develop setup dev release format lint test clean distclean lock sync
+
+# Python version to use
+PYTHON_VERSION := 3.11
+
+# Auto-detect available tools
+HAS_UV := $(shell command -v uv 2> /dev/null)
+HAS_PIP := $(shell command -v pip 2> /dev/null)
+
+# Install uv for faster workflows
+install-uv:
+	curl -LsSf https://astral.sh/uv/install.sh | sh
+	@echo "uv installed! You may need to restart your shell or run 'source ~/.bashrc'"
+	@echo "Then re-run make commands to use faster uv workflows"
+
+# Create virtual environment
+venv:
+	uv venv --python $(PYTHON_VERSION) .venv
+	@echo 'run `source .venv/bin/activate` to start develop with FlowTask.'
+
+# Install production dependencies using lock file
+install:
+	uv sync --frozen --no-dev
+	uv pip install navigator-api[uvloop,locale]
+	@echo "Production dependencies installed. Use 'make develop' for development setup."
+
+# Generate lock files (uv only)
+lock:
+ifdef HAS_UV
+	uv lock
+else
+	@echo "Lock files require uv. Install with: pip install uv"
+endif
+
+# Install all dependencies including dev dependencies
+develop:
+	uv sync --frozen --all-extras
+
+# Alternative: install without lock file (faster for development)
+develop-fast:
+	uv pip install -e .[dev]
+
+# Setup development environment from requirements file (if you still have one)
+setup:
+	uv pip install -r requirements/requirements-dev.txt
+
+# Install in development mode using flit (if you want to keep flit)
+dev:
+	uv pip install flit
+	flit install --symlink
+
+# Build and publish release
+release: lint test clean
+	uv build
+	uv publish
+
+# Alternative release using flit
+release-flit: lint test clean
+	flit publish
+
+# Format code
+format:
+	uv run black flowtask
+
+# Lint code
+lint:
+	uv run pylint --rcfile .pylint flowtask/*.py
+	uv run black --check flowtask
+
+# Run tests with coverage
+test:
+	uv run coverage run -m flowtask.tests
+	uv run coverage report
+	uv run mypy flowtask/*.py
+
+# Alternative test command using pytest directly
+test-pytest:
+	uv run pytest
+
+# Add new dependency and update lock file
+add:
+	@if [ -z "$(pkg)" ]; then echo "Usage: make add pkg=package-name"; exit 1; fi
+	uv add $(pkg)
+
+# Add development dependency
+add-dev:
+	@if [ -z "$(pkg)" ]; then echo "Usage: make add-dev pkg=package-name"; exit 1; fi
+	uv add --dev $(pkg)
+
+# Remove dependency
+remove:
+	@if [ -z "$(pkg)" ]; then echo "Usage: make remove pkg=package-name"; exit 1; fi
+	uv remove $(pkg)
+
+# Compile Cython extensions using setup.py
+build-cython:
+	@echo "Compiling Cython extensions..."
+	python setup.py build_ext
+
+# Build Cython extensions in place (for development)
+build-inplace:
+	@echo "Building Cython extensions in place..."
+	python setup.py build_ext --inplace
+
+# Full build using uv
+build: clean
+	@echo "Building package with uv..."
+	uv build
+
+# Update all dependencies
+update:
+	uv lock --upgrade
+
+# Show project info
+info:
+	uv tree
+
+# Clean build artifacts
+clean:
+	rm -rf build/
+	rm -rf dist/
+	rm -rf *.egg-info/
+	find . -name "*.pyc" -delete
+	find . -name "*.pyo" -delete
+	find . -name "*.so" -delete
+	find . -type d -name __pycache__ -delete
+	find . -type f -name "*.pyc" -delete
+	find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true
+	@echo "Clean complete."
+
+# Remove virtual environment
+distclean:
+	rm -rf .venv
+	rm -rf uv.lock
+
+# Version management
+bump-patch:
+	@python -c "import re; \
+	content = open('flowtask/version.py').read(); \
+	version = re.search(r'__version__ = \"(.+)\"', content).group(1); \
+	parts = version.split('.'); \
+	parts[2] = str(int(parts[2]) + 1); \
+	new_version = '.'.join(parts); \
+	new_content = re.sub(r'__version__ = \".+\"', f'__version__ = \"{new_version}\"', content); \
+	open('flowtask/version.py', 'w').write(new_content); \
+	print(f'Version bumped to {new_version}')"
+
+bump-minor:
+	@python -c "import re; \
+	content = open('flowtask/version.py').read(); \
+	version = re.search(r'__version__ = \"(.+)\"', content).group(1); \
+	parts = version.split('.'); \
+	parts[1] = str(int(parts[1]) + 1); \
+	parts[2] = '0'; \
+	new_version = '.'.join(parts); \
+	new_content = re.sub(r'__version__ = \".+\"', f'__version__ = \"{new_version}\"', content); \
+	open('flowtask/version.py', 'w').write(new_content); \
+	print(f'Version bumped to {new_version}')"
+
+bump-major:
+	@python -c "import re; \
+	content = open('flowtask/version.py').read(); \
+	version = re.search(r'__version__ = \"(.+)\"', content).group(1); \
+	parts = version.split('.'); \
+	parts[0] = str(int(parts[0]) + 1); \
+	parts[1] = '0'; \
+	parts[2] = '0'; \
+	new_version = '.'.join(parts); \
+	new_content = re.sub(r'__version__ = \".+\"', f'__version__ = \"{new_version}\"', content); \
+	open('flowtask/version.py', 'w').write(new_content); \
+	print(f'Version bumped to {new_version}')"
+
+help:
+	@echo "Available targets:"
+	@echo "  venv         - Create virtual environment"
+	@echo "  install      - Install production dependencies"
+	@echo "  develop      - Install development dependencies"
+	@echo "  build        - Build package"
+	@echo "  release      - Build and publish package"
+	@echo "  test         - Run tests"
+	@echo "  format       - Format code"
+	@echo "  lint         - Lint code"
+	@echo "  clean        - Clean build artifacts"
+	@echo "  detect-tools - Show detected tools"
+	@echo "  install-uv   - Install uv"
+	@echo "  build-inplace - Build Cython extensions in place"
+	@echo ""
+	@echo "Current setup: $(TOOL_INFO)"

flowtask-5.11.10/PKG-INFO

@@ -0,0 +1,222 @@
+Metadata-Version: 2.4
+Name: flowtask
+Version: 5.11.10
+Summary: Framework for Task orchestration
+Author-email: Jesus Lara <jesuslarag@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/phenobarbital/flowtask
+Project-URL: Source, https://github.com/phenobarbital/flowtask
+Project-URL: Funding, https://paypal.me/phenobarbital
+Project-URL: Say Thanks!, https://saythanks.io/to/phenobarbital
+Keywords: Flowtask,Data Integration,Task Orchestration,Task-Runner,Pipelines
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Framework :: AsyncIO
+Requires-Python: <3.14,>=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: borax==3.5.0
+Requires-Dist: PyDrive==1.3.1
+Requires-Dist: chardet==5.2.0
+Requires-Dist: aiohttp-jinja2==1.6
+Requires-Dist: asyncssh[bcrypt,fido2,libnacl,pkcs11,pyOpenSSL]>=2.18.0
+Requires-Dist: pyxlsb==1.0.10
+Requires-Dist: python-calamine==0.2.3
+Requires-Dist: pyecharts==2.0.8
+Requires-Dist: selenium>=4.35.0
+Requires-Dist: snapshot-selenium==0.0.2
+Requires-Dist: httpx[http2,socks]>=0.26.0
+Requires-Dist: urllib3[socks]>=2.3.0
+Requires-Dist: requests==2.32.4
+Requires-Dist: h2>=4.2.0
+Requires-Dist: blinker>=1.9.0
+Requires-Dist: webdriver-manager==4.0.2
+Requires-Dist: aioimaplib==1.1.0
+Requires-Dist: adal==1.2.7
+Requires-Dist: xlrd>=2.0.2
+Requires-Dist: xmltodict==1.0.2
+Requires-Dist: zeep[async]==4.3.2
+Requires-Dist: nltk>=3.9.3
+Requires-Dist: jdcal==1.4.1
+Requires-Dist: html5lib==1.1
+Requires-Dist: shapely==2.0.6
+Requires-Dist: tzwhere==3.0.3
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: python-magic==0.4.27
+Requires-Dist: pytomlpp==1.0.13
+Requires-Dist: psutil==6.0.0
+Requires-Dist: networkx>=3.4.1
+Requires-Dist: gitpython==3.1.43
+Requires-Dist: watchdog>=6.0.0
+Requires-Dist: hachiko==0.4.0
+Requires-Dist: paramiko>=4.0.0
+Requires-Dist: jira==3.10.5
+Requires-Dist: datatable==1.1.0
+Requires-Dist: imageio==2.37.0
+Requires-Dist: decorator>=5.1.1
+Requires-Dist: emoji==2.14.1
+Requires-Dist: caio==0.9.24
+Requires-Dist: Wand==0.6.13
+Requires-Dist: pylibdmtx==0.1.10
+Requires-Dist: aiofile>=3.9.0
+Requires-Dist: filetype==1.2.0
+Requires-Dist: proxylists>=0.14.0
+Requires-Dist: aioftp==0.23.1
+Requires-Dist: py7zr==0.22.0
+Requires-Dist: rarfile==4.2
+Requires-Dist: python-pptx==1.0.2
+Requires-Dist: gspread==6.2.0
+Requires-Dist: oauth2client>=4.1.3
+Requires-Dist: google-auth>=2.35.0
+Requires-Dist: google-cloud-language>=2.15.0
+Requires-Dist: googlemaps==4.10.0
+Requires-Dist: fastavro==1.9.7
+Requires-Dist: pgpy==0.6.0
+Requires-Dist: dropbox==12.0.2
+Requires-Dist: dask-expr==1.1.13
+Requires-Dist: jsonpath_ng==1.7.0
+Requires-Dist: ortools<=9.14.6206,>=9.11.4210
+Requires-Dist: gmqtt==0.7.0
+Requires-Dist: selectorlib==0.16.0
+Requires-Dist: playwright==1.52.0
+Requires-Dist: fuzzywuzzy==0.18.0
+Requires-Dist: praw==7.8.1
+Requires-Dist: prawcore==2.4.0
+Requires-Dist: osmnx==2.0.1
+Requires-Dist: osmium==4.0.2
+Requires-Dist: python-Levenshtein==0.26.1
+Requires-Dist: undetected_chromedriver==3.5.5
+Requires-Dist: duckduckgo-search==8.1.1
+Requires-Dist: numba>=0.62.1
+Requires-Dist: APScheduler<3.12.0,>=3.11.0
+Requires-Dist: google-cloud-bigquery>=3.30.0
+Requires-Dist: pyheif==0.8.0
+Requires-Dist: filetype==1.2.0
+Requires-Dist: exif==1.6.1
+Requires-Dist: pillow-avif-plugin==1.5.2
+Requires-Dist: pillow-heif==0.22.0
+Requires-Dist: pgvector==0.4.1
+Requires-Dist: proxylists>=0.14.0
+Requires-Dist: async-notify[all]>=1.4.0
+Requires-Dist: querysource>=3.17.0
+Requires-Dist: asyncdb[all]>=2.11.7
+Requires-Dist: navconfig[default,uvloop]>=1.7.13
+Requires-Dist: navigator-auth>=0.15.8
+Requires-Dist: navigator-api[locale,uvloop]>=2.12.22
+Requires-Dist: ai-parrot[llms]>=0.17.4
+Requires-Dist: msal>=1.30.0
+Requires-Dist: azure-core==1.38.0
+Requires-Dist: azure-identity==1.20.0
+Requires-Dist: microsoft-kiota-authentication-azure==1.9.6
+Requires-Dist: msgraph-sdk==1.22.0
+Requires-Dist: zendriver>=0.15.2
+Provides-Extra: ai
+Requires-Dist: ai-parrot[agents,embeddings,images,integrations,llms,scheduler]>=0.20.4; extra == "ai"
+Requires-Dist: ai-parrot-loaders[audio,document,ebook,ml,pdf,scraping,video,web,youtube]>=0.1.0; extra == "ai"
+Requires-Dist: ai-parrot-tools[analysis,arxiv,aws,codeinterpreter,db,docker,excel,finance,flowtask,git,google,jira,messaging,pdf,pulumi,sandbox,scraping,sitesearch,slack,weather,wikipedia]>=0.1.0; extra == "ai"
+Requires-Dist: ai-parrot-pipelines>=0.1.0; extra == "ai"
+Provides-Extra: dev
+Requires-Dist: pytest>=5.4.0; extra == "dev"
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: pytest-asyncio==0.23.8; extra == "dev"
+Requires-Dist: pytest-xdist==3.3.1; extra == "dev"
+Requires-Dist: pytest-assume==2.4.3; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+# FlowTask #
+
+FlowTask is a plugin-based, component-driven task execution framework for create complex Tasks.
+
+FlowTask runs Tasks defined in JSON, YAML or TOML files, any Task is a combination of Components,
+and every component in the Task run sequentially or depend of others, like a DAG.
+
+Can create a Task combining Commands, Shell scripts and other specific Components (as TableInput: Open a Table using a datasource, DownloadFromIMAP: Download a File from a IMAP Folder, and so on), any Python Callable can be a Component inside a Task, or can extends UserComponent to build your own componets.
+
+Every designed Task can run from CLI, programmatically, via RESTful API (using our aioHTTP-based Handler), called by WebHooks or even dispatched to a external Worker using our built-in Scheduler.
+
+## Quickstart ##
+
+```console
+pip install flowtask
+```
+
+Tasks can organizated into directory structure like this:
+
+tasks /
+    ├── programs /
+      ├── test /
+           ├── tasks /
+
+The main reason of this structure, is maintain organized several tasks by tenant/program, avoiding filling a directory with several task files.
+
+FlowTask support "TaskStorage", a Task Storage is the main repository for tasks, main Task Storage is a directory in any filesystem path (optionally you can syncronize that path using git), but Tasks can be saved onto a Database or a S3 bucket.
+
+## Dependencies ##
+
+ * aiohttp (Asyncio Web Framework and Server) (required by navigator)
+ * AsyncDB
+ * QuerySource
+ * Navigator-api
+ * (Optional) Qworker (for distributing asyncio Tasks on distributed workers).
+
+## Features ##
+
+* Component-based Task execution framework with several components covering several actions (download files, create pandas dataframes from files, mapping dataframe columns to a json-dictionary, etc)
+* Built-in API for execution of Tasks.
+
+### How I run a Task? ###
+
+Can run a Task from CLI:
+```console
+task --program=test --task=example
+```
+
+on CLI, you can pass an ENV (enviroment) to change the environment file on task execution.
+```console
+ENV=dev task --program=test --task=example
+```
+
+or Programmatically:
+```python
+from flowtask import Task
+import asyncio
+
+task = Task(program='test', task='example')
+results = asyncio.run(task.run())
+# we can alternatively, using the execution mode of task object:
+results = asyncio.run(task())
+```
+
+### Requirements ###
+
+* Python >= 3.9
+* asyncio (https://pypi.python.org/pypi/asyncio/)
+* aiohttp >= 3.6.2
+
+### Contribution guidelines ###
+
+Please have a look at the Contribution Guide
+
+* Writing tests
+* Code review
+* Other guidelines
+
+### Who do I talk to? ###
+
+* Repo owner or admin
+* Other community or team contact
+
+### License ###
+
+Navigator is licensed under Apache 2.0 License. See the LICENSE file for more details.