pyfreeflow 0.1.0__py3-none-any.whl

pyfreeflow/registry.py

@@ -0,0 +1,34 @@
+class ClassRegistry():
+    REGISTRY = {}
+
+    @classmethod
+    def class_register_class(cls, other):
+        typename = other.__typename__
+        version = other.__version__
+
+        if typename is None or version is None:
+            return
+
+        if typename not in cls.REGISTRY.keys():
+            cls.REGISTRY[typename] = {}
+
+        if version not in cls.REGISTRY[typename].keys():
+            cls.REGISTRY[typename][version] = other
+
+    @classmethod
+    def get_registered_class(cls, typename, version):
+        return cls.REGISTRY[typename][version]
+
+
+class ExtRegistry(ClassRegistry):
+    REGISTRY = {}
+
+
+class ExtRegister(type):
+    __typename__ = None
+    __version__ = None
+
+    def __new__(meta, name, bases, class_dict):
+        cls = type.__new__(meta, name, bases, class_dict)
+        ExtRegistry.class_register_class(cls)
+        return cls

pyfreeflow/utils.py

@@ -0,0 +1,67 @@
+import copy
+import asyncio
+import pyparsing as pp
+import datetime as dt
+
+
+def deepupdate(base, other, keep=True):
+    assert (isinstance(base, dict) and isinstance(other, dict))
+
+    if not keep:
+        bk = list(base.keys())
+        for k in bk:
+            if k not in other:
+                del base[k]
+
+    for k, v in other.items():
+        if k not in base:
+            base[k] = copy.deepcopy(v)
+        else:
+            if isinstance(v, dict):
+                deepupdate(base[k], v, keep=keep)
+            else:
+                base[k] = copy.deepcopy(v)
+
+
+def asyncio_run(fn, force=False):
+    try:
+        return asyncio.create_task(fn)
+    except RuntimeError:
+        if not force:
+            raise RuntimeError("No loop!")
+        else:
+            return asyncio.run(fn)
+
+
+class DurationParser():
+    INTEGER = pp.Word(pp.nums).setParseAction(lambda t: int(t[0]))
+
+    SECONDS = pp.Opt(pp.Group(INTEGER + "s"))
+    MINUTES = pp.Opt(pp.Group(INTEGER + "m"))
+    HOURS = pp.Opt(pp.Group(INTEGER + "h"))
+    DAYS = pp.Opt(pp.Group(INTEGER + "d"))
+    WEEKS = pp.Opt(pp.Group(INTEGER + "w"))
+    YEARS = pp.Opt(pp.Group(INTEGER + "y"))
+
+    PARSER = YEARS + WEEKS + DAYS + HOURS + MINUTES + SECONDS
+
+    OP = {
+        "y": lambda x: ("days", x * 365),
+        "w": lambda x: ("weeks", x),
+        "d": lambda x: ("days", x),
+        "h": lambda x: ("hours", x),
+        "m": lambda x: ("minutes", x),
+        "s": lambda x: ("seconds", x),
+    }
+
+    @classmethod
+    def parse(cls, duration):
+
+        parsed_duration = cls.PARSER.parseString(duration)
+        delta = {}
+
+        for val, unit in parsed_duration:
+            k, v = cls.OP[unit](val)
+            delta[k] = delta.get(k, 0) + v
+
+        return int(dt.timedelta(**delta).total_seconds())

pyfreeflow-0.1.0.data/scripts/pyfreeflow-cli.py

@@ -0,0 +1,86 @@
+#!python
+import sys
+import argparse
+import pyfreeflow
+import json
+import yaml
+import asyncio
+from platform import system
+
+if system() == "Linux":
+    import uvloop
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+
+
+def json_formatter(x, path):
+    if path:
+        with open(path, "w") as f:
+            json.dump(x, f)
+    else:
+        print(json.dumps(x))
+
+
+def _str_presenter(dumper, data):
+    if "\n" in data:
+        block = "\n".join([line.rstrip() for line in data.splitlines()])
+        if data.endswith("\n"):
+            block += "\n"
+        return dumper.represent_scalar("tag:yaml.org,2002:str", block, style="|")
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+def yaml_formatter(x, path):
+    yaml.add_representer(str, _str_presenter)
+    yaml.representer.SafeRepresenter.add_representer(str, _str_presenter)
+    if path:
+        with open(path, "w") as f:
+            yaml.dump(
+                x, f, default_flow_style=False, allow_unicode=True)
+    else:
+        print(yaml.dump(
+            x, default_flow_style=False, allow_unicode=True))
+
+
+OUTPUT_FORMATTER = {
+    "json": json_formatter,
+    "yaml": yaml_formatter,
+}
+
+
+async def cli(argv):
+    argparser = argparse.ArgumentParser("pyfreeflow-cli")
+
+    argparser.add_argument("--config", "-c", dest="config", type=str,
+                           action="store", default="pyfreeflow.yaml",
+                           required=False, help="Pipeline configuration file")
+    argparser.add_argument("--output", "-o", dest="output", type=str,
+                           action="store",
+                           required=False, help="Pipeline output file")
+    argparser.add_argument("--format", "-f", dest="fmt", type=str,
+                           action="store", choices=OUTPUT_FORMATTER.keys(),
+                           default="json", required=False,
+                           help="Pipeline output file format")
+
+    args = argparser.parse_args(argv)
+
+    with open(args.config, "r") as f:
+        config = yaml.safe_load(f)
+
+    for ext in config.get("ext", []):
+        pyfreeflow.load_extension(ext)
+
+    assert ("pipeline" in config.keys())
+    pipe = pyfreeflow.pipeline.Pipeline(**config.get("pipeline"))
+    output = await pipe.run(config.get("args", {}))
+
+    OUTPUT_FORMATTER[args.fmt](output[0], args.output)
+
+    return output[1]
+
+
+def main(argv):
+    return asyncio.run(cli(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

pyfreeflow-0.1.0.dist-info/METADATA

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: pyfreeflow
+Version: 0.1.0
+Summary: Async service toolchain
+Home-page: https://github.com/senatoreg/pyfreeflow
+Author: Giovanni Senatore
+Author-email: 
+License: AGPL-3.0-or-later
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiofiles>=24.1.0
+Requires-Dist: aiohttp>=3.11.16
+Requires-Dist: networkx>=3.2.1
+Requires-Dist: pydot>=2.0.0
+Requires-Dist: PyYAML>=5.4.1
+Requires-Dist: tomli_w>=1.2.0
+Requires-Dist: psycopg>=3.2.6
+Requires-Dist: pyparsing>=3.1.2
+Requires-Dist: PyJWT==2.10.1
+Requires-Dist: lupa>=2.4
+Requires-Dist: uvloop>=0.21.0
+Requires-Dist: cryptography>=36.0.1
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# PyFreeFlow
+
+## Definition
+
+This library enables the creation of services by composing elementary modules into an operation flow.
+It is designed to be usable even by non-programmers: the library automatically manages parallel execution and error handling. The user only needs to describe the process to be executed.
+
+The flow is defined as a directed graph:
+
+it can have 1..N input nodes and 1..N output nodes;
+
+if multiple output nodes exist, one can be selected as the final result (all nodes are still executed);
+
+each node’s output is passed as input to the next node in the graph.
+
+A special module type, DataTransformer, allows data transformation between nodes with different types, formats, or structures. Since each node’s output is consumed by the following node, a state object is also passed to each node. This makes it possible to store data (usually through DataTransformer) for later use within the flow.
+
+The library is extensible through dynamically loadable modules that follow a defined specification.
+
+To configure a pipeline (the executable unit), two elements are required:
+
+- the list of nodes;
+- the graph definition, written in dot syntax.
+
+## Command line tool
+
+A command-line tool is included to execute pipelines defined in a Yaml file.
+The file must contain three sections: ext, args, pipeline.
+
+- **ext**: list of additional extensions to load.
+- **args**: input parameters for the pipeline.
+- **pipeline**: definition of the flow.
+
+Example: password encryption
+
+```yaml
+ext:
+- pyfreeflow.ext.crypto_operator
+args:
+  username: jdoe
+  password: password
+pipeline:
+  digraph:
+  - parseArgs -> createJson
+  - createJson -> prepareEncrypt
+  - prepareEncrypt -> encrypt
+  - encrypt -> prepareSaveFile
+  - prepareSaveFile -> saveFile
+  name: crypto
+  node:
+  - config:
+      transformer: |2
+        data = {op = "write", data = {username = data.username, password = data.password}}
+    name: parseArgs
+    type: DataTransformer
+    version: '1.0'
+  - name: createJson
+    type: JsonBufferOperator
+    version: '1.0'
+  - config:
+      transformer: |2
+        data = {op = "encrypt", data = data, key = "fernet.key"}
+    name: prepareEncrypt
+    type: DataTransformer
+    version: '1.0'
+  - name: encrypt
+    type: FernetCryptoOperator
+    version: '1.0'
+  - config:
+      transformer: |2
+        data = {op = "write", data = data, path = "pass.enc"}
+    name: prepareSaveFile
+    type: DataTransformer
+    version: '1.0'
+  - config:
+      binary: false
+    name: saveFile
+    type: AnyFileOperator
+    version: '1.0'
+```
+
+Pipeline configuration parameters
+
+- **name**: pipeline name (optional)
+- **digraph**: connections between nodes, identified by unique names
+- **node**: node definitions used in the digraph
+
+Node definition parameters
+
+- **config**: configuration parameters specific to the module
+- **name**: unique node name
+- **type**: module type implementing the node
+- **version**: version of the module
+
+For details about individual modules, refer to their documentation.
+
+# License
+
+This software is available under dual licensing:
+
+## Non-Commercial Use - AGPL v3
+For non-commercial use, this software is released under the GNU Affero General Public License v3.0.
+Any modifications must be shared under the same license.
+
+## Commercial Use
+For commercial use, please contact me.

pyfreeflow-0.1.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+pyfreeflow/__init__.py,sha256=v69pBbooCheyw0O8PvAR12dTI93v2boN4m6AAQy3ysk,900
+pyfreeflow/pipeline.py,sha256=_9qqa9BDiOiGb2R4_1SCmdJ3u214h6pG-vTw8TdKASc,3504
+pyfreeflow/registry.py,sha256=FHkQA9fjY_MRfctwFi8qCoVzC3grhNuBxgCCZmqsppA,861
+pyfreeflow/utils.py,sha256=YFK1i3kfKcBqN77qagHlLgaTTG0U66OGboN4vyRQkGE,1769
+pyfreeflow/ext/__init__.py,sha256=6aIHFWxOnBFB5ztIOBDfHw3uV6uqxxCkV8-dZ7NSqp8,258
+pyfreeflow/ext/buffer_operator.py,sha256=dgYpPPkXVS4Hnq4qLnT4vqYQf52OVuO92LqbUhTkF9g,4970
+pyfreeflow/ext/crypto_operator.py,sha256=ODtAc2RQi9eNw5dLF9W1SQ7agfeWxP5vuJSklLiJLBo,1296
+pyfreeflow/ext/data_transformer.py,sha256=RxSWTBqLO6xYqrRsg4GtMurLxGwKBKuPCPinXDZLspc,6451
+pyfreeflow/ext/env_operator.py,sha256=IoIpkEByrXXyBYHs4gYEPsVc3-PxfPRfxAHU-vj4SMU,605
+pyfreeflow/ext/file_operator.py,sha256=-a8dtZFAOG9V-e3O8bcaEgdY1Kfd3oTDBjsiRhB7JTc,6712
+pyfreeflow/ext/jwt_operator.py,sha256=gGxybcxshodRR84yR-h12AVM_kzAekzkyeBS8b9TFAU,4474
+pyfreeflow/ext/pgsql_executor.py,sha256=o1_uw4W_mNy1l32cKl_isROlk38JYwvjBjlCNIuzyG0,5093
+pyfreeflow/ext/rest_api_requester.py,sha256=GbV5_6-K2WVvTEhC-bo3OBs5U-M0DY6zk-nZrKoQWfA,5441
+pyfreeflow/ext/sleep_operator.py,sha256=8Uy8NWdHdpsHd1AW_Ih9U7lnmYupYWxQtsEafLBCCCQ,1403
+pyfreeflow/ext/types.py,sha256=Y2_7s6JBENhPoF-Ruu6ZcmBwS-iwyzJb56sdLwjL_es,1555
+pyfreeflow-0.1.0.data/scripts/pyfreeflow-cli.py,sha256=Nonkc7HCvdj5z2Xo1aeqHv7YIhuhrRtc6SB51jXwEmk,2468
+pyfreeflow-0.1.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+pyfreeflow-0.1.0.dist-info/METADATA,sha256=--L5qtGpeArUnvMBptIy3Y-SDNLcNXhXPzYj_TcX_uc,4381
+pyfreeflow-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pyfreeflow-0.1.0.dist-info/top_level.txt,sha256=gKG9ntX49i_frKkCy7JrPv5QUx8L14eD53bSDTGPspE,11
+pyfreeflow-0.1.0.dist-info/RECORD,,

pyfreeflow-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+