pyfreeflow 0.1.0__py3-none-any.whl

pyfreeflow/__init__.py

@@ -0,0 +1,34 @@
+import importlib
+import logging
+import pyfreeflow.ext
+import pyfreeflow.pipeline
+from sys import version_info
+
+
+handler = logging.StreamHandler()
+if version_info.major > 3 or (version_info.major == 3 and
+                              version_info.minor > 11):
+    formatter = logging.Formatter('[%(asctime)s] %(name)s - TaskName[%(taskName)s] - %(levelname)s - %(message)s')
+else:
+    formatter = logging.Formatter('[%(asctime)s] %(name)s - %(levelname)s - %(message)s')
+
+handler.setFormatter(formatter)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.WARNING)
+logger.propagate = True
+logger.addHandler(handler)
+
+
+def load_extension(ext_name):
+    logger.info("Loading extension: %s", ext_name)
+    importlib.import_module(ext_name)
+    logger.info("Loaded extension: %s", ext_name)
+
+
+def set_loglevel(level):
+    logger.setLevel(level)
+
+
+def get_logformat():
+    return formatter._fmt

pyfreeflow/ext/__init__.py

@@ -0,0 +1,13 @@
+import importlib
+
+__EXTENSIONS__ = [
+    "file_operator",
+    "buffer_operator",
+    "rest_api_requester",
+    "data_transformer",
+    "pgsql_executor",
+]
+
+for m in __EXTENSIONS__:
+    ext_name = ".".join([__name__, m])
+    importlib.import_module(ext_name)

pyfreeflow/ext/buffer_operator.py

@@ -0,0 +1,159 @@
+from .types import FreeFlowExt
+import json
+import yaml
+import logging
+from sys import version_info
+
+__JSON_TYPENAME__ = "JsonBufferOperator"
+__YAML_TYPENAME__ = "YamlBufferOperator"
+__TOML_TYPENAME__ = "TomlBufferOperator"
+
+
+class JsonBufferOperator(FreeFlowExt):
+    __typename__ = __JSON_TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+        self._action = {
+            "read": self._read,
+            "write": self._write,
+        }
+
+    async def _read(self, raw):
+        if not isinstance(raw, str):
+            self._logger.error("Invalid input format '{}'".format(type(raw)))
+            return None, 101
+
+        try:
+            j = json.loads(raw)
+            return j, 0
+        except Exception as ex:
+            self._logger.error("Cannot load json data '{}' {}".format(raw, ex))
+            return None, 102
+
+    async def _write(self, raw):
+        if not isinstance(raw, (dict, list, tuple)):
+            self._logger.error("Invalid input format '{}'".format(type(raw)))
+            return None, 101
+
+        try:
+            j = json.dumps(raw)
+            return j, 0
+        except Exception as ex:
+            self._logger.error("Cannot write json data '{}' {}".format(
+                raw, ex))
+            return None, 103
+
+    async def do(self, state, data):
+        op = data.get("op", "read")
+        raw = data.get("data", {} if op == "read" else "")
+        rval = await self._action[op](raw)
+        return state, rval
+
+
+class YamlBufferOperator(FreeFlowExt):
+    __typename__ = __YAML_TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+        self._action = {
+            "read": self._read,
+            "write": self._write,
+        }
+
+    async def _read(self, raw):
+        if not isinstance(raw, str):
+            self._logger.error("Invalid input format '{}'".format(type(raw)))
+            return None, 101
+
+        try:
+            j = yaml.safe_load(raw)
+            return j, 0
+        except Exception as ex:
+            self._logger.error("Cannot write yaml data '{}' {}".format(
+                raw, ex))
+            return None, 102
+
+    async def _write(self, raw):
+        if not isinstance(raw, (dict, list, tuple)):
+            self._logger.error("Invalid input format '{}'".format(type(raw)))
+            return None, 101
+
+        try:
+            j = yaml.safe_dump(raw)
+            return j, 0
+        except Exception as ex:
+            self._logger.error("Cannot write yaml data '{}' {}".format(
+                raw, ex))
+            return None, 103
+
+    async def do(self, state, data):
+        op = data.get("op", "read")
+        raw = data.get("data", {} if op == "read" else "")
+        rval = await self._action[op](raw)
+        return state, rval
+
+
+if version_info.major > 3 or (version_info.major == 3 and
+                              version_info.minor > 10):
+    import tomllib
+    import tomli_w
+
+    class TomlBufferOperator(FreeFlowExt):
+        __typename__ = __TOML_TYPENAME__
+        __version__ = "1.0"
+
+        def __init__(self, name, max_tasks=4):
+            super().__init__(name, max_tasks=max_tasks)
+
+            self._logger = logging.getLogger(".".join(
+                [__name__, self.__typename__, self._name]))
+
+            self._action = {
+                "read": self._read,
+                "write": self._write,
+            }
+
+        async def _read(self, raw):
+            if not isinstance(raw, str):
+                self._logger.error("Invalid input format '{}'".format(
+                    type(raw)))
+                return None, 101
+
+            try:
+                j = tomllib.loads(raw)
+                return j, 0
+            except Exception as ex:
+                self._logger.error("Invalid input format '{}' {}".format(
+                    type(raw), ex))
+                return None, 102
+
+        async def _write(self, raw):
+            if not isinstance(raw, (dict, list, tuple)):
+                self._logger.error("Invalid input format '{}'".format(
+                    type(raw)))
+                return None, 101
+
+            try:
+                j = tomli_w.dumps(raw)
+                return j, 0
+            except Exception as ex:
+                self._logger.error("Invalid input format '{}' {}".format(
+                    type(raw), ex))
+                return None, 102
+
+        async def do(self, state, data):
+            op = data.get("op", "read")
+            raw = data.get("data", {} if op == "read" else "")
+            rval = await self._action[op](raw)
+            return state, rval

pyfreeflow/ext/crypto_operator.py

@@ -0,0 +1,44 @@
+from .types import FreeFlowExt
+from cryptography.fernet import Fernet
+import aiofiles
+import logging
+
+__TYPENAME__ = "{}CryptoOperator"
+
+
+class FernetCryptoOperator(FreeFlowExt):
+    __typename__ = __TYPENAME__.format("Fernet")
+    __version__ = "1.0"
+
+    def __init__(self, name, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+        self._action = {
+            "decrypt": self._dec,
+            "encrypt": self._enc,
+        }
+
+    async def _read_key(self, path):
+        async with aiofiles.open(path, "rb") as f:
+            key = await f.read()
+
+        return Fernet(key)
+
+    async def _enc(self, data, key):
+        cipher = await self._read_key(key)
+        d = cipher.encrypt(data.encode('utf-8'))
+        return d.decode('utf-8'), 0
+
+    async def _dec(self, data, key):
+        cipher = await self._read_key(key)
+        d = cipher.decrypt(data.encode('utf-8'))
+        return d.decode('utf-8'), 0
+
+    async def do(self, state, data):
+        op = data.get("op", "decrypt")
+        key = data.get("key")
+        raw = data.get("data", "")
+        rval = await self._action[op](raw, key)
+        return state, rval

pyfreeflow/ext/data_transformer.py

@@ -0,0 +1,217 @@
+from .types import FreeFlowExt
+import logging
+import datetime as dt
+from ..utils import deepupdate, DurationParser
+
+try:
+    import lupa.luajit21 as lupa
+except ImportError:
+    try:
+        import lupa.lua54 as lupa
+    except ImportError:
+        try:
+            import lupa.lua53 as lupa
+        except ImportError:
+            import lupa.lua as lupa
+
+__TYPENAME__ = "DataTransformer"
+
+
+"""
+run parameter:
+{
+  "state": { ... },
+  "data": { ... }
+}
+"""
+
+
+class DataTransformerV1_0(FreeFlowExt):
+    __typename__ = __TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, transformer="", lua_func=[],
+                 force=False, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+        self._force = force
+        self._env = self._create_safe_lua_env()
+
+        self._env.globals().safe_env["now"] = self._dt_now_ts
+        self._env.globals().safe_env["timedelta"] = self._dt_delta_ts
+        self._env.globals().safe_env["dict"] = dict
+        self._env.globals().safe_env["list"] = list
+
+        for _name, _func in lua_func:
+            self._env.globals().safe_env[_name] = _func
+
+        self._transformer = self._env.globals().eval_safe(
+            "\n".join(["function f(state, data)",
+                       transformer,
+                       "return state, data",
+                       "end"]))
+
+        assert (self._transformer is not None)
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+    def __str__(self):
+        return "{typ}(name: {n}, version: {v})".format(
+            typ=self.__typename__, n=self._name, v=self.__version__)
+
+    def _create_safe_lua_env(self):
+        lua = lupa.LuaRuntime(unpack_returned_tuples=True)
+
+        lua.execute("""
+            if not table.find then
+              function table.find(t, value, start_index)
+                local si = start_index or 1
+                for i = si, #t do
+                  if t[i] == value then
+                    return i
+                  end
+                end
+                return nil
+              end
+            end
+
+            if not table.find_value then
+              function table.find_key(t, key)
+                for k, v in pairs(t) do
+                  if k == key then
+                    return v
+                  end
+                end
+                return nil
+              end
+            end
+
+            if not table.find_key then
+              function table.find_key(t, value)
+                for k, v in pairs(t) do
+                  if v == value then
+                    return k
+                  end
+                end
+                return nil
+              end
+            end
+
+            if not string.split then
+              function string.split(s, pattern)
+                local t = {}
+
+                -- Se non è specificato un pattern, usa il default per le
+                -- parole
+                if not pattern then
+                  for m in string.gmatch(s, "%S+") do
+                    table.insert(t, m)
+                  end
+                  return t
+                end
+
+                -- Se il pattern è un singolo carattere (non regex),
+                -- usa la logica del separatore
+                if #pattern == 1 and pattern:match("^[%w%p%s]$") then
+                  for m in string.gmatch(s, "([^" .. pattern:gsub("[%(%)%.%+%-%*%?%[%]%^%$%%]", "%%%1") .. "]+)") do
+                    if m ~= "" then  -- evita stringhe vuote
+                      table.insert(t, m)
+                    end
+                  end
+                else
+                  -- Per pattern regex complessi, usa direttamente il pattern
+                  for m in string.gmatch(s, pattern) do
+                    table.insert(t, m)
+                  end
+                end
+
+                return t
+              end
+            end
+
+            safe_env = {
+              assert = assert,
+              pairs = pairs,
+              ipairs = ipairs,
+              next = next,
+              type = type,
+              tostring = tostring,
+              tonumber = tonumber,
+              string = string,
+              math = math,
+              table = table,
+              print = print,
+              lookup_object_by_key = lookup_object_by_key,
+              NIL = setmetatable({},{__tostring=function() return "nil" end}),
+            }
+
+            function eval_safe(code)
+              local f, e = load(code, "config", "t", safe_env)
+              if not f then
+                print(e)
+                return nil
+              end
+              f()
+              return safe_env.f
+            end
+        """)
+
+        return lua
+
+    def _dt_now_ts(self):
+        return int(dt.datetime.now(dt.UTC).timestamp())
+
+    def _dt_delta_ts(self, duration):
+        return DurationParser.parse(duration)
+
+    def _lua_nil_to_none(self, x):
+        return str(x) == "nil"
+
+    def _lua_to_py(self, a):
+        if lupa.lua_type(a) == "table":
+            if self._lua_nil_to_none(a):
+                return None
+
+            if len(a) == 0:
+                return {k: self._lua_to_py(v) for k, v in a.items()}
+            elif all([isinstance(x, int) for x in a.keys()]):
+                return [self._lua_to_py(v) for v in a.values()]
+            else:
+                return {}
+        else:
+            return a
+
+    def _py_to_lua(self, a):
+        if isinstance(a, dict):
+            return self._env.table_from({k: self._py_to_lua(v) for k, v in a.items()})
+        elif isinstance(a, (list, tuple)):
+            return self._env.table_from([self._py_to_lua(v) for v in a])
+        else:
+            return a
+
+    async def run(self, state, data=({}, 0)):
+        if isinstance(data, list):
+            _data = [x[0] for x in data if x[1] == 0]
+            err = len(_data) == 0
+        else:
+            _data = data[0]
+            err = data[1] != 0
+
+        if err:
+            return state, (None, 103)
+
+        try:
+            s, d = self._transformer(self._py_to_lua(state),
+                                     self._py_to_lua(_data))
+
+            s = self._lua_to_py(s)
+            d = self._lua_to_py(d)
+
+            deepupdate(state, s)
+            if not self._force:
+                return state, (d, 0)
+            else:
+                return state, d
+
+        except Exception as ex:
+            self._logger.error(ex)
+            return state, (None, 101)

pyfreeflow/ext/env_operator.py

@@ -0,0 +1,23 @@
+from .types import FreeFlowExt
+import os
+import logging
+
+__TYPENAME__ = "EnvOperator"
+
+
+class EnvOperator(FreeFlowExt):
+    __typename__ = __TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, vars=[], max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+        self._vars = vars
+
+    async def do(self, state, data):
+        rval = {}
+        for v in self._vars:
+            rval[v] = os.getenv(v)
+        return state, (rval, 0)

pyfreeflow/ext/file_operator.py

@@ -0,0 +1,207 @@
+from .types import FreeFlowExt
+import aiofiles
+import json
+import yaml
+import logging
+from sys import version_info
+
+__ANY_TYPENAME__ = "AnyFileOperator"
+__JSON_TYPENAME__ = "JsonFileOperator"
+__YAML_TYPENAME__ = "YamlFileOperator"
+__TOML_TYPENAME__ = "TomlFileOperator"
+
+
+class AnyFileOperator(FreeFlowExt):
+    __typename__ = __ANY_TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, max_tasks=4, binary=False):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._mode = "b" if binary else ""
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+        self._action = {
+            "read": self._read,
+            "write": self._write,
+        }
+
+    async def _read(self, path, raw):
+        try:
+            async with aiofiles.open(path, "r" + self._mode) as f:
+                contents = await f.read()
+                return contents, 0
+        except Exception as ex:
+            self._logger.error("Cannot load file '{}' {}".format(path, ex))
+            return None, 102
+
+    async def _write(self, path, raw):
+        if self._mode == "b" and isinstance(raw, str):
+            raw_ = raw.encode()
+        else:
+            raw_ = raw
+        try:
+            async with aiofiles.open(path, "w" + self._mode) as f:
+                await f.write(raw_)
+                return raw, 0
+        except Exception as ex:
+            self._logger.error("Cannot write json file '{}' {}".format(
+                path, ex))
+            return raw, 103
+
+    async def do(self, state, data):
+        op = data.get("op", "read")
+        raw = data.get("data", {})
+        path = data.get("path")
+        rval = await self._action[op](path, raw)
+        return state, rval
+
+
+class JsonFileOperator(FreeFlowExt):
+    __typename__ = __JSON_TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+        self._action = {
+            "read": self._read,
+            "write": self._write,
+        }
+
+    async def _read(self, path, raw):
+        try:
+            async with aiofiles.open(path, "r") as f:
+                contents = await f.read()
+                j = json.loads(contents)
+                return j, 0
+        except Exception as ex:
+            self._logger.error("Cannot load json file '{}' {}".format(
+                path, ex))
+            return None, 102
+
+    async def _write(self, path, raw):
+        if not isinstance(raw, (dict, list, tuple)):
+            self._logger.error("Invalid input format '{}'".format(type(raw)))
+            return raw, 101
+
+        try:
+            async with aiofiles.open(path, "w") as f:
+                await f.write(json.dumps(raw))
+                return raw, 0
+        except Exception as ex:
+            self._logger.error("Cannot write json file '{}' {}".format(
+                path, ex))
+            return raw, 103
+
+    async def do(self, state, data):
+        op = data.get("op", "read")
+        raw = data.get("data", {})
+        path = data.get("path")
+        rval = await self._action[op](path, raw)
+        return state, rval
+
+
+class YamlFileOperator(FreeFlowExt):
+    __typename__ = __YAML_TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+        self._action = {
+            "read": self._read,
+            "write": self._write,
+        }
+
+    async def _read(self, path, raw):
+        try:
+            async with aiofiles.open(path, "r") as f:
+                contents = await f.read()
+                j = yaml.safe_load(contents)
+                return j, 0
+        except Exception as ex:
+            self._logger.error("Cannot load yaml file '{}' {}".format(
+                path, ex))
+            return None, 102
+
+    async def _write(self, path, raw):
+        if not isinstance(raw, (dict, list, tuple)):
+            self._logger.error("Invalid input format '{}'".format(type(raw)))
+            return raw, 101
+
+        try:
+            async with aiofiles.open(path, "w") as f:
+                await f.write(yaml.safe_dump(raw))
+                return raw, 0
+        except Exception as ex:
+            self._logger.error("Cannot write yaml file '{}' {}".format(
+                path, ex))
+            return raw, 103
+
+    async def do(self, state, data):
+        op = data.get("op", "read")
+        raw = data.get("data", {})
+        path = data.get("path")
+        rval = await self._action[op](path, raw)
+        return state, rval
+
+
+if version_info.major > 3 or (version_info.major == 3 and
+                              version_info.minor > 10):
+    import tomllib
+    import tomli_w
+
+    class TomlFileOperator(FreeFlowExt):
+        __typename__ = __TOML_TYPENAME__
+        __version__ = "1.0"
+
+        def __init__(self, name, max_tasks=4):
+            super().__init__(name, max_tasks=max_tasks)
+
+            self._logger = logging.getLogger(
+                ".".join([__name__, self.__typename__, self._name]))
+            self._action = {
+                "read": self._read,
+                "write": self._write,
+            }
+
+        async def _read(self, path, raw):
+            try:
+                async with aiofiles.open(path, "r") as f:
+                    contents = await f.read()
+                    j = tomllib.loads(contents)
+                    return j, 0
+            except Exception as ex:
+                self._logger.error("Cannot read toml file '{}' {}".format(
+                    path, ex))
+                return None, 102
+
+        async def _write(self, path, raw):
+            if not isinstance(raw, (dict, list, tuple)):
+                self._logger.error("Invalid input format '{}'".format(
+                    type(raw)))
+                return raw, 101
+
+            try:
+                async with aiofiles.open(path, "w") as f:
+                    await f.write(tomli_w.dump(raw))
+                    return raw, 0
+            except Exception as ex:
+                self._logger.error("Cannot write toml file '{}' {}".format(
+                    path, ex))
+                return raw, 103
+
+        async def do(self, state, data):
+            op = data.get("op", "read")
+            raw = data.get("data", {})
+            path = data.get("path")
+            rval = await self._action[op](path, raw)
+            return state, rval