pyfreeflow 0.1.0__py3-none-any.whl

pyfreeflow/ext/jwt_operator.py

@@ -0,0 +1,140 @@
+from .types import FreeFlowExt
+import jwt
+import hashlib
+import logging
+import datetime as dt
+from ..utils import DurationParser
+
+__TYPENAME__ = "JwtOperator"
+
+
+"""
+run parameter:
+{
+  "state": { ... },
+  "data": {
+    "op": "encode|decode",  # operation (default encode)
+    "headers": {},  # for encode (optional)
+    "algorithm": {},  # for encode (optional)
+    "body": ""  # for encode
+    "token": ""  # for decode
+    "headers_only": True|False  # only unverified headers (default False)
+  }
+}
+"""
+
+
+class JwtOperatorV1_0(FreeFlowExt):
+    __typename__ = __TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, pubkey_files, privkey_files, algorithms=["HS256"],
+                 headers={}, verify_sign=True, verify_exp=True,
+                 required_claims=[], duration=None, not_before=None,
+                 issuer=None, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._algorithms = algorithms
+
+        self._pub_key = {}
+        for key_file in pubkey_files:
+            with open(key_file, "rb") as f:
+                content = f.read()
+                h = hashlib.sha256(content).hexdigest()
+                self._pub_key[h] = content
+
+        self._priv_key = {}
+        for key_file in privkey_files:
+            with open(key_file, "rb") as f:
+                content = f.read()
+                h = hashlib.sha256(content).hexdigest()
+                self._priv_key[h] = content
+
+        self._default_pub_key = list(self._pub_key.keys())[0]
+        self._default_priv_key = list(self._priv_key.keys())[0]
+
+        self._headers = headers
+        self._options = {
+            "require": [x for x in required_claims],
+            "verify_signature": verify_sign,
+            "verify_exp": verify_exp,
+        }
+
+        self._duration = DurationParser.parse(duration) if isinstance(duration, str) else None
+        self._not_before = DurationParser.parse(not_before) if isinstance(not_before, str) else None
+        self._issuer = issuer
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+        self._action = {
+            "encode": self._do_encode,
+            "decode": self._do_decode,
+        }
+
+    def _get_priv_key(self, data):
+        kid = data.get("kid", self._default_priv_key)
+        return kid, self._priv_key[kid]
+
+    def _get_pub_key(self, headers):
+        kid = headers.get("kid", self._default_pub_key)
+        return kid, self._pub_key[kid]
+
+    async def _do_encode(self, data):
+        body = data.get("body")
+
+        if not isinstance(body, dict):
+            self._logger.error("Invalid input format '{}'".format(type(body)))
+            return None, 101
+
+        hdr = self._headers | data.get("headers", {})
+
+        kid, key = self._get_priv_key(data)
+
+        hdr["kid"] = kid
+
+        if self._duration is not None and "exp" not in body:
+            body["exp"] = int(dt.datetime.now(dt.UTC).timestamp() + self._duration)
+
+        if self._not_before is not None and "nbf" not in body:
+            body["nbf"] = int(dt.datetime.now(dt.UTC).timestamp() + self._not_before)
+
+        if "iat" not in body:
+            body["iat"] = int(dt.datetime.now(dt.UTC).timestamp())
+
+        if self._issuer is not None and "iss" not in body:
+            body["iss"] = self._issuer
+
+        algorithm = data.get("algorithm", self._algorithms[0])
+
+        return {"token": jwt.encode(body, key,
+                                    algorithm=algorithm, headers=hdr)}, 0
+
+    async def _do_decode(self, data):
+        token = data.get("token")
+        hdr_only = data.get("headers_only", False)
+
+        if not isinstance(token, str):
+            self.logger.error("Invalid input format '{}'".format(type(token)))
+            return None, 101
+
+        try:
+            hdr = jwt.get_unverified_header(token)
+
+            kid, key = self._get_pub_key(hdr)
+
+            if not hdr_only:
+                body = jwt.decode(token, key,
+                                  algorithms=self._algorithms,
+                                  options=self._options)
+            else:
+                body = None
+
+            return {"headers": hdr, "body": body}, 0
+        except Exception as ex:
+            self._logger.error(ex)
+            return {"headers": None, "body": None}, 102
+
+    async def do(self, state, data):
+        op = data.get("op", "encode")
+        return state, await self._action[op](data)

pyfreeflow/ext/pgsql_executor.py

@@ -0,0 +1,167 @@
+from .types import FreeFlowExt
+import psycopg
+import asyncio
+import logging
+
+__TYPENAME__ = "PgSqlExecutor"
+
+
+#
+# Connection Pool
+#
+class ConnectionPool():
+    CLIENT = {}
+    POOL = {}
+    LOCK = asyncio.Lock()
+    LOGGER = logging.getLogger(".".join([__name__, "ConnectionPool"]))
+
+    @classmethod
+    def register(cls, client_name, conninfo, max_size=4):
+        if client_name not in cls.CLIENT.keys():
+            cls.CLIENT[client_name] = {
+                "conninfo": conninfo,
+                "lock": asyncio.BoundedSemaphore(max_size)}
+
+            if client_name not in cls.POOL.keys():
+                cls.POOL[client_name] = asyncio.Queue()
+
+    @classmethod
+    async def get(cls, client_name):
+        if client_name not in cls.CLIENT.keys():
+            return None
+
+        lock = cls.CLIENT[client_name]["lock"]
+        cls.LOGGER.debug("GET {} Lock[{}/{}/{}] Queue[{}]".format(
+            client_name, len(lock._waiters) if lock._waiters else 0,
+            lock._value, lock._bound_value,
+            cls.POOL[client_name].qsize()))
+        await lock.acquire()
+
+        try:
+            while not cls.POOL[client_name].empty():
+                conn = await cls.POOL[client_name].get()
+                if await cls.is_alive(conn):
+                    return conn
+        except psycopg.errors.Error as ex:
+            lock.release()
+            raise ex
+
+        conninfo = cls.CLIENT[client_name]["conninfo"]
+        return await psycopg.AsyncConnection.connect(conninfo)
+
+    @classmethod
+    async def release(cls, client_name, conn):
+        if client_name in cls.CLIENT.keys():
+            lock = cls.CLIENT[client_name]["lock"]
+            await cls.POOL[client_name].put(conn)
+            lock.release()
+            cls.LOGGER.debug("RELEASE {} Lock[{}/{}/{}] Queue[{}]".format(
+                client_name, len(lock._waiters) if lock._waiters else 0,
+                lock._value, lock._bound_value,
+                cls.POOL[client_name].qsize()))
+        else:
+            conn.close()
+
+    @staticmethod
+    async def is_alive(conn):
+        try:
+            async with conn.cursor() as cur:
+                cur = await cur.execute("SELECT 1;")
+                d = await cur.fetchall()
+                del d
+            await conn.commit()
+            return True
+        except psycopg.errors.Error:
+            return False
+
+
+#
+# PgSql Executor
+#
+class PgSqlExecutor(FreeFlowExt):
+    __typename__ = __TYPENAME__
+    __version__ = "1.0"
+
+    CONNECTION_STRING = "postgresql://{userspec}{hostspec}{dbspec}{paramspec}"
+
+    def __init__(self, name, username=None, password=None, host=[],
+                 dbname=None, param={}, statement=None, max_connections=4,
+                 max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        userspec = self._conninfo_helper(username, password, sep=":")
+
+        hostspec = ",".join(host)
+        hostspec = "@" + hostspec if len(hostspec) > 0 else ""
+
+        dbspec = self._conninfo_helper(None, dbname, sep="/")
+
+        if "connect_timeout" not in param.keys():
+            param["connect_timeout"] = 30
+
+        paramspec = "?" + "&".join([k + "=" + str(v) for k, v in param.items()])
+
+        self._conninfo = self.CONNECTION_STRING.format(
+            userspec=userspec, hostspec=hostspec, dbspec=dbspec,
+            paramspec=paramspec)
+        self._stm = statement
+        assert (self._stm is not None)
+
+        ConnectionPool.register(self._name, self._conninfo,
+                                max_size=max_connections)
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+    def _conninfo_helper(self, a, b, sep=":"):
+        return "{a}{s}{b}".format(
+            a=a if a else "",
+            s=sep if b else "",
+            b=b if b else ""
+        )
+
+    async def __aenter__(self):
+        await self._ensure_connection()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self._close()
+
+    def __del__(self):
+        pass
+
+    async def do(self, state, data):
+        if self._stm is None:
+            return state, (data, 101)
+
+        rs = {"resultset": []}
+        rc = 0
+
+        try:
+            conn = await ConnectionPool.get(self._name)
+        except psycopg.errors.Error as ex:
+            self._logger.error(ex)
+            return state, (rs, 101)
+
+        try:
+            async with conn.cursor() as cur:
+                value = data.get("value")
+
+                if value and len(value) > 1:
+                    await cur.executemany(self._stm, value)
+                else:
+                    await cur.execute(self._stm, value)
+
+                if cur.description:
+                    rs["resultset"] = await cur.fetchall()
+
+            await conn.commit()
+        except psycopg.errors.Error as ex:
+            rc = 102
+            if not conn.closed:
+                await conn.rollback()
+            self._logger.error(ex)
+        finally:
+            await ConnectionPool.release(self._name, conn)
+
+        return state, (rs, rc)

pyfreeflow/ext/rest_api_requester.py

@@ -0,0 +1,159 @@
+from .types import FreeFlowExt
+import aiohttp
+import yarl
+import multidict
+import json
+import ssl
+import asyncio
+import logging
+from ..utils import asyncio_run
+
+__TYPENAME__ = "RestApiRequester"
+
+
+"""
+run parameter:
+{
+  "state": { ... },
+  "data": {
+    "headers": {},
+    "body": {}
+  }
+}
+"""
+
+
+class RestApiRequesterV1_0(FreeFlowExt):
+    __typename__ = __TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, url, method="GET", headers={}, timeout=300,
+                 sslenabled=True, insecure=False, cafile=None, capath=None,
+                 cadata=None, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+
+        self._url = url
+        self._timeout = timeout
+        self._headers = headers
+        self._method = method.upper()
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+        if sslenabled:
+            self._ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+            if insecure:
+                self._ssl_context.check_hostname = False
+                self._ssl_context.verify_mode = ssl.CERT_NONE
+            else:
+                self._ssl_context.check_hostname = True
+                self._ssl_context.verify_mode = ssl.CERT_REQUIRED
+            if cafile or capath or cadata:
+                self._ssl_context.load_verify_locations(
+                    cafile=cafile, capath=capath, cadata=cadata)
+        else:
+            self._ssl_context = None
+
+        self._session = None
+
+        self._method_op = {
+            "GET": self._do_get,
+            "POST": self._do_post,
+        }
+
+    def __str__(self):
+        return "{typ}(name: {n}, version: {v}, url: {u}, headers: {h}, timeout: {tm_out})".format(
+            typ=self.__typename__, n=self._name, v=self.__version__,
+            u=self._url, h=self._headers, tm_out=self._timeout)
+
+    def _validate_ssl_ca_config(self, config):
+        keys = ["file", "path", "data"]
+        return isinstance(config, dict) and len([k for k in config.keys() if k in keys]) > 0
+
+    async def _ensure_session(self) -> aiohttp.ClientSession:
+        if self._session is None or self._session.closed:
+            t = aiohttp.ClientTimeout(total=self._timeout)
+            self._session = aiohttp.ClientSession(timeout=t)
+
+    async def _close(self):
+        if self._session and not self._session.closed:
+            await self._session.close()
+
+    async def __aenter__(self):
+        await self._ensure_session()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self._close()
+
+    def __del__(self):
+        asyncio_run(self._close(), force=True)
+
+    def _multidict_to_dict(self, x):
+        if isinstance(x, yarl.URL):
+            return str(x)
+        elif isinstance(x, multidict._multidict.CIMultiDictProxy):
+            return dict(x)
+        return x
+
+    def _prepare_request_p(self, x):
+        return {k: str(v) for k, v in x.items()} if x is not None else x
+
+    async def _do_request(self, method, url, headers=None, params=None,
+                          data=None):
+        try:
+            await self._ensure_session()
+
+            async with self._session.request(
+                    method, url, headers=headers, params=params, data=data,
+                    ssl=self._ssl_context, allow_redirects=True) as resp:
+
+                if resp.status >= 400:
+                    self._logger.error(f"'{url}' response code {resp.status}")
+                    return (
+                        {"req": {}, "headers": {}, "body": {}}, 102)
+
+                raw = await resp.read()
+
+                try:
+                    body = json.loads(raw.decode("utf-8"))
+                except json.JSONDecodeError:
+                    body = raw.decode("utf-8")  # oppure tenere i bytes
+            req_info = {k: self._multidict_to_dict(v)
+                        for k, v in dict(resp._request_info._asdict()).items()}
+            return (
+                {"req": req_info, "headers": dict(resp.headers), "body": body}, 0)
+
+        except aiohttp.ClientError as ex:
+            self._logger.error("aiohttp request error %s", ex)
+            return (
+                {"req": {}, "headers": {}, "body": {}}, 101)
+        except asyncio.exceptions.TimeoutError as ex:
+            self._logger.error("aiohttp timeout on %s error %s", url, ex)
+            return (
+                {"req": {}, "headers": {}, "body": {}}, 104)
+
+    async def _do_get(self, state, data):
+        headers = self._headers | data.get("headers", {})
+
+        url = self._url.format(**data.get("urlcomp", {}))
+        query_params = data.get("body", {})
+
+        return await self._do_request(
+            "GET", url, headers=headers, params=query_params)
+
+    async def _do_post(self, state, data):
+        headers = self._headers | data.get("headers", {})
+
+        url = self._url.format(**data.get("urlcomp", {}))
+        body_bytes = json.dumps(data.get("body", {})).encode("utf-8")
+
+        return await self._do_request("POST", url, headers=headers,
+                                      data=body_bytes)
+
+    async def do(self, state, data):
+        if isinstance(data, dict):
+            rval = await self._method_op[self._method](state, data)
+            return state, rval
+        self._logger.error("Bad request expected 'dict' got '{}'".format(type(data)))
+        return (state, ({"headers": {}, "body": {}}, 103))

pyfreeflow/ext/sleep_operator.py

@@ -0,0 +1,53 @@
+from .types import FreeFlowExt
+import random
+import logging
+import asyncio
+
+__TYPENAME__ = "SleepOperator"
+
+
+"""
+run parameter:
+{
+  "state": { ... },
+  "data": {}
+}
+"""
+
+
+class SleepOperatorV1_0(FreeFlowExt):
+    __typename__ = __TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, sleep=5, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+        self._sleep = sleep
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+    async def run(self, state, data):
+        self._logger.debug("%s sleeping for %d", self._name, self._sleep)
+        await asyncio.sleep(self._sleep)
+
+        return state, data
+
+
+class RandomSleepOperatorV1_0(FreeFlowExt):
+    __typename__ = "Random" + __TYPENAME__
+    __version__ = "1.0"
+
+    def __init__(self, name, sleep_min=5, sleep_max=10, max_tasks=4):
+        super().__init__(name, max_tasks=max_tasks)
+        self._sleep_min = sleep_min
+        self._sleep_max = sleep_max
+
+        self._logger = logging.getLogger(".".join([__name__, self.__typename__,
+                                                   self._name]))
+
+    async def run(self, state, data):
+        t = random.randint(self._sleep_min, self._sleep_max)
+        self._logger.debug("%s sleeping for %d", self._name, t)
+        await asyncio.sleep(t)
+
+        return state, data

pyfreeflow/ext/types.py

@@ -0,0 +1,58 @@
+from ..registry import ExtRegister
+import asyncio
+
+"""
+run parameter:
+{
+  "state": { ... },
+  "param": {
+    <CLASS SPECIFIC PARAMETERS>
+  }
+}
+"""
+
+
+class FreeFlowExt(metaclass=ExtRegister):
+    def __init__(self, name, max_tasks=4):
+        self._name = name
+        self._max_tasks = max_tasks
+
+    async def do(self, state, data):
+        raise NotImplementedError
+
+    async def unpack(self, state, data):
+        if isinstance(data, list):
+            loop = asyncio.get_running_loop()
+
+            # [param0, param1, ...]
+            cur = self._max_tasks
+            _data = []
+            aws = []
+
+            for i, p in enumerate(data):
+                if cur == 0:
+                    await asyncio.wait(aws, loop=loop,
+                                       return_when=asyncio.FIRST_COMPLETED)
+                    cur += 1
+
+                if p[1] == 0:
+                    aws.append(loop.create_task(
+                        self.do(state, p[0]),
+                        name=self._name + "-unpack-" + str(i)))
+                    cur -= 1
+
+            await asyncio.wait(aws, loop=loop,
+                               return_when=asyncio.ALL_COMPLETED)
+
+            for task in aws:
+                state, p = await task
+                _data.append(p)
+            return state, _data
+        else:
+            # param0 or param1 or ...
+            if data[1] == 0:
+                return await self.do(state, data[0])
+            return state, data
+
+    async def run(self, state, data={}):
+        return await self.unpack(state, data)

pyfreeflow/pipeline.py

@@ -0,0 +1,127 @@
+from .registry import ExtRegistry
+import networkx as nx
+import io
+import copy
+import asyncio
+import logging
+
+"""
+Example of configuratio file
+
+last: "D"  # Optional
+node:
+- name: "A"
+  type: "RestApiRequester"
+  version: "1.0"
+  config: {}
+- name: "B"
+  type: "DataTransformer"
+  version: "1.0"
+  config: {}
+- name: "C"
+  type: "RestApiRequester"
+  version: "1.0"
+  config: {}
+- name: "D"
+  type: "DataTransformer"
+  version: "1.0"
+  config: {}
+digraph:
+  - A -> B
+  - B -> C
+  - A -> D
+"""
+
+
+class Pipeline():
+    def __init__(self, node, digraph, last=None, name="stream"):
+        self._name = name
+        self._registry = {}
+        self._data = {}
+        self._state = {}
+        self._last = last
+        self._lock = asyncio.Lock()
+        self._cond = asyncio.Condition()
+
+        self._logger = logging.getLogger(".".join([__name__, "Pipeline",
+                                                   self._name]))
+
+        for cls in node:
+            cls_name = cls.get("name")
+            cls_config = cls.get("config", {})
+            cls_type = cls.get("type")
+            cls_version = cls.get("version")
+
+            assert (cls_name not in self._registry.keys())
+
+            self._registry[cls_name] = ExtRegistry.get_registered_class(
+                cls_type, cls_version)(cls_name, **cls_config)
+
+        dot = io.StringIO("digraph D {" + "\n".join(digraph) + "}")
+        self._G = nx.nx_pydot.read_dot(dot)
+
+        self._tree = list(nx.topological_sort(self._G))
+
+    async def _cleanup(self):
+        t = self._state
+        self._state = {}
+        del t
+
+        t = self._data
+        self._data = {}
+        del t
+
+    async def _task(self, n, _data):
+        try:
+            self._state, self._data[n] = await self._registry[n].run(
+                self._state, _data)
+
+        except Exception as ex:
+            self._logger.error(ex)
+        finally:
+            async with self._cond:
+                self._cond.notify()
+
+    async def run(self, data={}):
+        async with self._lock:
+            degrees = {x[0]: x[1] for x in self._G.in_degree()}
+            loop = asyncio.get_running_loop()
+
+            pending = len(self._tree)
+            task = {}
+
+            while pending > 0:
+                nodes = [k for k, v in degrees.items() if v == 0]
+                for n in nodes:
+                    _prev = list(self._G.predecessors(n))
+                    if len(_prev) > 1:
+                        _data = [self._data.get(x) for x in _prev]
+                    elif len(_prev) == 1:
+                        _data = self._data.get(_prev[0])
+                    else:
+                        _data = (data, 0)
+
+                    task[n] = loop.create_task(self._task(n, _data),
+                                               name=n)
+                    degrees[n] -= 1
+
+                async with self._cond:
+                    await self._cond.wait()
+
+                nodes.clear()
+                for tname, t in {k: v for k, v in task.items() if v.done()}.items():
+                    degrees[tname] -= 1
+                    pending -= 1
+                    del task[tname]
+                    for succ in self._G.successors(tname):
+                        degrees[succ] -= 1
+
+            if self._last is not None:
+                _data = self._data.get(self._last, {})
+            else:
+                _data = self._data.get(self._tree[-1], {})
+
+            rep = (copy.deepcopy(_data[0]), _data[1])
+
+            await self._cleanup()
+        return rep