flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/Odoo.py

@@ -0,0 +1,175 @@
+import asyncio
+import pandas as pd
+from urllib.parse import urljoin
+from xmlrpc.client import ServerProxy
+from .flow import FlowComponent
+from ..interfaces.http import HTTPService
+from ..exceptions import ComponentError
+
+
+class Odoo(HTTPService, FlowComponent):
+    """
+    Odoo
+
+        Overview
+
+            This component interacts with an Odoo server to perform various operations like `search_read` and `create`.
+
+        .. table:: Properties
+        :widths: auto
+
+
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | Name                         | Required | Summary                                                             |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | credentials                  |   Yes    | A dictionary containing connection details to the Odoo server:      |
+        |                              |          | "HOST", "PORT", "DB", "USERNAME", "PASSWORD".                       |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | method                       |   Yes    | The Odoo method to be called (e.g., "search_read", "create").       |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | model                        |   Yes    | The Odoo model to be used for the method call (e.g., "res.partner").|
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | domain                       |   No     | Domain filter for searching records, applicable for "search_read".  |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | fields                       |   No     | Fields to be retrieved, applicable for "search_read".               |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | values                       |   No     | Values to be used for creating records, applicable for "create".    |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+        | use_field_from_previous_step |   No     | Field from previous step to filter records in "search_read".        |
+        +------------------------------+----------+-----------+---------------------------------------------------------+
+
+        Returns
+
+            This component returns a pandas DataFrame containing the results of the Odoo operation.
+    
+
+        Example:
+
+        ```yaml
+        Odoo:
+          credentials:
+            HOST: ODOO_HOST
+            PORT: ODOO_PORT
+            DB: ODOO_DB
+            USERNAME: ODOO_USERNAME
+            PASSWORD: ODOO_PASSWORD
+          model: stock.warehouse
+          method: search_read
+          domain:
+          - - - company_id.name
+              - '='
+              - Pokemon
+          fields:
+          - id
+        ```
+
+    """  # noqa
+    _credentials: dict = {
+        "HOST": str,
+        "PORT": int,
+        "DB": str,
+        "USERNAME": str,
+        "PASSWORD": str,
+    }
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+
+        self.processing_credentials()
+
+        self.common = self.get_server_proxy("common")
+        self.uid = self.common.authenticate(
+            self.credentials["DB"],
+            self.credentials["USERNAME"],
+            self.credentials["PASSWORD"],
+            {},
+        )
+
+        self.models = self.get_server_proxy("object")
+
+        return True
+
+    async def run(self):
+        method_call = getattr(self, f"odoo_{self.method}")
+
+        if not method_call:
+            raise ComponentError("incorrect method or method not provided")
+
+        method_call_task = asyncio.to_thread(method_call)
+
+        method_call_result = await method_call_task
+
+        df = pd.DataFrame(method_call_result)
+        if df.empty:
+            self._logger.warning("Empty DataFrame")
+
+        self.add_metric("NUMROWS", len(df.index))
+        self.add_metric("NUMCOLS", len(df.columns))
+
+        self._result = df
+
+        if self._debug is True:
+            print("::: Printing Result Data === ")
+            print("Data: ", self._result)
+            for column, t in df.dtypes.items():
+                print(column, "->", t, "->", df[column].iloc[0])
+
+        return self._result
+
+    async def close(self):
+        return True
+
+    def get_server_proxy(self, endpoint):
+        port = (
+            f":{self.credentials['PORT']}" if self.credentials["PORT"] != 80 else ""
+        )
+        base_url = f"{self.credentials['HOST']}{port}/xmlrpc/2/"
+        url = urljoin(base_url, endpoint)
+        return ServerProxy(url)
+
+    def model_call(self, method, *args, **kwargs):
+        return self.models.execute_kw(
+            self.credentials["DB"],
+            self.uid,
+            self.credentials["PASSWORD"],
+            self.model,
+            method,
+            *args,
+            **kwargs,
+        )
+
+    def get_values_from_previous_data(self, field=None):
+        if isinstance(self.data, pd.DataFrame) and not self.data.empty:
+            if not field:
+                return [self.data.to_dict("records")]
+
+            return [[(field, "in", self.data[field].to_list())]] or [[]]
+
+    def odoo_search_read(self):
+        prev_step_field = getattr(self, "use_field_from_previous_step", None)
+
+        domain = getattr(
+            self,
+            "domain",
+            self.get_values_from_previous_data(prev_step_field),
+        )
+        fields = getattr(self, "fields", {})
+
+        self._logger.debug(f"Search Read with domain: {domain} and fields: {fields}")
+        return self.model_call(
+            "search_read", domain or [], fields and {"fields": fields}
+        )
+
+    def odoo_create(self):
+        try:
+            # values on step arguments have precedence
+            values = getattr(self, "values", self.get_values_from_previous_data())
+            assert values
+
+            return self.model_call("create", values)
+
+        except AssertionError:
+            raise ComponentError(
+                '"values" required for "create" method'
+            )

flowtask/components/OdooInjector.py

@@ -0,0 +1,192 @@
+import asyncio
+import math
+from collections.abc import Callable
+from urllib.parse import urljoin
+import numpy as np
+import pandas as pd
+from .flow import FlowComponent
+from ..interfaces.http import HTTPService
+from ..exceptions import ComponentError, TaskError
+
+
+class OdooInjector(HTTPService, FlowComponent):
+    """
+    OdooInjector
+
+        Overview
+
+            The OdooInjector class is a component for injecting data into an Odoo server using a provided API key.
+            This component takes data from a Pandas DataFrame, formats it as payload, and sends it to an Odoo endpoint
+            specified in the credentials, facilitating seamless integration with Odoo’s API.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | credentials    |   Yes    | A dictionary containing connection details for the Odoo server:        |
+            |                |          | "HOST", "PORT", "APIKEY", and "INJECTOR_URL".                          |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | model          |   Yes    | The Odoo model into which data will be injected.                       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | headers        |   No     | Optional headers to include with the API request. Defaults to API key.  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | data           |   Yes    | The data to inject, formatted as a list of dictionaries from DataFrame. |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a Boolean indicating whether the data injection was successful.
+            In case of errors, detailed logging is provided, and an exception is raised with the error message.
+            Additionally, the component tracks successful API interactions and logs any unsuccessful payload deliveries
+            for debugging and tracking.
+    
+
+        Example:
+
+        ```yaml
+        OdooInjector:
+          credentials:
+            APIKEY: ODOO_APIKEY
+            HOST: ODOO_HOST
+            PORT: ODOO_PORT
+            INJECTOR_URL: ODOO_INJECTOR_URL
+          model: fsm.location
+          chunk_size: 10
+        ```
+
+    """ #noqa
+
+    accept: str = "application/json"
+    auth_type: str = "api_key"
+    download = None
+    chunk_size: int = 2000
+    _credentials: dict = {
+        "HOST": str,
+        "PORT": str,
+        "APIKEY": str,
+        "INJECTOR_URL": str,
+    }
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        super().__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+
+    async def start(self, **kwargs):
+        if self.previous and isinstance(self.input, pd.DataFrame):
+            self.data = self.split_chunk_df(self.input, self.chunk_size)
+
+        self.processing_credentials()
+
+        self.headers = {"api-key": self.credentials["APIKEY"]}
+
+        self.url = self.get_url()
+
+        return True
+
+    async def run(self):
+        total_rows = 0
+        error_count = 0
+        errors = []
+
+        for idx, chunk in enumerate(self.data):
+            payload = self.get_payload(chunk.to_dict(orient="records"))
+
+            try:
+                self._logger.info(f"Sending chunk {idx} ...")
+                result, error = await self.session(
+                    url=self.url, method="post", data=payload, use_json=True
+                )
+
+                if isinstance(result, dict) and result.get("ids"):
+                    self._logger.debug(result)
+                    total_rows += len(result["ids"])
+                else:
+                    self._logger.error(f"Chunk {idx} failed: result: {result} :: error: {error}")
+                    error_count += len(chunk)  # Assuming the whole chunk failed
+                    errors.append(f"Chunk {idx} failed with error: {error}")
+
+            except Exception as e:
+                self._logger.error(f"Error while sending chunk {idx}: {str(e)}. Continuing with next chunk...")
+                error_count += len(chunk)  # Assuming the whole chunk failed
+                errors.append(f"Chunk {idx} exception: {str(e)}")
+
+        # Register metrics
+        self.add_metric("PROCESSED ROWS", total_rows)
+        self.add_metric("FAILED ROWS", error_count)
+
+        # Raise error if there were any failures
+        if errors:
+            errors.append(f"PROCESSED ROWS: {total_rows}, FAILED ROWS: {error_count}")
+            raise TaskError("\n".join(errors))
+
+        return True
+
+
+    async def close(self):
+        return True
+
+    def get_url(self):
+        port = (
+            f":{self.credentials['PORT']}" if self.credentials["PORT"] != "80" else ""
+        )
+        base_url = f"{self.credentials['HOST']}{port}"
+        url = urljoin(base_url, self.credentials["INJECTOR_URL"])
+        return url
+
+    def get_payload(self, chunk_data):
+        return {
+            "model": self.model,
+            "options": {
+                # 'has_headers': True,
+                "advanced": False,
+                "keep_matches": False,
+                "name_create_enabled_fields": getattr(self, "name_create_enabled_fields", {}),
+                "import_set_empty_fields": [],
+                "import_skip_records": [],
+                "fallback_values": {},
+                "skip": 0,
+                "limit": self.chunk_size,
+                # 'encoding': '',
+                # 'separator': '',
+                "quoting": '"',
+                # 'sheet': 'Sheet1',
+                "date_format": "",
+                "datetime_format": "",
+                "float_thousand_separator": ",",
+                "float_decimal_separator": ".",
+                "fields": [],
+            },
+            "data": chunk_data,
+        }
+
+    def split_chunk_df(self, df: pd.DataFrame, chunk_size: int):
+        """
+        Splits a DataFrame into chunks of a specified size.
+
+        Parameters:
+        df (pd.DataFrame): The DataFrame to be split.
+        chunk_size (int): The maximum number of rows per chunk.
+
+        Returns:
+        list: A list of DataFrame chunks.
+            If the DataFrame is empty, returns an empty list.
+        """
+        if df.empty:
+            return []
+
+        split_n = math.ceil(len(df) / chunk_size)
+
+        # Split into chunks of n rows
+        return np.array_split(df, split_n)  # Returns list of DataFrames

flowtask/components/OpenFromXML.py

@@ -0,0 +1,126 @@
+import io
+from pathlib import Path
+from lxml import etree
+from ..exceptions import ComponentError, FileError
+from .OpenWithBase import FlowComponent
+
+
+class OpenFromXML(FlowComponent):
+    """
+    OpenFromXML
+
+
+        Overview
+
+            This component opens an XML file and returns it as an XML etree Object.
+
+        .. table:: Properties
+        :widths: auto
+
+
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | Name         | Required | Summary                                                           |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | directory    |   No     | The directory where the XML file is located.                      |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | filename     |   Yes    | The name of the XML file to be opened.                            |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | use_strings  |   No     | If True, the component will treat the input as a string containing|
+        |              |          | XML data.                                                         |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | as_nodes     |   No     | If set, specifies the node to be extracted from the XML tree.     |
+        +--------------+----------+-----------+-------------------------------------------------------+
+
+    Returns:
+        - An `lxml.etree.ElementTree` object if the entire XML tree is parsed.
+        - A list of nodes (`lxml.etree.Element`) if `as_nodes` is set.
+    """
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Get if directory exists
+        """
+        if self.previous:
+            print("PREVIOUS FILE >", self.previous, type(self.input))
+            if isinstance(self.input, dict):
+                try:
+                    filenames = list(self.input.keys())
+                    if filenames:
+                        self.filename = filenames[0]
+                except (TypeError, IndexError) as exc:
+                    raise FileError("File is empty or doesn't exists") from exc
+            else:
+                self.data = self.input
+        if hasattr(self, "directory"):
+            self.path = Path(self.directory).resolve()
+            if not self.path.exists() or not self.path.is_dir():
+                raise ComponentError(
+                    f"Directory not found: {self.directory}", status=404
+                )
+            # TODO: all logic for opening from File, pattern, etc
+            if not self.filename:
+                raise ComponentError("Filename empty", status=404)
+
+    async def close(self):
+        """
+        close.
+            close method
+        """
+        pass
+
+    async def run(self):
+        """
+        run.
+
+            Open the XML file and return the object
+        """
+        root = None
+        if hasattr(self, "use_strings") and self.use_strings is True:
+            fp = open(self.filename, "r")
+            self.data = fp.read()
+            self.filename = None
+        if self.filename:
+            print(f"Opening XML filename {self.filename}")
+            # Create a parser object
+            parser = etree.XMLParser(encoding="utf-8")
+            # open XML from File
+            tree = etree.parse(str(self.filename), parser)
+            try:
+                root = tree.getroot()
+                if etree.iselement(root):
+                    self._result = tree
+                    numrows = int(tree.xpath("count(/*)"))
+                    self.add_metric("NUMROWS", numrows)
+                    self.add_metric("OPENED_FILE", self.filename)
+                    if hasattr(self, "as_nodes"):
+                        objs = root.findall(self.node)
+                        if objs:
+                            self._result = objs
+                    return self._result
+            except Exception as err:
+                print(err)
+                return False
+        elif self.data:
+            if isinstance(self.data, str):
+                # open XML from string
+                xml = io.BytesIO(self.data.encode("utf-8"))
+                parser = etree.XMLParser(recover=True, encoding="utf-8")
+                root = etree.parse(xml, parser)
+                try:
+                    if etree.iselement(root.getroot()):
+                        self._result = root
+                except Exception as err:
+                    print("Error on parsing XML Tree: ", err)
+                    return False
+            else:
+                #  TODO: check if data is already an XML element
+                try:
+                    print(etree.iselement(self.data.getroot()))
+                except Exception as err:
+                    raise ComponentError(f"Invalid Object lxml, Error: {err}") from err
+                root = self.data
+            self._result = root
+            return self._result
+        else:
+            return False

flowtask/components/OpenWeather.py

@@ -0,0 +1,41 @@
+from querysource.exceptions import DataNotFound as QSNotFound
+from ..exceptions import ComponentError, DataNotFound
+from .QSBase import QSBase
+
+class OpenWeather(QSBase):
+    type = "weather"
+    _driver = "openweather"
+    _mapping: dict = {
+        "dt": "timestamp",
+        "main": "temperature",
+        "coord": "coordinates",
+        "name": "city",
+        "id": "city_id"
+    }
+    _from_conditions: list = [
+        "country",
+        "store_id",
+        "store_name"
+    ]
+
+    async def weather(self):
+        try:
+            rst = []
+            resultset = await self._qs.weather()
+            print('CONDITIONS > ', self._qs._conditions)
+            data = resultset.copy()
+            for key, value in data.items():
+                if key in self._mapping:
+                    resultset[self._mapping[key]] = value
+                    del resultset[key]
+            if self._from_conditions:
+                for key in self._from_conditions:
+                    if key in self._qs._conditions:
+                        resultset[key] = self._qs._conditions[key]
+            rst.append(resultset)
+            return rst
+        except QSNotFound as err:
+            raise DataNotFound(f"Coordinates Not Found: {err}") from err
+        except Exception as err:
+            self._logger.exception(err)
+            raise ComponentError(f"OpenWeather ERROR: {err!s}") from err