flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/VivaTracker.py

@@ -0,0 +1,140 @@
+from pathlib import Path
+from urllib.parse import urljoin
+from bs4 import BeautifulSoup
+import magic
+import orjson
+import pandas as pd
+from ..exceptions import ComponentError
+from .ASPX import ASPX
+from ..conf import TASK_PATH
+
+
+class VivaTracker(ASPX):
+    async def start(self, **kwargs):
+        directory = Path(TASK_PATH, self._program, "reports")
+        report_json_path = Path(directory).joinpath(self.report_filename)
+
+        try:
+            with open(report_json_path) as f:
+                report_values_dict = orjson.loads(f.read())
+        except FileNotFoundError as ex:
+            raise ComponentError(
+                f"Can't read the additional params file. Make sure there's a file {report_json_path}"
+            ) from ex
+
+        additional_component_attrs = {
+            "base_url": "https://mso.vivatracker.com",
+            "login_user_payload_key": "ctl00$ContentPlaceHolder1$txtEmail",
+            "login_password_payload_key": "ctl00$ContentPlaceHolder1$txtPassword",
+            "login_button_payload_key": "ctl00$ContentPlaceHolder1$btnSubmit",
+            "login_button_payload_value": "Login",
+            **report_values_dict,
+        }
+
+        for attr, value in additional_component_attrs.items():
+            self.__setattr__(attr, value)
+
+        self.additional_payload_params = self.process_mask("additional_payload_params")
+
+        self._report_url = urljoin(self.base_url, f"{self.report_path}.aspx")
+
+        return await super().start()
+
+    async def _go_to_report(self):
+        await self.aspx_session(self._report_url)
+
+    def _get_report_payload(self, btn_id, btn_value):
+        return {
+            **self._views,
+            **self.additional_payload_params,
+            f"ctl00$ContentPlaceHolder1${btn_id}": btn_value,
+        }
+
+    def _extract_additional_values(self, soup: BeautifulSoup):
+        """Extract additional values from the html report given a df column and
+        an element id present in the DOM
+
+        Args:
+            response (requests.Response): http response
+
+        Returns:
+            dict: key: dataframe column, value: value extracted
+        
+
+        Example:
+
+        ```yaml
+        VivaTracker:
+          credentials:
+            username: VIVATRACKER_USERNAME
+            password: VIVATRACKER_PASSWORD
+          report_filename: sales_by_invoice_report.json
+          masks:
+            '{yesterday}':
+            - yesterday
+            - mask: '%m-%d-%Y'
+            '{today}':
+            - today
+            - mask: '%m-%d-%Y'
+        ```
+
+    """
+        self._extracted_additional_values = {}
+
+        if hasattr(self, "additional_values_to_extract"):
+            for _field, _elem_id in self.additional_values_to_extract.items():
+                s_elem = soup.find(id=_elem_id)
+
+                if s_elem:
+                    self._extracted_additional_values[_field] = s_elem.get(
+                        "value", s_elem.text
+                    )
+
+    async def _get_report_html(self):
+        self._logger.info("Waiting for data to be displayed in html...")
+        report_payload = self._get_report_payload(
+            btn_id="btnSubmit", btn_value="Get Report"
+        )
+        payload = self._get_payload_with_views(**report_payload)
+        result = await self.aspx_session(self._report_url, method="post", data=payload)
+        self._extract_additional_values(result)
+
+    async def _extract_file_report(self) -> bytes:
+        self._logger.info("Downloading file...")
+        report_payload = self._get_report_payload(
+            btn_id=self.export_to_excel_btn_id,
+            btn_value="Export to Excel",
+        )
+        payload = self._get_payload_with_views(**report_payload)
+
+        self.accept = "application/xml"
+        result = await self.aspx_session(
+            self._report_url,
+            method="post",
+            data=payload,
+            follow_redirects=True,
+        )
+
+        return result
+
+    async def _get_df_from_report_bytes(self, result: bytes) -> pd.DataFrame:
+        if "text/html" in magic.from_buffer(result, mime=True):
+            df = pd.read_html(result, header=0)[0]
+        else:
+            df = pd.read_excel(result, header=0)
+
+        df = df.assign(**self._extracted_additional_values)
+        df.reset_index(drop=True, inplace=True)
+
+        return await self.create_dataframe(df)
+
+    async def run(self):
+        await self._go_to_report()
+        await self._get_report_html()
+        result = await self._extract_file_report()
+        df = await self._get_df_from_report_bytes(result)
+
+        await super().run()
+
+        self._result = df
+        return self._result

flowtask/components/WSDLClient.py

@@ -0,0 +1,123 @@
+import os
+from pathlib import Path
+
+# zeep integration
+from zeep import Client, Settings, helpers
+from zeep.transports import Transport
+from ..exceptions import ComponentError
+from .flow import FlowComponent
+
+
+class WSDLClient(FlowComponent):
+    """
+    WSDLClient.
+
+        Client for WSDL SOAP Web Services using Zeep
+    """
+
+    transport = None
+    transport_options = {"timeout": 10}
+    method = ""
+    settings = None
+    settings_options = {"strict": True, "xml_huge_tree": False}
+    _wsdl = None
+    url: str = ""
+    raw_response = False
+    _filename: str = ""
+    _directory: str = ""
+    saving_xml = False
+
+    async def start(self, **kwargs):
+        """Obtain Pandas Dataframe."""
+        if self.previous:
+            self.data = self.input
+        # defining methods for WSDL Client
+        self.transport = Transport(**self.transport_options)
+        self.settings = Settings(**self.settings_options)
+        if not hasattr(self, "method"):
+            raise ComponentError(
+                "WSDL Error: You need to define a Method using *method* attribute"
+            )
+        if not self.url:
+            raise ComponentError(
+                "WSDL Error: You need to define a WSDL endpoint using the *url* attribute"
+            )
+        # creating a client:
+        self._wsdl = Client(self.url, settings=self.settings)
+        # check if we need to save to an xml file:
+        if hasattr(self, "to_file"):
+            self.saving_xml = True
+            self._directory = Path(self.to_file["directory"])
+            if "filename" in self.to_file:
+                self._filename = self.to_file["filename"]
+            else:
+                # using a pattern:
+                f = self.to_file["file"]
+                file = f["pattern"]
+                if hasattr(self, "masks"):
+                    for mask, replace in self.masks.items():
+                        if isinstance(replace, str):
+                            m = mask.translate({ord("{"): None, ord("}"): None})
+                            if m in self.params.keys():
+                                # using params instead mask value
+                                file = file.replace(mask, self.params[m])
+                            else:
+                                # replace inmediately
+                                file = file.replace(mask, replace)
+                        # else:
+                        #    file = file.replace(mask, convert(replace))
+                self._filename = file
+
+    @property
+    def client(self):
+        return self._wsdl
+
+    async def close(self):
+        """Method."""
+        self._wsdl = None
+
+    def saving_file(self, content):
+        if not self._directory.exists():
+            raise ComponentError(
+                f"Directory for saving XML file doesn't exists: {self._directory}"
+            )
+        path = self._directory.joinpath(self._filename)
+        if path.exists():
+            if "replace" in self.to_file:
+                os.remove(path)
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(content)
+        if self._debug:
+            print(f"Saving XML File on: {path}")
+        # self._result = path
+
+    def queryMethod(self, method, **kwargs):
+        response = None
+        if not method:
+            method = self.method
+        try:
+            fn = getattr(self._wsdl.service, method)
+            response = fn(**kwargs)
+        except Exception as err:
+            raise ComponentError(
+                f"Error Calling method {method} over WSDL client, error: {err}"
+            ) from err
+        finally:
+            return response
+
+    async def run(self):
+        response = None
+        try:
+            with self._wsdl.settings(raw_response=self.raw_response):
+                obj = self.queryMethod(self.method, **self.params)
+                if obj:
+                    if hasattr(self, "serialize"):
+                        response = helpers.serialize_object(obj, dict)
+                    else:
+                        response = obj
+                    self._result = response
+                    return self._result
+                else:
+                    return False
+        except (ComponentError, Exception) as err:
+            raise ComponentError(str(err)) from err

flowtask/components/Wait.py

@@ -0,0 +1,18 @@
+import asyncio
+from .flow import FlowComponent
+
+
+class Wait(FlowComponent):
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        await asyncio.sleep(self.wait)
+        self.add_metric("WAIT", self.wait)
+        self._result = self.data
+        return self._result

flowtask/components/Walmart.py

@@ -0,0 +1,199 @@
+"""
+Scrapping a Web Page Using Selenium + ChromeDriver + BeautifulSoup.
+"""
+import asyncio
+from collections.abc import Callable
+import random
+import json
+from urllib.parse import quote_plus
+import httpx
+import pandas as pd
+import backoff
+from ..interfaces.http import ua
+from .reviewscrap import ReviewScrapper, bad_gateway_exception
+
+
+class Walmart(ReviewScrapper):
+    """Walmart.
+
+    Combining API Key and Web Scrapping, this component will be able to extract
+    Walmart Information (reviews, etc).
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        super(Walmart, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        # Always use proxies:
+        self.use_proxy: bool = True
+        self._free_proxy: bool = False
+        self.cookies = {
+            "_pxhd": "dbf5757b1f867196173eab3a4ab6377bbcd202cdd59b89b4372a1cf3f681b1aa:69fe9bd8-cfbd-11ef-9827-8c8dc170f864",  # noqa
+        }
+        self.headers: dict = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "en-US",
+            "Content-Type": "application/json",
+            "Referer": "https://www.walmart.com/reviews/",
+            "Sec-CH-UA": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
+            "Sec-CH-UA-Mobile": "?0",
+            "Sec-CH-UA-Platform": '"Linux"',
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "Connection": "keep-alive",
+            "User-Agent": random.choice(ua),
+        }
+        self.semaphore = asyncio.Semaphore(10)
+
+    @backoff.on_exception(
+        backoff.expo,
+        (httpx.ConnectTimeout, httpx.HTTPStatusError),
+        max_tries=2,
+        giveup=lambda e: not bad_gateway_exception(e) and not isinstance(e, httpx.ConnectTimeout)
+    )
+    async def _product_reviews(self, idx, row, cookies):
+        async with self.semaphore:
+            # Prepare payload for the API request
+            sku = row['itemId']
+            max_pages = 2   # Maximum number of pages to fetch
+            all_reviews = []
+            total_reviews = 0
+            current_page = 1
+            try:
+                variables = {
+                    "itemId": sku,
+                    "page": 1,  # Start with page 1
+                    "sort": "submission-desc",
+                    "limit": 10,
+                    "filters": [],
+                    "aspect": None,
+                    "filterCriteria": {
+                        "rating": [],
+                        "reviewAttributes": [],
+                        "aspectId": None
+                    }
+                }
+                while True:
+                    self.headers['Referer'] = f"https://www.walmart.com/reviews/product/{sku}?sort=submission-desc&page=1"  # noqa
+                    variables['page'] = current_page
+                    variables_json = json.dumps(variables)  # Proper JSON encoding
+                    variables_encoded = quote_plus(variables_json)
+                    url = f"https://www.walmart.com/orchestra/home/graphql/ReviewsById/{self.api_token}?variables={variables_encoded}"  # noqa
+                    print('URL > ', url)
+                    result = await self.api_get(
+                        url=url,
+                        cookies=cookies,
+                        headers=self.headers
+                    )
+                    if not result:
+                        self._logger.warning(
+                            f"No Product Reviews found for {sku}."
+                        )
+                        break
+                    # Extract the reviews data from the API response
+                    data = result.get('data', {})
+                    reviews_data = data.get('reviews', {})
+                    customer_reviews = reviews_data.get('customerReviews', [])
+                    pagination = reviews_data.get('pagination', {})
+                    # Update total_reviews
+                    total_reviews = data.get('reviews', {}).get('totalReviewCount', 0)
+                    if not customer_reviews:
+                        self._logger.info(f"No more reviews found for itemId {row['itemId']} on page {current_page}.")
+                        break
+
+                    all_reviews.extend(customer_reviews)
+                    if len(all_reviews) >= total_reviews:
+                        self._logger.info(f"Fetched all reviews for itemId {row['itemId']}.")
+                        break
+
+                    current_page += 1
+                    if current_page > max_pages:
+                        self._logger.warning(f"Reached maximum page limit for itemId {row['itemId']}.")
+                        break
+            except (httpx.TimeoutException, httpx.HTTPError) as ex:
+                self._logger.warning(f"Request failed: {ex}")
+                return []
+            except Exception as ex:
+                self._logger.error(f"An error occurred: {ex}")
+                return []
+
+            # Extract the reviews data from the API response
+            reviews = []
+            for item in all_reviews:
+                # Exclude certain keys
+                # Extract relevant fields
+                # Combine with original row data
+                review_data = row.to_dict()
+                review = {
+                    **review_data,
+                    "authorId": item.get("authorId"),
+                    "userNickname": item.get("userNickname"),
+                    "rating": item.get("rating"),
+                    "reviewTitle": item.get("reviewTitle"),
+                    "review": item.get("reviewText"),
+                    "reviewSubmissionTime": item.get("reviewSubmissionTime"),
+                    "clientResponses": item.get("clientResponses"),
+                    "media": item.get("media"),
+                    "itemId": sku,
+                    "productName": row.get("productName"),
+                    "productCategory": row.get("productCategory"),
+                }
+                review['total_reviews'] = total_reviews
+                reviews.append(review)
+            self._logger.info(
+                f"Fetched {len(reviews)} reviews for SKU {sku}."
+            )
+            return reviews
+
+    async def reviews(self):
+        """reviews.
+
+        Target Product Reviews.
+        """
+        httpx_cookies = httpx.Cookies()
+        for key, value in self.cookies.items():
+            httpx_cookies.set(
+                key, value,
+                domain='.walmart.com',
+                path='/'
+            )
+
+        # Iterate over each row in the DataFrame
+        print('starting ...')
+
+        tasks = [
+            self._product_reviews(
+                idx,
+                row,
+                httpx_cookies
+            ) for idx, row in self.data.iterrows()
+        ]
+        # Gather results concurrently
+        all_reviews_nested = await self._processing_tasks(tasks)
+
+        # Flatten the list of lists
+        all_reviews = [review for reviews in all_reviews_nested for review in reviews]
+
+        # Convert to DataFrame
+        reviews_df = pd.DataFrame(all_reviews)
+
+        # show the num of rows in final dataframe:
+        self._logger.notice(
+            f"Ending Product Reviews: {len(reviews_df)}"
+        )
+
+        # Override previous dataframe:
+        self.data = reviews_df
+
+        # return existing data
+        return self.data

flowtask/components/Workplace.py

@@ -0,0 +1,134 @@
+from asyncdb.utils.types import SafeDict
+from .user import UserComponent
+from ..conf import WORKPLACE_ACCESS_TOKEN
+
+
+class Workplace(UserComponent):
+    _messages_url = "https://graph.facebook.com/{thread_id}/messages?access_token={access_token}&limit=10&user={member_id}&fields=id,message,created_time,attachments,from,to,tags"
+    _attachments_url = "https://graph.facebook.com/v18.0/{message_id}?access_token={access_token}&limit=20&user={member_id}&fields=id,created_time,attachments"
+    _members_url = "https://graph.facebook.com/community/members?access_token={access_token}&limit=100&fields=member_id,first_name,last_name,name,email,title,organization,division,department,primary_phone,primary_address,picture,link,locale,updated_time,account_invite_time,account_claim_time,account_deactivate_time,external_id,start_date,about,cost_center,work_locale,frontline,active"
+    _member_threads = "https://graph.facebook.com/{member_id}/conversations/?access_token={access_token}&limit=100&fields=id,name,subject,participants,updated_time,messages"
+    accept = "application/json"
+
+    async def start(self, **kwargs):
+        ## Access Token
+        if hasattr(self, "access_token"):
+            access_token = self.access_token
+        else:
+            access_token = WORKPLACE_ACCESS_TOKEN
+        # Workplaces APIs.
+        if self.type == "messages":
+            self.member_id = self.set_variables(self.member_id)
+            self.thread_id = self.set_variables(self.thread_id)
+            self.messages_url = self._messages_url.format_map(
+                SafeDict(
+                    access_token=access_token,
+                    member_id=self.member_id,
+                    thread_id=self.thread_id,
+                )
+            )
+            self._kwargs = {"url": self.messages_url, "method": "get"}
+        elif self.type == "attachments":
+            self.member_id = self.set_variables(self.member_id)
+            self.message_id = self.set_variables(self.message_id)
+            url = self._attachments_url.format_map(
+                SafeDict(
+                    access_token=access_token,
+                    message_id=self.message_id,
+                    member_id=self.member_id,
+                )
+            )
+            self._kwargs = {"url": url, "method": "get"}
+        elif self.type == "members":
+            self.members_url = self._members_url.format_map(
+                SafeDict(access_token=access_token)
+            )
+            self._kwargs = {"url": self.members_url, "method": "get"}
+        elif self.type == "member_threads":
+            self.member_id = self.set_variables(self.member_id)
+            self.member_threads_url = self._member_threads.format_map(
+                SafeDict(access_token=access_token, member_id=self.member_id)
+            )
+            self._kwargs = {"url": self.member_threads_url, "method": "get"}
+
+    async def run(self):
+        results = []
+        result = await self.session(**self._kwargs)
+        if not result:
+            return False
+        if "data" in result:
+            results += result["data"]
+        elif isinstance(result, dict):
+            if self.type == "attachments":
+                if "attachments" not in result:
+                    return False
+            results += [result]
+        else:
+            results += result
+        if "paging" in result:
+            url = result["paging"]["next"] if "next" in result["paging"] else None
+            while url is not None:
+                self._kwargs["url"] = url
+                resultset = await self.session(**self._kwargs)
+                results += resultset["data"]
+                if "paging" in result:
+                    url = (
+                        resultset["paging"]["next"]
+                        if "next" in resultset["paging"]
+                        else None
+                    )
+                else:
+                    url = None
+        # Iterate over a field to download more results:
+        if hasattr(self, "page_over"):
+            columns = self.page_over
+            for row in results:
+                for column in columns:
+                    try:
+                        rw = row[column]
+                    except KeyError:
+                        # there is no messages in this thread
+                        continue
+                    if "data" in rw:
+                        # replacing "data" with current value of data:
+                        row[column] = row[column]["data"]
+                    if "paging" in row[column]:
+                        url = (
+                            row[column]["paging"]["next"]
+                            if "next" in row[column]["paging"]
+                            else None
+                        )
+                        while url is not None:
+                            resultset = await self.session(url=url, method="get")
+                            row[column] += resultset["data"]
+                            if "paging" in result:
+                                url = (
+                                    resultset["paging"]["next"]
+                                    if "next" in resultset["paging"]
+                                    else None
+                                )
+                            else:
+                                url = None
+        if hasattr(self, "flatten_cols"):
+            columns = self.flatten_cols
+            for row in results:
+                for column in columns:
+                    try:
+                        if "data" in row[column]:
+                            # replacing "data" with current value of data:
+                            row[column] = row[column]["data"]
+                    except KeyError:
+                        # there is no column in this thread
+                        continue
+        # Create a Dataframe from Results:
+        self._result = await self.create_dataframe(results)
+        if self.type == "member_threads":
+            # add the value of member_id over all rows:
+            self._result["member_id"] = self.member_id
+        elif self.type in ("messages"):
+            self._result["member_id"] = self.member_id
+            self._result["thread_id"] = self.thread_id
+        return self._result
+
+    async def close(self):
+        pass