flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/Target.py

@@ -0,0 +1,223 @@
+"""
+Scrapping a Web Page Using Selenium + ChromeDriver + BeautifulSoup.
+"""
+import asyncio
+from collections.abc import Callable
+import random
+import httpx
+import pandas as pd
+import backoff
+# Internals
+from ..exceptions import (
+    ComponentError,
+    ConfigError,
+    NotSupported
+)
+from ..interfaces.http import ua
+from .reviewscrap import ReviewScrapper, bad_gateway_exception
+
+
+class Target(ReviewScrapper):
+    """Target.
+
+    Combining API Key and Web Scrapping, this component will be able to extract
+    TARGET Information (reviews, etc).
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._fn = kwargs.pop('type', None)
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.task_parts: int = kwargs.get('task_parts', 10)
+        self.product_info: bool = kwargs.get('product_info', False)
+        if not self._fn:
+            raise ConfigError(
+                "BestBuy: require a `type` Function to be called, ex: availability"
+            )
+        super(Target, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        # Always use proxies:
+        self.use_proxy: bool = True
+        self._free_proxy: bool = False
+        self.cookies = {
+            "Sid": "WcMH6IPyatK95shr5A-IWjsGf-qeWLo_",
+            "UserLocation": "45510|40.050|-4.210|TO|ES",
+            "__eoi": "ID=9528bd272a92431a:T=1736552136:RT=1736554579:S=AA-AfjZSTuM0txUk7Lsy-pJtCFok",
+            "accessToken": "eyJraWQiOiJlYXMyIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI4M2JkOGI3NC0xYjA2LTRkMjQtOTA1Yi1iNTdjNWMwODg4ZGMiLCJpc3MiOiJNSTYiLCJleHAiOjE3MzY2Mzg1MzYsImlhdCI6MTczNjU1MjEzNiwianRpIjoiVEdULmU3MmZkNmQzMzM1OTQ4NzJhOWNiYTQ4NzY1OTc1NTA1LWwiLCJza3kiOiJlYXMyIiwic3V0IjoiRyIsImRpZCI6ImZjZGEzYmE4MGJmMmU3YmMxY2E1NDgxNzk3MjM1MzVkMDRmOGRlYTE5YzRiZjJmNWQ1NGJmNjQyNjY1NTEzZTciLCJzY28iOiJlY29tLm5vbmUsb3BlbmlkIiwiY2xpIjoiZWNvbS13ZWItMS4wLjAiLCJhc2wiOiJMIn0.DBH4-I6n69roAHnMrp9P1mYWEuGDyvIZR8EeR2nM3SDuVeiLs679E76XXyh4x7A5jnnjFKuaTcaNkEQy40J3eiNbtOkk-897OkqVP6ElCn8NB9ShTFKGvuBdpQy9H-qwieD5DUre_1UE94fIS2-U04WTl1rBs5Glrd1wsd5e4ajLvLH5pVfYyFg1o00b-B-CRn7Q68GzZ1V-MKt_gf-pXZH8nhMq7SCqRCooeSXGiGwuG78OfujIKVHxHgCwBO9nQzvxQ1y4eaDgXc9zzSpgZQ_fGSz9t_Jeuz4UgUbZwyStH9KYHUHYIu6TnqlEIPNuO2NbqJBLOP6wJFezQKN7iA",
+            "adScriptData": "TO",
+            "crl8.fpcuid": "c2aa8017-a2e2-4d24-a61e-2093551bb881",
+            "egsSessionId": "ffbc34de-8a71-4eeb-9490-6382040b0124",
+            "ffsession": "{%22sessionHash%22:%224a74f57be4bb61736552136688%22}",
+            "fiatsCookie": "DSI_1092|DSN_Beechmont%20Area|DSZ_45255",
+            "idToken": "eyJhbGciOiJub25lIn0.eyJzdWIiOiI4M2JkOGI3NC0xYjA2LTRkMjQtOTA1Yi1iNTdjNWMwODg4ZGMiLCJpc3MiOiJNSTYiLCJleHAiOjE3MzY2Mzg1MzYsImlhdCI6MTczNjU1MjEzNiwiYXNzIjoiTCIsInN1dCI6IkciLCJjbGkiOiJlY29tLXdlYi0xLjAuMCIsInBybyI6eyJmbiI6bnVsbCwiZW0iOm51bGwsInBoIjpmYWxzZSwibGVkIjpudWxsLCJsdHkiOmZhbHNlLCJzdCI6IlRPIn19.",
+            "refreshToken": "GQHHnolvR5mW5lY6C9BbK1XbpFSE3F_yFwtKjSUCJOaikGHKg5Ju0eyjhRlm9NmaYegjQuxPnVSv1kh75Y1VOg",
+            "sapphire": "1",
+            "visitorId": "01945292BA770201AEAA83A7E83DE4E9",
+            "kampyleSessionPageCounter": "6",
+            "kampyleUserSession": "1736552137812",
+            "kampyleUserSessionsCount": "4",
+            "mdLogger": "false",
+        }
+        self.headers: dict = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "es-US,es;q=0.9,en-US;q=0.8,en;q=0.7,es-419;q=0.6",
+            "Origin": "https://www.target.com",
+            "Referer": "https://www.target.com/p/",
+            "Sec-CH-UA": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
+            "Sec-CH-UA-Mobile": "?0",
+            "Sec-CH-UA-Platform": '"Linux"',
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+            "User-Agent": random.choice(ua),
+        }
+        self.semaphore = asyncio.Semaphore(10)
+
+    async def close(self, **kwargs) -> bool:
+        self.close_driver()
+        return True
+
+    async def start(self, **kwargs) -> bool:
+        await super(Target, self).start(**kwargs)
+        if self.previous:
+            self.data = self.input
+            if not isinstance(self.data, pd.DataFrame):
+                raise ComponentError(
+                    "Incompatible Pandas Dataframe"
+                )
+        self.api_token = self.get_env_value(self.api_token) if hasattr(self, 'api_token') else self.get_env_value('TARGET_API_KEY')  # noqa
+        if not hasattr(self, self._fn):
+            raise ConfigError(
+                f"BestBuy: Unable to found Function {self._fn} in Target Component."
+            )
+
+    @backoff.on_exception(
+        backoff.expo,
+        (httpx.ConnectTimeout, httpx.HTTPStatusError),
+        max_tries=2,
+        giveup=lambda e: not bad_gateway_exception(e) and not isinstance(e, httpx.ConnectTimeout)
+    )
+    async def _product_reviews(self, idx, row, cookies):
+        async with self.semaphore:
+            # Prepare payload for the API request
+            sku = row['sku']
+            pagesize = 20
+            max_pages = 2   # Maximum number of pages to fetch
+            current_page = 0
+            all_reviews = []
+            total_reviews = 0
+            try:
+                while current_page <= max_pages:
+                    payload = {
+                        "key": "c6b68aaef0eac4df4931aae70500b7056531cb37",  # Ensure this key is valid and authorized
+                        "hasOnlyPhotos": "false",
+                        "includes": "reviews,reviewsWithPhotos,entities,metadata,statistics",
+                        "page": current_page,
+                        "entity": "",
+                        "reviewedId": sku,  # Ensure sku corresponds to 'reviewedId'
+                        "reviewType": "PRODUCT",
+                        "size": pagesize,
+                        "sortBy": "most_recent",
+                        "verifiedOnly": "false",
+                    }
+                    url = "https://r2d2.target.com/ggc/v2/summary"
+                    result = await self.api_get(
+                        url=url,
+                        cookies=cookies,
+                        params=payload,
+                        headers=self.headers
+                    )
+                    total_reviews = result.get('statistics', {}).get('review_count', 0)
+                    if not result:
+                        self._logger.warning(
+                            f"No Product Reviews found for {sku}."
+                        )
+                        break
+                    # Extract the reviews data from the API response
+                    reviews_section = result.get('reviews', {})
+                    items = reviews_section.get('results', [])
+                    if len(items) == 0:
+                        break
+                    all_reviews.extend(items)
+                    # Determine if we've reached the last page
+                    total_pages = reviews_section.get('total_pages', max_pages)
+                    if current_page >= total_pages:
+                        break
+                    current_page += 1  # Move to the next page
+            except (httpx.TimeoutException, httpx.HTTPError) as ex:
+                self._logger.warning(f"Request failed: {ex}")
+                return []
+            except Exception as ex:
+                self._logger.error(f"An error occurred: {ex}")
+                return []
+
+            # Extract the reviews data from the API response
+            reviews = []
+            for item in all_reviews:
+                # Exclude certain keys
+                filtered_item = {
+                    k: v for k, v in item.items()
+                    if k not in ('photos', 'Badges', 'entities', 'tags', 'reviewer_attributes')
+                }
+                # Combine with original row data
+                review_data = row.to_dict()
+                review_data['total_reviews'] = total_reviews
+                review_data.update(filtered_item)
+                reviews.append(review_data)
+            self._logger.info(
+                f"Fetched {len(reviews)} reviews for SKU {sku}."
+            )
+            return reviews
+
+    async def reviews(self):
+        """reviews.
+
+        Target Product Reviews.
+        """
+        httpx_cookies = httpx.Cookies()
+        for key, value in self.cookies.items():
+            httpx_cookies.set(
+                key, value,
+                domain='.target.com',
+                path='/'
+            )
+
+        # Iterate over each row in the DataFrame
+        print('starting ...')
+
+        tasks = [
+            self._product_reviews(
+                idx,
+                row,
+                httpx_cookies
+            ) for idx, row in self.data.iterrows()
+        ]
+        # Gather results concurrently
+        all_reviews_nested = await self._processing_tasks(tasks)
+
+        # Flatten the list of lists
+        all_reviews = [review for reviews in all_reviews_nested for review in reviews]
+
+        # Convert to DataFrame
+        reviews_df = pd.DataFrame(all_reviews)
+
+        # show the num of rows in final dataframe:
+        self._logger.notice(
+            f"Ending Product Reviews: {len(reviews_df)}"
+        )
+
+        # Override previous dataframe:
+        reviews_df.rename(columns={"text": "review", "Rating": "rating"})
+        self.data = reviews_df
+
+        # return existing data
+        return self.data

flowtask/components/ThumbnailGenerator.py

@@ -0,0 +1,156 @@
+from typing import Callable
+import asyncio
+import os
+from pathlib import Path
+import aiofiles
+from io import BytesIO
+import pyheif
+from PIL import Image, UnidentifiedImageError
+import filetype
+import pandas as pd
+from .flow import FlowComponent
+
+
+class ThumbnailGenerator(FlowComponent):
+    """
+    ThumbnailGenerator.
+
+        Overview
+        This component generates thumbnails for images stored in a DataFrame. It takes an image column, resizes the images
+        to a specified size, and saves them in a specified directory with a given filename format. The generated thumbnail
+        paths are added to a new column in the DataFrame.
+        .. table:: Properties
+        :widths: auto
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | Name           | Required | Summary                                                                   |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | data_column    |   Yes    | The name of the column containing the image data.                       |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | thumbnail_column|  Yes    | The name of the column to store the generated thumbnail paths.         |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | size           |   Yes    | The size of the thumbnail. Can be a tuple (width, height) or a single |
+        |                |          | integer for a square thumbnail.                                     |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | format         |   Yes    | The format of the thumbnail (e.g., 'JPEG', 'PNG').                    |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | directory      |   Yes    | The directory where the thumbnails will be saved.                     |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        | filename       |   Yes    | The filename template for the thumbnails. It can include placeholders  |
+        |                | for DataFrame columns (e.g., '{column_name}.jpg').                    |
+        +----------------+----------+-----------+---------------------------------------------------------------+
+        Returns
+        This component returns a DataFrame with a new column containing the paths of the generated thumbnails.
+        Example:
+        ```
+        - ThumbnailGenerator:
+            data_column: image
+            thumbnail_column: thumbnail
+            size: (128, 128)
+            format: JPEG
+            directory: /path/to/thumbnails
+            filename: thumbnail_{id}.jpg
+        ```
+    """  # noqa: E501
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.data_column = kwargs.pop("data_column", None)
+        self.thumbnail_column = kwargs.pop("thumbnail_column", 'thumbnail')
+        if not self.data_column:
+            raise ValueError("data_column must be specified.")
+        self.size = kwargs.pop("size", (128, 128))
+        self.size = self.size if isinstance(self.size, tuple) else (self.size, self.size)
+        self.image_format = kwargs.pop("format", "JPEG").upper()
+        self.directory = kwargs.pop("directory", "./thumbnails")
+        self.filename_template = kwargs.pop("filename", "thumbnail_{id}.jpg")
+        super(ThumbnailGenerator, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self._semaphore = asyncio.Semaphore(10)  # Adjust the limit as needed
+
+    async def start(self, **kwargs) -> bool:
+        if self.previous:
+            self.data = self.input
+        if isinstance(self.directory, str):
+            self.directory = Path(self.directory).resolve()
+        # check if directory exists
+        if self.directory.exists() and not self.directory.is_dir():
+            raise ValueError(f"{self.directory} is not a directory.")
+        if not self.directory.exists():
+            self.directory.mkdir(parents=True, exist_ok=True)
+        return True
+
+    async def close(self):
+        return True
+
+    async def run(self) -> pd.DataFrame:
+        # check for duplicates
+        async def handle(idx):
+            async with self._semaphore:
+                row = self.data.loc[idx].to_dict()
+                file_obj = row[self.data_column]
+                if not file_obj:
+                    self.logger.error(f"Row {idx}: No file object found.")
+                    return None
+                stream = file_obj.getvalue() if isinstance(file_obj, BytesIO) else file_obj
+                # Detect MIME type first
+                kind = filetype.guess(stream)
+                if kind is None:
+                    self.logger.error(
+                        f"Row {idx}: Cannot detect MIME type. Please check the file, skipping"
+                    )
+                    return None
+                self.logger.info(
+                    f"Row {idx}: Detected MIME type: {kind.mime}"
+                )
+                filename = self.filename_template.format(**row)
+                filename = self.mask_replacement(filename)
+                filepath = self.directory.joinpath(filename)
+                try:
+                    if kind == 'image/heic':
+                        try:
+                            i = pyheif.read_heif(stream)
+                            image = Image.frombytes(mode=i.mode, size=i.size, data=i.data)
+                        except Exception as e:
+                            self._logger.error(
+                                f"Unable to parse Apple Heic Photo at row {row}"
+                            )
+                            return None
+                    else:
+                        image = Image.open(BytesIO(stream))
+                    image.thumbnail(self.size)
+                    # Set the thumbnail path in the DataFrame
+                    self.data.at[idx, self.thumbnail_column] = str(filepath)
+                    # check if file exists
+                    if filepath.exists():
+                        return filepath
+                    image.save(filepath, self.image_format)
+                except UnidentifiedImageError:
+                    self._logger.error(
+                        f"Row {idx}: PIL cannot identify image file. MIME: {kind.mime}"
+                    )
+                    bad_folder = self.directory.joinpath('bad_images')
+                    if not bad_folder.exists():
+                        bad_folder.mkdir(parents=True, exist_ok=True)
+                    try:
+                        # Save bad file into disk:
+                        bad_file = bad_folder.joinpath(filename)
+                        async with aiofiles.open(bad_file, "wb") as fp:
+                            await fp.write(stream)
+                    except Exception as e:
+                        self._logger.warning(
+                            f"Unable to save {bad_file} on disk, error: {e}"
+                        )
+                    return
+                except Exception as e:
+                    self._logger.exception(
+                        f"Row {idx}: Unexpected error processing image: {e}"
+                    )
+                    return
+
+        await asyncio.gather(*(handle(i) for i in self.data.index))
+
+        self._result = self.data
+        return self._result

flowtask/components/ToPandas.py

@@ -0,0 +1,67 @@
+from collections.abc import Callable
+import asyncio
+from .flow import FlowComponent
+from ..exceptions import DataNotFound
+from ..interfaces.dataframes import (
+    PandasDataframe,
+    PolarsDataframe,
+    ArrowDataframe,
+    DtDataframe
+)
+
+
+class ToPandas(FlowComponent):
+    """
+    Convert any input into a DataFrame.
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._frame = kwargs.get('frame', 'pandas')
+        self._type = kwargs.get('type', 'dict')
+        self._data_path = kwargs.get('data_path', None)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Initialize the component."""
+        if self.previous:
+            self.data = self.input
+        if self.data is None:
+            raise DataNotFound(
+                f"No data found for component '{self.name}'."
+            )
+        # convert into a List.
+        if isinstance(self.data, dict):
+            if self._data_path:
+                self.data = self.data.get(self._data_path, {})
+            self.data = [self.data]
+        return True
+
+    async def run(self):
+        """Run the component."""
+        if self._frame == 'pandas':
+            self._result = await PandasDataframe().create_dataframe(
+                self.data
+            )
+        elif self._frame == 'polars':
+            self._result = await PolarsDataframe().create_dataframe(
+                self.data
+            )
+        elif self._frame == 'arrow':
+            self._result = await ArrowDataframe().create_dataframe(
+                self.data
+            )
+        elif self._frame == 'dt':
+            self._result = await DtDataframe().create_dataframe(
+                self.data
+            )
+        self._print_data_(self._result)
+        return self._result
+
+    async def close(self):
+        """Close the component."""
+        pass