flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/ProductCompetitors/parsers/bestbuy.py

@@ -0,0 +1,86 @@
+from .base import ProductCompetitorsBase
+
+
+class BestBuyScrapper(ProductCompetitorsBase):
+    domain: str = 'bestbuy.com'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.headless = False
+        self.use_wire = False
+
+    async def product_information(self, response: object, idx: int, row: dict) -> tuple:
+        """Get the product information from BestBuy."""
+        try:
+            document = self.get_bs(response)
+            competitors_found = {}
+
+            # Inicializar valores vacíos para todos los competidores
+            for competitor in self.competitors:
+                self.set_empty_values(row, competitor)
+
+            # Find all product cards in the carousel
+            carousel = document.find('ul', {'class': 'c-carousel-list'})
+            if carousel:
+                for product in carousel.find_all('li', {'class': 'product-carousel-v2_brix-item'}):
+                    try:
+                        # Extract product info
+                        product_link = product.find('a', {'data-testid': 'product-link'})
+                        if not product_link:
+                            continue
+
+                        # Get product name and brand
+                        title_div = product_link.find('div', {'class': 'title-block__title'})
+                        if not title_div:
+                            continue
+                        
+                        full_name = title_div.text.strip()
+                        brand = full_name.split(' - ')[0] if ' - ' in full_name else ''
+
+                        # Check if this brand is in our competitors list and we haven't found it yet
+                        if brand not in self.competitors or brand in competitors_found:
+                            continue
+
+                        # Get product URL and SKU
+                        url = f"https://www.bestbuy.com{product_link.get('href')}"
+                        sku = product_link.get('data-cy', '').replace('product-link-', '')
+
+                        # Get price
+                        price_div = product.find('div', {'class': 'priceView-hero-price'})
+                        price = price_div.text.strip() if price_div else None
+
+                        # Get rating and reviews
+                        rating_p = product.find('p', {'class': 'visually-hidden'})
+                        rating = None
+                        num_reviews = None
+                        if rating_p:
+                            rating_text = rating_p.text
+                            try:
+                                rating = rating_text.split('rating,')[1].split('out')[0].strip() if 'rating,' in rating_text else None
+                                num_reviews = rating_text.split('with')[1].split('reviews')[0].strip() if 'with' in rating_text else None
+                            except (IndexError, AttributeError):
+                                pass
+
+                        # Store competitor info
+                        row.update({
+                            f'competitor_brand_{brand}': brand,
+                            f'competitor_name_{brand}': full_name,
+                            f'competitor_url_{brand}': url,
+                            f'competitor_sku_{brand}': sku,
+                            f'competitor_price_{brand}': price,
+                            f'competitor_rating_{brand}': rating,
+                            f'competitor_reviews_{brand}': num_reviews
+                        })
+                        competitors_found[brand] = True
+
+                        # Si ya encontramos todos los competidores, podemos salir
+                        if len(competitors_found) == len(self.competitors):
+                            break
+                    except Exception as err:
+                        self._logger.warning(f'Error processing product in carousel: {err}')
+                        continue
+
+            return idx, row
+        except Exception as err:
+            self._logger.error(f'Error getting product information from BestBuy: {err}')
+            return idx, row 

flowtask/components/ProductCompetitors/parsers/lowes.py

@@ -0,0 +1,103 @@
+from .base import ProductCompetitorsBase
+
+
+class LowesScrapper(ProductCompetitorsBase):
+    domain: str = 'lowes.com'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.expected_columns = [
+            'price',
+            'availability',
+            'product_name',
+            'product_description',
+            'product_id'
+        ]
+        self.headless = False
+        self.use_wire = False
+
+    async def connect(self):
+        """Creates the Driver and Connects to the Site."""
+        self._driver = await self.get_driver()
+        await self.start()
+
+    async def disconnect(self):
+        """Disconnects the Driver and closes the Connection."""
+        if self._driver:
+            self.close_driver()
+
+    async def product_information(self, response: object, idx: int, row: dict) -> tuple:
+        """Get the product information from Lowes."""
+        try:
+            document = self.get_bs(response)
+            competitors_found = {}
+
+            # Inicializar valores vacíos para todos los competidores
+            for competitor in self.competitors:
+                self.set_empty_values(row, competitor)
+
+            # Find all product cards in the carousel
+            carousel = document.find('div', {'class': 'carousel-inner-container'})
+            if carousel:
+                for product in carousel.find_all('div', {'class': 'product-card-wrapper'}):
+                    try:
+                        # Extract product info
+                        product_link = product.find('a', {'class': 'carousel-container'})
+                        if not product_link:
+                            continue
+
+                        # Get product name and brand
+                        title_div = product_link.find('span', {'class': 'brand-name'})
+                        if not title_div:
+                            continue
+                        
+                        brand = title_div.text.strip()
+                        full_name = title_div.find_next('span', {'class': 'product-desc'}).text.strip()
+
+                        # Check if this brand is in our competitors list and we haven't found it yet
+                        if brand not in self.competitors or brand in competitors_found:
+                            continue
+
+                        # Get product URL and SKU
+                        url = f"https://www.lowes.com{product_link.get('href')}"
+                        sku = product_link.get('data-productid', '')
+
+                        # Get price
+                        price_div = product.find('span', {'class': 'final-price'})
+                        price = price_div.text.strip() if price_div else None
+
+                        # Get rating and reviews
+                        rating_div = product.find('span', {'class': 'rating'})
+                        rating = None
+                        num_reviews = None
+                        if rating_div:
+                            try:
+                                rating = rating_div.get('aria-label', '').split(' ')[0]
+                                reviews_span = product.find('span', {'class': 'rating-count'})
+                                num_reviews = reviews_span.text.strip() if reviews_span else None
+                            except (IndexError, AttributeError):
+                                pass
+
+                        # Store competitor info
+                        row.update({
+                            f'competitor_brand_{brand}': brand,
+                            f'competitor_name_{brand}': full_name,
+                            f'competitor_url_{brand}': url,
+                            f'competitor_sku_{brand}': sku,
+                            f'competitor_price_{brand}': price,
+                            f'competitor_rating_{brand}': rating,
+                            f'competitor_reviews_{brand}': num_reviews
+                        })
+                        competitors_found[brand] = True
+
+                        # Si ya encontramos todos los competidores, podemos salir
+                        if len(competitors_found) == len(self.competitors):
+                            break
+                    except Exception as err:
+                        self._logger.warning(f'Error processing product in carousel: {err}')
+                        continue
+
+            return idx, row
+        except Exception as err:
+            self._logger.error(f'Error getting product information from Lowes: {err}')
+            return idx, row 

flowtask/components/ProductCompetitors/scrapper.py

@@ -0,0 +1,155 @@
+from collections.abc import Callable, List
+import asyncio
+import random
+import pandas as pd
+from tqdm.asyncio import tqdm
+from ...exceptions import ComponentError, ConfigError
+from ...interfaces import HTTPService, SeleniumService
+from ...interfaces.http import ua
+from ..flow import FlowComponent
+from .parsers import (
+    BestBuyScrapper,
+    LowesScrapper,
+)
+
+
+class ProductCompetitors(FlowComponent, SeleniumService, HTTPService):
+    """
+    Product Competitors Scraper Component
+
+    Overview:
+    Pluggable component for scraping product information from competitors (BestBuy and Lowes).
+
+    Properties:
+    - url_column (str): Name of the column containing URLs to scrape (default: 'url')
+    - account_name_column (str): Name of the column containing retailer name (default: 'account_name')
+    - product_id_column (str): Name of the column containing product IDs (default: 'product_id')
+    - competitors (list): List of competitor brands to search for (e.g. ['Insignia', 'TCL', 'LG', 'Sony', 'Samsung'])
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        self.info_column: str = kwargs.get('url_column', 'url')
+        self.account_column: str = kwargs.get('account_name_column', 'account_name')
+        self.product_id_column: str = kwargs.get('product_id_column', 'product_id')
+        self.competitors: List[str] = kwargs.get('competitors', ['Insignia', 'TCL', 'LG', 'Sony', 'Samsung'])
+        self.competitors_bucket = {comp: [] for comp in self.competitors}
+        self.scrapper_class: Callable = None
+        self._scrapper_func: str = 'product_information'
+        self.use_proxy = True
+        self._free_proxy = False
+        self.paid_proxy = True
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.concurrently: bool = kwargs.get('concurrently', False)
+        self.task_parts: int = kwargs.get('task_parts', 10)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.headers: dict = {
+            "Accept": self.accept,
+            "TE": "trailers",
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": random.choice(ua),
+            **kwargs.get('headers', {})
+        }
+
+    def _get_scrapper(self, account_name: str):
+        """Get the appropriate scrapper based on account name."""
+        try:
+            if account_name == "Best Buy":
+                return BestBuyScrapper()
+            elif account_name == "Lowe's":
+                return LowesScrapper()
+            else:
+                return None
+        except Exception as err:
+            self._logger.error(f"Error while getting scrapper: {err}")
+            raise ConfigError(f"Error while getting scrapper: {err}")
+
+    async def start(self, **kwargs) -> bool:
+        """Initialize the component and validate required parameters."""
+        if self.previous:
+            self.data = self.input
+
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError("Input must be a DataFrame", status=404)
+
+        required_columns = [self.info_column, self.account_column, self.product_id_column]
+        for col in required_columns:
+            if col not in self.data.columns:
+                raise ConfigError(f"Column {col} not found in DataFrame")
+
+        return True
+
+    async def _start_scrapping(self, idx, row):
+        """Handle scraping for a single row."""
+        try:
+            async with self._semaphore:
+                url = row[self.info_column]
+                account_name = row[self.account_column]
+                
+                self._logger.debug(f"Scraping URL: {url} for {account_name}")
+                
+                scrapper = self._get_scrapper(account_name)
+                if not scrapper:
+                    self._logger.error(f"No scrapper found for {account_name}")
+                    return idx, row
+
+                async with scrapper as s:
+                    response = await s.get(url, headers=self.headers)
+                    if response:
+                        try:
+                            idx, row = await s.product_information(response, idx, row)
+                        except Exception as err:
+                            self._logger.error(f"Scraping error: {err}")
+                            return idx, row
+                return idx, row
+        except Exception as err:
+            self._logger.error(f"Error while scraping: {err}")
+            raise ComponentError(f"Error while scraping: {err}")
+
+    async def run(self):
+        """Execute scraping for all URLs in the DataFrame."""
+        tasks = [
+            self._start_scrapping(idx, row)
+            for idx, row in self.data.iterrows()
+        ]
+        
+        results = []
+        total_tasks = len(tasks)
+        
+        with tqdm(total=total_tasks, desc="Scraping Progress", unit="task") as pbar:
+            if not self.concurrently:
+                for task in tasks:
+                    try:
+                        idx, row = await task
+                        results.append((idx, row))
+                        await asyncio.sleep(random.uniform(0.25, 1.5))
+                    except Exception as e:
+                        self._logger.error(f"Task error: {str(e)}")
+                    finally:
+                        pbar.update(1)
+            else:
+                for chunk in self.split_parts(tasks, self.task_parts):
+                    chunk_results = await asyncio.gather(*chunk, return_exceptions=True)
+                    results.extend(chunk_results)
+                    pbar.update(len(chunk))
+
+        if not results:
+            return pd.DataFrame()
+
+        indices, data_dicts = zip(*results)
+        df = pd.DataFrame(data_dicts, index=indices)
+        self._result = df
+        self._print_data_(self._result, 'Competitor Scraping Results')
+        return self._result
+
+    async def close(self):
+        """Clean up resources."""
+        return True 

flowtask/components/ProductCompliant.py

@@ -0,0 +1,169 @@
+from collections.abc import Callable
+import asyncio
+from typing import Any
+# Bot Infraestructure:
+from parrot.bots.basic import BasicBot
+from .flow import FlowComponent
+from ..exceptions import ComponentError, ConfigError
+from ..conf import TASK_STORAGES
+
+class ProductCompliant(FlowComponent):
+    """
+        ProductCompliant
+
+        Overview
+
+            The ProductCompliant class is a component for interacting with an IA Agent for making Customer Satisfaction Analysis.
+            It extends the FlowComponent class.
+
+        .. table:: Properties
+        :widths: auto
+
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | Name             | Required | Description                                                                                      |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | output_column    |   Yes    | Column for saving the Customer Satisfaction information.                                         |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+        Return
+
+            A Pandas Dataframe with the Customer Satisfaction statistics.
+
+    """ # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        super().__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+        # System Prompt:
+        self.system_prompt = "Product Compliant: "
+        self._bot_name = kwargs.get('bot_name', 'CompliantBot')
+        # TaskStorage
+        # Find in the taskstorage, the "prompts" directory.
+        prompt_path = self._taskstore.path.joinpath(self._program, 'prompts')
+        if not prompt_path.exists():
+            raise ConfigError(
+                f"{self.system_prompt} Prompts Path Not Found: {prompt_path}"
+            )
+        self.prompt_path = prompt_path
+        # is hardcoded to this particular Bot.
+        self.system_prompt_file = 'compliantbot.txt'
+        # Bot Object:
+        self._bot: Any = None
+
+    async def start(self, **kwargs):
+        """
+            start
+
+            Overview
+
+                The start method is a method for starting the ProductCompliant component.
+
+            Parameters
+
+                kwargs: dict
+                    A dictionary containing the parameters for the ProductCompliant component.
+
+            Return
+
+                True if the ProductCompliant component started successfully.
+
+        """
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError(
+                "CompliantBot: Data Was Not Found"
+            )
+        if not self.output_column:
+            raise ConfigError(
+                "ProductCompliant: output_column is required"
+            )
+        # check if Prompt File exists
+        prompt_file = self.prompt_path.joinpath(self.system_prompt_file)
+        if not prompt_file.exists():
+            raise ConfigError(
+                f"{self.system_prompt} Prompt File Not Found: {prompt_file}"
+            )
+        self.system_prompt_file = prompt_file.name
+        # read the prompt file as text:
+        with open(prompt_file, 'r') as f:
+            self.system_prompt = f.read()
+        # Set the Bot:
+        try:
+            self._bot = BasicBot(
+                name=self._bot_name,
+                system_prompt=self.system_prompt,
+                goal="Your task is to provide a concise and insightful analysis on negative reviews of products",
+                use_llm=self.llm.get('name', 'name'),
+                model_name=self.llm.get('model_name', 'gemini-2.0-pro'),
+            )
+            # configure the bot:
+            await self._bot.configure()
+        except Exception as err:
+            raise ComponentError(
+                f"{self.system_prompt} Error Configuring Bot: {err}"
+            ) from err
+        return True
+
+    def format_question(self, product_name, reviews):
+        question = f"""
+            Product: {product_name}
+
+            Question:
+            "What are the primary customer concerns, problems, and issues based on these negative product reviews for {product_name}?"
+
+            Negative Customer Reviews:
+
+        """
+        for review in reviews:
+            question += f"* {review}\n"
+        return question
+
+    async def run(self):
+        """
+            run
+
+            Overview
+
+                The run method is a method for running the ProductCompliant component.
+
+            Return
+
+                A Pandas Dataframe with the Product Compliant statistics.
+
+        """
+        # Group reviews by product_name and aggregate them into a list
+        grouped = self.data.groupby(self.product_column)[self.review_column].apply(list).reset_index()
+        products_evaluation = {}
+        for _, row in grouped.iterrows():
+            product_name = row[self.product_column]
+            reviews = row[self.review_column]
+            formatted_question = self.format_question(product_name, reviews)
+            result = await self._bot.question(
+                question=formatted_question,
+                return_docs=False
+            )
+            products_evaluation[product_name] = {
+                "answer": result.answer
+            }
+        # Then, create a dataframe only with the columns in "self.columns" grouped.
+        grouped_df = self.data.groupby(self.columns).agg(
+            num_reviews=(self.review_column, "count"),
+            avg_rating=("rating", "mean")
+        ).reset_index()
+        # Add the Product Compliant column, using the dictionary and match per product_name column
+        grouped_df[self.output_column] = grouped_df[self.product_column].map(
+            lambda x: products_evaluation[x]['answer']
+        )
+        # return the grouped dataframe
+        self._result = grouped_df
+        return self._result
+
+    async def close(self):
+        pass

flowtask/components/ProductInfo/__init__.py

@@ -0,0 +1 @@
+from .scraper import ProductInfo 

flowtask/components/ProductInfo/parsers/__init__.py

@@ -0,0 +1,5 @@
+from .epson import EpsonParser
+from .hp import HPParser
+from .canon import CanonParser
+from .brother import BrotherParser
+from .samsung import SamsungParser 

flowtask/components/ProductInfo/parsers/base.py

@@ -0,0 +1,83 @@
+from abc import abstractmethod
+import re
+import logging
+from typing import Dict, Any, Optional, List
+from bs4 import BeautifulSoup
+from ....interfaces import HTTPService, SeleniumService
+
+class ParserBase(HTTPService, SeleniumService):
+    """
+    Base class for product information parsers.
+    
+    Defines the interface and common functionality for all product parsers.
+    """
+    domain: str
+    search_format: str
+    model_pattern: Optional[str] = None  # Hacemos que sea opcional con valor predeterminado None
+    
+    def __init__(self, *args, **kwargs):
+        self.cookies = kwargs.get('cookies', None)
+        self._logger = logging.getLogger(self.__class__.__name__)
+        super().__init__(*args, **kwargs)
+    
+    @abstractmethod
+    async def parse(self, url: str, search_term: str) -> Dict[str, Any]:
+        """
+        Parse product information from a URL.
+        
+        Args:
+            url: URL to parse
+            search_term: Original search term
+            
+        Returns:
+            Dictionary with extracted product information
+        """
+        pass
+    
+    def create_search_query(self, term: str) -> str:
+        """
+        Create a search query for the given term.
+        
+        Args:
+            term: Search term (typically product model)
+            
+        Returns:
+            Formatted search query
+        """
+        return self.search_format.format(term)
+    
+    def extract_model_code(self, url: str) -> Optional[str]:
+        """
+        Extract model code from URL using the regex pattern if defined.
+        
+        Args:
+            url: URL to extract model code from
+            
+        Returns:
+            Extracted model code or None if not found or pattern not defined
+        """
+        if not hasattr(self, 'model_pattern') or self.model_pattern is None:
+            return None  # Si no hay patrón definido, devolvemos None
+            
+        match = re.search(self.model_pattern, url)
+        if match and match.group(1):
+            return match.group(1)
+        return None
+    
+    def get_product_urls(self, search_results: List[Dict[str, str]], max_urls: int = 5) -> List[str]:
+        """
+        Extract relevant product URLs from search results.
+        
+        Args:
+            search_results: List of search result dictionaries
+            max_urls: Maximum number of URLs to return
+            
+        Returns:
+            List of product URLs
+        """
+        urls = []
+        for result in search_results[:max_urls]:
+            url = result.get('link') or result.get('href') or result.get('url')
+            if url and self.domain in url:
+                urls.append(url)
+        return urls