flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/CSVToGCS.py

@@ -0,0 +1,120 @@
+import asyncio
+from pathlib import Path
+from typing import Callable, Tuple
+from asyncdb import AsyncDB
+from querysource.datasources.drivers.bigquery import bigquery_default
+from .flow import FlowComponent
+from ..exceptions import ComponentError
+
+class CSVToGCS(FlowComponent):
+    """
+    CSVToGCS.
+
+    Este componente sube un archivo CSV desde el sistema local a un bucket específico de Google Cloud Storage (GCS).
+    Opcionalmente, puede crear el bucket si no existe.
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.csv_path: Path = Path(kwargs.pop('csv_path'))
+        self.bucket_uri: str = kwargs.pop('bucket_uri', None)  # Puede ser proporcionado directamente o generado
+        self.object_name: str = kwargs.pop('object_name', self.csv_path.name)
+        self.overwrite: bool = kwargs.pop('overwrite', False)
+        self.create_bucket: bool = kwargs.pop('create_bucket', False)
+        self.storage_class: str = kwargs.pop('storage_class', 'STANDARD')
+        self.location: str = kwargs.pop('location', 'US')
+        self.delete_local: bool = kwargs.pop('delete_local', False)
+        self.bq = None  # Instancia de AsyncDB
+        self.bucket_name: str = kwargs.pop('bucket_name', None)  # Necesario si bucket_uri no se proporciona
+        super(CSVToGCS, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Inicializa el componente configurando la conexión AsyncDB."""
+        # Validar parámetros requeridos
+        if not self.csv_path.exists():
+            raise ComponentError(f"CSVToGCS: El archivo CSV '{self.csv_path}' no existe.")
+
+        if not bigquery_default:
+            raise ComponentError("CSVToGCS: 'bigquery_default' no está configurado correctamente.")
+
+        # Obtener credenciales y parámetros del driver
+        credentials = bigquery_default.get_credentials()
+
+        # Inicializar AsyncDB con el driver de BigQuery
+        try:
+            self.bq = AsyncDB("bigquery", params=credentials)
+            self._logger.info("CSVToGCS: Instancia de AsyncDB creada exitosamente.")
+        except Exception as e:
+            raise ComponentError(f"CSVToGCS: Error al inicializar AsyncDB: {e}") from e
+
+    async def run(self) -> Tuple[str, str]:
+        """Ejecuta la carga del archivo CSV a GCS y retorna bucket_uri y object_uri."""
+        if not self.bq:
+            raise ComponentError("CSVToGCS: AsyncDB no está inicializado. Asegúrate de ejecutar 'start' antes de 'run'.")
+
+        try:
+            async with await self.bq.connection() as conn:
+                # Obtener bucket_uri y bucket_name del componente anterior si no se proporcionan
+                if not self.bucket_uri:
+                    self.bucket_uri = self.getTaskVar('bucket_uri')
+                if not self.bucket_name:
+                    self.bucket_name = self.getTaskVar('bucket_name')
+
+                if not self.bucket_uri:
+                    if not self.bucket_name:
+                        raise ComponentError("CSVToGCS: 'bucket_uri' o 'bucket_name' deben ser proporcionados.")
+                    self.bucket_uri = f"gs://{self.bucket_name}"
+
+                # Verificar si el bucket existe
+                bucket_exists = await conn.bucket_exists(self.bucket_name)
+                if not bucket_exists:
+                    if self.create_bucket:
+                        await conn.create_bucket(
+                            bucket_name=self.bucket_name,
+                            location=self.location,
+                            storage_class=self.storage_class
+                        )
+                        self._logger.info(f"CSVToGCS: Bucket '{self.bucket_name}' creado exitosamente en la región '{self.location}' con clase de almacenamiento '{self.storage_class}'.")
+                    else:
+                        raise ComponentError(f"CSVToGCS: El bucket '{self.bucket_name}' no existe y 'create_bucket' está establecido en False.")
+                else:
+                    self._logger.info(f"CSVToGCS: Bucket '{self.bucket_name}' ya existe.")
+
+                # Subir el archivo CSV a GCS
+                object_uri, message = await conn.create_gcs_from_csv(
+                    bucket_uri=self.bucket_uri,
+                    object_name=self.object_name,
+                    csv_data=self.csv_path,
+                    overwrite=self.overwrite
+                )
+                self._logger.info(f"CSVToGCS: {message}")
+
+                # Guardar bucket_uri y object_uri para el siguiente componente
+                self.setTaskVar('bucket_uri', self.bucket_uri)
+                self.setTaskVar('object_uri', object_uri)
+
+                # Opcionalmente eliminar el archivo local
+                if self.delete_local and object_uri:
+                    self.csv_path.unlink()
+                    self._logger.info(f"CSVToGCS: Archivo local '{self.csv_path}' eliminado exitosamente después de la carga.")
+
+                return self.bucket_uri, object_uri
+
+        except ComponentError as ce:
+            raise ce  # Re-lanzar errores específicos de componentes
+        except Exception as e:
+            raise ComponentError(f"CSVToGCS: Error durante la carga a GCS: {e}") from e
+
+    async def close(self):
+        """Cierra la conexión AsyncDB."""
+        try:
+            if self.bq:
+                await self.bq.close()
+                self._logger.info("CSVToGCS: AsyncDB cerrado exitosamente.")
+        except Exception as e:
+            self._logger.error(f"CSVToGCS: Error al cerrar AsyncDB: {e}")

flowtask/components/CompanyScraper/__init__.py

@@ -0,0 +1 @@
+from .scrapper import CompanyScraper

flowtask/components/CompanyScraper/parsers/__init__.py

@@ -0,0 +1,6 @@
+from .leadiq import LeadiqScrapper
+from .explorium import ExploriumScrapper
+from .zoominfo import ZoomInfoScrapper
+from .siccode import SicCodeScrapper
+from .rocket import RocketReachScrapper
+from .visualvisitor import VisualVisitorScrapper

flowtask/components/CompanyScraper/parsers/base.py

@@ -0,0 +1,102 @@
+from typing import Any, List, Dict
+from bs4 import BeautifulSoup as bs
+from abc import abstractmethod
+from ....interfaces import SeleniumService, HTTPService
+import re
+import logging
+
+class ScrapperBase(SeleniumService, HTTPService):
+    """
+    ScrapperBase Model.
+
+
+    Define how scrappers should be work.-
+    """
+    domain: str
+    search_term: str
+    cookies: Any
+    keywords: List[str]
+
+    def __init__(self, *args, **kwargs):
+        self.cookies = kwargs.get('cookies', None)
+        self._logger = logging.getLogger(self.__class__.__name__)
+        self._counter: int = 0
+        self.search_term_used: str = ''
+        super().__init__(*args, **kwargs)
+
+    @abstractmethod
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        pass
+
+    @abstractmethod
+    def define_search_term(self, term: str):
+        pass
+
+    async def get(self, url, headers: dict):
+        return await self._get(url, headers=headers, use_proxy=True)
+
+    def _parse_address(self, address: str) -> Dict[str, str]:
+        """
+        Parse address string to extract state, zipcode and country.
+
+        Args:
+            address (str): Raw address string
+
+        Returns:
+            Dict with parsed address components:
+            {
+                'address': str,
+                'state': str,
+                'zipcode': str,
+                'country': str
+            }
+        """
+        if not address:
+            return {
+                'address': None,
+                'state': None,
+                'zipcode': None,
+                'country': None
+            }
+
+        # Mantener la dirección original
+        result = {'address': address}
+
+        # Primera regex para formato completo
+        pattern1 = r'^.*,\s+([^,]+?)\s+([\w\s-]+)\s+([A-Z]{2})$'
+        # Segunda regex como fallback
+        pattern2 = r'^.*,\s*([^,]+?),\s+([\w\s-]+?)\s*([A-Z]{2})'
+
+        try:
+            # Intentar con la primera regex
+            match = re.search(pattern1, address)
+            if not match:
+                # Si no hay match, intentar con la segunda
+                match = re.search(pattern2, address)
+
+            if match:
+                result['state'] = match.group(1).strip()
+                result['zipcode'] = match.group(2).strip()
+                result['country'] = match.group(3).strip()
+            else:
+                self._logger.warning(f"Could not parse address: {address}")
+                result.update({
+                    'state': None,
+                    'zipcode': None,
+                    'country': None
+                })
+        except Exception as e:
+            self._logger.error(f"Error parsing address {address}: {str(e)}")
+            result.update({
+                'state': None,
+                'zipcode': None,
+                'country': None
+            })
+
+        return result
+
+    def _standardize_name(self, text: str) -> str:
+        """Estandariza el formato del texto: lowercase y guiones en lugar de espacios."""
+        # Primero limpiamos caracteres especiales y espacios extras
+        cleaned = text.strip().lower().replace(' ', '-')
+        return f"\'{cleaned}\'"

flowtask/components/CompanyScraper/parsers/explorium.py

@@ -0,0 +1,192 @@
+from bs4 import BeautifulSoup as bs
+from .base import ScrapperBase
+import json
+
+
+class ExploriumScrapper(ScrapperBase):
+    """
+    ExploriumScrapper Model.
+    """
+    domain: str = 'explorium.ai'
+    search_term: str = 'site:explorium.ai {}'
+    keywords: list = [
+        'overview - services',
+    ]
+
+    def define_search_term(self, term: str):
+        cleaned = term.strip().lower()
+        return self.search_term.format(cleaned)
+
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        """
+        Scrape company information from Explorium.
+        Updates the existing row with new data from Explorium.
+        """
+        # Start with the existing row data
+        result = row.copy()
+
+        # Actualizamos solo los campos específicos de Explorium
+        result.update({
+            'source_platform': 'explorium',
+            'scrape_status': 'pending',
+            'search_term': self.search_term_used
+        })
+
+        try:
+            # Extraer información de la compañía
+            company_info = document.find('div', {'class': 'company-info'})
+            if company_info:
+                # Nombre de la compañía
+                company_name = company_info.find('h1', {'class': 'company-name'})
+                if company_name:
+                    result['company_name'] = company_name.text.strip()
+
+                # Dirección
+                address = company_info.find('div', {'class': 'address'})
+                if address:
+                    address_info = self._parse_address(address.text.strip())
+                    result.update(address_info)
+
+                # Otros detalles de la compañía
+                details = company_info.find_all('div', {'class': 'detail-item'})
+                for detail in details:
+                    label = detail.find('span', {'class': 'label'})
+                    value = detail.find('span', {'class': 'value'})
+                    if label and value:
+                        field = label.text.strip().lower()
+                        val = value.text.strip()
+
+                        if 'phone' in field:
+                            result['phone_number'] = val
+                        elif 'website' in field:
+                            result['website'] = val
+                        elif 'employees' in field:
+                            result['employee_count'] = val
+                        elif 'revenue' in field:
+                            result['revenue_range'] = val
+                        elif 'naics' in field:
+                            result['naics_code'] = val
+                        elif 'sic' in field:
+                            result['sic_code'] = val
+
+            # 🔍 Extract NAICS & SIC codes and industry descriptions
+            result.update(self._extract_naics_sic(document))
+
+            # Extract company logo, headquarters, country, and description
+            result.update(self._extract_company_info(document))
+
+            # Verificamos si se encontró algún dato
+            has_data = any([
+                result.get('company_name'),
+                result.get('headquarters'),
+                result.get('country'),
+                result.get('phone_number'),
+                result.get('website'),
+                result.get('stock_symbol'),
+                result.get('naics_code'),
+                result.get('sic_code'),
+                result.get('employee_count'),
+                result.get('revenue_range'),
+                result.get('company_description'),
+                result.get('logo_url')
+            ])
+
+            # Establecemos el estado según si encontramos datos o no
+            result['scrape_status'] = 'success' if has_data else 'no_data'
+
+            # Siempre devolvemos el resultado, tenga datos o no
+            return idx, result
+
+        except Exception as e:
+            self._logger.error(f"Error parsing Explorium data: {str(e)}")
+            result['scrape_status'] = f'error: {str(e)[:50]}'
+            return idx, result
+
+    def _extract_naics_sic(self, document: bs):
+        """
+        Extract NAICS & SIC codes along with their industry descriptions.
+
+        Returns:
+            dict: A dictionary containing 'naics_code', 'sic_code', and 'industry' (comma-separated).
+        """
+        result = {
+            'naics_code': None,
+            'sic_code': None,
+            'industry': None
+        }
+
+        naics_codes = []
+        sic_codes = []
+        industries = []
+
+        # Extract NAICS section
+        naics_section = document.find('div', {'data-id': 'company-stat-naics'})
+        if naics_section:
+            naics_entries = naics_section.find_all('p', {'class': 'ExpTypography-root'})
+            for entry in naics_entries:
+                code = entry.text.strip().strip(',')
+                industry_desc = entry.get('aria-label', '').strip()
+                if code:
+                    naics_codes.append(code)
+                if industry_desc:
+                    industries.append(industry_desc)
+
+        # Extract SIC section
+        sic_section = document.find('div', {'data-id': 'company-stat-sic'})
+        if sic_section:
+            sic_entries = sic_section.find_all('p', {'class': 'ExpTypography-root'})
+            for entry in sic_entries:
+                code = entry.text.strip().strip(',')
+                industry_desc = entry.get('aria-label', '').strip()
+                if code:
+                    sic_codes.append(code)
+                if industry_desc:
+                    industries.append(industry_desc)
+
+        # Convert lists to comma-separated strings
+        if naics_codes:
+            result['naics_code'] = ', '.join(naics_codes)
+        if sic_codes:
+            result['sic_code'] = ', '.join(sic_codes)
+        if industries:
+            result['industry'] = ', '.join(industries)
+
+        return result
+
+    def _extract_company_info(self, document: bs):
+        """
+        Extract headquarters, country, company description, and logo.
+        """
+        result = {
+            'headquarters': None,
+            'country': None,
+            'company_description': None,
+            'logo_url': None
+        }
+
+        # Extract headquarters address
+        address_section = document.find('div', {'data-id': 'info-address'})
+        if address_section:
+            address_element = address_section.find('p', {'aria-label': True})
+            if address_element:
+                address_text = address_element.get('aria-label', '').strip()
+                result['headquarters'] = address_text
+
+                # Extract country (last word in the address)
+                country = address_text.split(',')[-1].strip()
+                result['country'] = country if country else None
+
+        # Extract company description
+        name_element = document.find('h1', {'data-id': 'txt-company-name'})
+        description_element = document.find('p', {'class': 'ExpTypography-root ExpTypography-body1'})
+        if name_element and description_element:
+            company_name = name_element.text.strip()
+            company_desc = description_element.text.strip()
+            result['company_description'] = f"{company_name}: {company_desc}"
+
+        # Extract company logo
+        logo_element = document.find('img', {'alt': True, 'src': True})
+        if logo_element:
+            result['logo_url'] = logo_element['src']
+
+        return result

flowtask/components/CompanyScraper/parsers/leadiq.py

@@ -0,0 +1,206 @@
+from bs4 import BeautifulSoup as bs
+from .base import ScrapperBase
+import json
+
+
+class LeadiqScrapper(ScrapperBase):
+    """
+    LeadiqScrapper Model.
+    """
+    domain: str = 'leadiq.com'
+    search_term: str = "site:leadiq.com {}"
+    keywords: list = [
+        'Email Formats & Email Address',
+        'Company Overview',
+        'Employee Directory',
+        'Contact Details & Competitors',
+        'Email Format'
+    ]
+
+    def define_search_term(self, term: str):
+        standardized_term = self._standardize_name(term)
+        search_term = self.search_term.format(standardized_term)
+        return search_term
+
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        """
+        Scrape company information from LeadIQ.
+        Updates the existing row with new data from LeadIQ.
+        """
+        # Start with the existing row data
+        result = row.copy()
+
+        # Actualizamos solo los campos específicos de LeadIQ
+        result.update({
+            'source_platform': 'leadiq',
+            'scrape_status': 'pending',
+            'search_term': self.search_term_used
+        })
+
+        try:
+            # Get company name and logo URL from logo image
+            logo = document.find('img', {'alt': True, 'width': '76.747'})
+            if logo:
+                result['company_name'] = logo.get('alt')
+                result['logo_url'] = logo.get('src')
+
+            # Get company revenue range from highlight-right section
+            highlight_right = document.find('div', {'class': 'highlight-right'})
+            if highlight_right:
+                revenue_span = highlight_right.find('span', {'class': 'start'})
+                if revenue_span:
+                    start_value = revenue_span.text.strip()
+                    end_span = revenue_span.find_next_sibling('span', {'class': 'end'})
+                    if end_span:
+                        end_value = end_span.text.strip()
+                        result['revenue_range'] = f"{start_value} - {end_value}"
+                    else:
+                        result['revenue_range'] = start_value
+
+            # First find the highlight-left section that contains company info
+            highlight_left = document.find('div', {'class': 'highlight-left'})
+            if not highlight_left:
+                self._logger.warning("Could not find highlight-left section")
+                return idx, result
+
+            # Then find the card span within highlight-left
+            overview_section = highlight_left.find('div', {'class': 'card span'})
+            if not overview_section:
+                return idx, result
+
+            # Extract information from dl/dt/dd elements
+            dl_element = overview_section.find('dl')
+            if dl_element:
+                for item in dl_element.find_all('div', {'class': 'item'}):
+                    dt = item.find('dt')
+                    dd = item.find('dd')
+                    if dt and dd:
+                        field = dt.text.strip().lower()
+                        value = dd.text.strip()
+
+                        # Map fields to our column names
+                        if field == 'headquarters':
+                            address_info = self._parse_address(value)
+                            result.update(address_info)
+                            # Extract country from headquarters
+                            parts = value.split()
+                            result['country'] = parts[-1] if len(parts) > 1 else None
+                        elif field == 'phone number':
+                            phone = value.replace('****', '0000')
+                            result['phone_number'] = phone
+                        elif field == 'website':
+                            website = dd.find('a')
+                            result['website'] = website['href'] if website else value
+                        elif field == 'stock symbol':
+                            result['stock_symbol'] = value
+                        elif field == 'naics code':
+                            result['naics_code'] = value
+                        elif field == 'employees':
+                            result['employee_count'] = value
+                        elif field == 'sic code':
+                            result['sic_code'] = value
+
+            # Extract information from the hero section
+            hero_section = document.find('div', {'class': 'card hero snug'})
+            if hero_section:
+                # Company name
+                company_name_element = hero_section.find('h1')
+                if company_name_element:
+                    result['company_name'] = company_name_element.text.strip()
+
+                # Industry, location, and number of employees
+                info_p = hero_section.find('p', {'class': 'info'})
+                if info_p:
+                    spans = info_p.find_all('span')
+                    if len(spans) >= 3:
+                        result['industry'] = spans[0].text.strip()
+                        result['location'] = spans[1].text.strip()
+                        result['number_employees'] = spans[2].text.strip()
+
+                # Company description
+                description_p = hero_section.find('pre')
+                if description_p:
+                    result['company_description'] = description_p.text.strip()
+
+            # Extract similar companies
+            similar_companies = []
+            similar_section = document.find('div', {'id': 'similar'})
+            if similar_section:
+                for company in similar_section.find_all('li'):
+                    company_link = company.find('a')
+                    if not company_link:
+                        continue
+
+                    company_logo = company_link.find('img')
+                    company_name = company_link.find('h3')
+
+                    # Find revenue span
+                    revenue_spans = company_link.find_all('span')
+                    revenue_span = None
+                    for span in revenue_spans:
+                        if span.find('span', {'class': 'start'}):
+                            revenue_span = span
+                            break
+
+                    if company_name:
+                        similar_company = {
+                            'name': company_name.text.strip(),  # No escapamos las comillas
+                            'leadiq_url': company_link['href'],
+                            'logo_url': company_logo['src'] if company_logo else None,
+                        }
+
+                        # Extract revenue range
+                        if revenue_span:
+                            start = revenue_span.find('span', {'class': 'start'})
+                            end = revenue_span.find('span', {'class': 'end'})
+
+                            if start:
+                                start_value = start.text.strip()
+                                if end:
+                                    end_value = end.text.strip()
+                                    similar_company['revenue_range'] = f"{start_value} - {end_value}"
+                                else:
+                                    similar_company['revenue_range'] = start_value
+
+                        similar_companies.append(similar_company)
+
+            if similar_companies:
+                try:
+                    result['similar_companies'] = json.dumps(
+                        similar_companies,
+                        ensure_ascii=False,
+                        allow_nan=False,
+                        separators=(',', ':')
+                    )
+                except Exception as e:
+                    self._logger.error(
+                        f"Error formatting similar companies JSON: {str(e)}"
+                    )
+                    result['similar_companies'] = None
+
+            # Actualizamos el contador y el estado
+            self._counter += 1
+
+            # Verificamos si se encontró algún dato
+            has_data = any([
+                result.get('company_name'),
+                result.get('logo_url'),
+                result.get('address'),
+                result.get('phone_number'),
+                result.get('website'),
+                result.get('stock_symbol'),
+                result.get('naics_code'),
+                result.get('employee_count'),
+                result.get('revenue_range'),
+                result.get('similar_companies')
+            ])
+
+            # Establecemos el estado según si encontramos datos o no
+            result['scrape_status'] = 'success' if has_data else 'no_data'
+            # Siempre devolvemos el resultado, tenga datos o no
+            return idx, result
+
+        except Exception as e:
+            self._logger.error(f"Error parsing LeadIQ data: {str(e)}")
+            result['scrape_status'] = f'error: {str(e)[:50]}'
+            return idx, result