flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/CompanyScraper/parsers/rocket.py

@@ -0,0 +1,133 @@
+import re
+from bs4 import BeautifulSoup as bs
+from .base import ScrapperBase
+
+
+class RocketReachScrapper(ScrapperBase):
+    """
+    RocketReachScrapper Model.
+    """
+    domain: str = 'https://rocketreach.co/'
+    search_term: str = "site:rocketreach.co '{}'"
+    keywords: list = [
+        ' Information',
+        ' Information - ',
+        ' Information - RocketReach',
+        ': Contact Details'
+    ]
+
+    def define_search_term(self, term: str):
+        # standardized_term = self._standardize_name(term)
+        standardized_term = term.strip()
+        return self.search_term.format(standardized_term)
+
+    def _extract_codes(self, value):
+        """
+        Extracts NAICS/SIC codes from RocketReach company info.
+        """
+        codes = []
+        for link in value.find_all("a"):  # Iterate over <a> elements
+            match = re.search(r"\b\d+\b", link.text)  # Extract only numbers
+            if match:
+                codes.append(match.group())  # Store only the number part
+        return codes  # Return the list of codes
+
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        """
+        Scrape company information from LeadIQ.
+        Updates the existing row with new data from LeadIQ.
+        """
+        # Start with the existing row data
+        result = row.copy()
+
+        # Actualizamos solo los campos específicos de LeadIQ
+        result.update({
+            'source_platform': 'rocketreach',
+            'scrape_status': 'pending',
+            'search_term': self.search_term_used
+        })
+        try:
+            # Extract `company-header` details
+            company_header = document.select_one(".company-header")
+            if company_header:
+                # Extract company logo
+                img_tag = company_header.select_one(".company-logo")
+                result["logo_url"] = img_tag["src"] if img_tag else None
+
+                # Extract company name
+                title_tag = company_header.select_one(".company-title")
+                if title_tag:
+                    result["company_name"] = title_tag.text.replace(" Information", "").strip()
+
+            # Extract company description from `headline-summary`
+            headline_summary = document.select_one(".headline-summary p")
+            result["company_description"] = headline_summary.text.strip() if headline_summary else None
+
+            # Extract details from the information table
+            info_table = document.select(".headline-summary table tbody tr")
+            for row in info_table:
+                key = row.select_one("td strong")
+                value = row.select_one("td:nth-of-type(2)")
+
+                if key and value:
+                    key_text = key.text.strip().lower()
+                    value_text = value.text.strip()
+
+                    if "website" in key_text:
+                        result["website"] = value.select_one("a")["href"] if value.select_one("a") else value_text
+
+                    elif "ticker" in key_text:
+                        result["stock_symbol"] = value_text
+
+                    elif "revenue" in key_text:
+                        result["revenue_range"] = value_text
+
+                    elif "funding" in key_text:
+                        result["funding"] = value_text
+
+                    elif "employees" in key_text:
+                        result["employee_count"] = value_text.split()[0]
+                        result['number_employees'] = value_text
+
+                    elif "founded" in key_text:
+                        result["founded"] = value_text
+
+                    elif "address" in key_text:
+                        result["headquarters"] = value.select_one("a").text.strip() if value.select_one("a") else value_text
+
+                    elif "phone" in key_text:
+                        result["phone_number"] = value.select_one("a").text.strip() if value.select_one("a") else value_text
+
+                    elif "industry" in key_text:
+                        result["industry"] = [i.strip() for i in value_text.split(",")]
+
+                    elif "keywords" in key_text:
+                        result["keywords"] = [i.strip() for i in value_text.split(",")]
+
+                    elif "sic" in key_text:
+                        result["sic_code"] = self._extract_codes(value)
+
+                    elif "naics" in key_text:
+                        result["naics_code"] = self._extract_codes(value)
+
+            # Validate if any meaningful data was found
+            has_data = any([
+                result.get('company_name'),
+                result.get('logo_url'),
+                result.get('headquarters'),
+                result.get('phone_number'),
+                result.get('website'),
+                result.get('stock_symbol'),
+                result.get('naics_code'),
+                result.get('sic_code'),
+                result.get('employee_count'),
+                result.get('revenue_range'),
+                result.get('company_description')
+            ])
+            result['scrape_status'] = 'success' if has_data else 'no_data'
+            return idx, result
+
+        except Exception as e:
+            self._logger.error(f"Error parsing LeadIQ data: {str(e)}")
+            result['scrape_status'] = f'error: {str(e)[:50]}'
+            return idx, result

flowtask/components/CompanyScraper/parsers/siccode.py

@@ -0,0 +1,109 @@
+import time
+from bs4 import BeautifulSoup as bs
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.common.exceptions import (
+    TimeoutException,
+    NoSuchElementException,
+    WebDriverException
+)
+from .base import ScrapperBase
+
+
+class SicCodeScrapper(ScrapperBase):
+    """
+    SicCodeScrapper Model.
+    """
+    domain: str = 'siccode.com'
+    search_term: str = "site:siccode.com '{}' +NAICS"
+    keywords: list = [
+        ' - ZIP',
+        ' - ZIP '
+    ]
+
+    def define_search_term(self, term: str):
+        cleaned = term.strip().lower()
+        return self.search_term.format(cleaned)
+
+    # async def get(self, url, headers: dict):
+    #     self.use_proxy = True
+    #     self._free_proxy = False
+    #     driver = await self.get_driver()
+    #     try:
+    #         try:
+    #             driver.get(url)
+    #             # WebDriverWait(driver, 2).until(
+    #             #     EC.presence_of_element_located((By.ID, "main"))
+    #             # )
+    #             return bs(driver.page_source, 'html.parser')
+    #         except TimeoutException:
+    #             return None
+    #     finally:
+    #         self.close_driver()
+
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        """
+        Scrapes company information from siccode.com and updates the row.
+        """
+        result = row.copy()
+        result.update({
+            'source_platform': 'siccode',
+            'scrape_status': 'pending',
+            'search_term': self.search_term_used
+        })
+
+        try:
+            header = document.select_one("div.main-title")
+            # Extract company name
+            result["company_name"] = (
+                header.select_one("h1.size-h2 a span") and
+                header.select_one("h1.size-h2 a span").text.strip()
+            )
+            # Extract Industry Category
+            result["industry_category"] = header.select_one("b.p-category").text.strip()
+
+            # Extract SIC and NAICS Codes
+            desc = document.find('div', {'id': 'description'})
+            sic_code_elem = desc.select_one("a.sic")
+            naics_code_elem = desc.select_one("a.naics")
+
+            sic = sic_code_elem.text.split("SIC CODE")[-1].strip() if sic_code_elem else None
+            naics = naics_code_elem.text.split("NAICS CODE")[-1].strip() if naics_code_elem else None
+            result["sic_code"], result["industry"] = sic.split(' - ')
+            result["naics_code"], result["category"] = naics.split(' - ')
+            # Extract Location Details
+            overview = document.find('div', {'id': 'overview'})
+            result['company_description'] = overview.select_one("p.p-note").text.strip()
+
+            result["city"] = overview.select_one(".p-locality") and overview.select_one(".p-locality").text.strip()
+            result["state"] = overview.select_one(".p-region") and overview.select_one(".p-region").text.strip()
+            result["zip_code"] = overview.select_one(".p-postal-code") and overview.select_one(".p-postal-code").text.strip()
+            result["country"] = overview.select_one(".p-country-name") and overview.select_one(".p-country-name").text.strip()
+            result["metro_area"] = overview.select_one("div[title]") and overview.select_one("div[title]").text.strip()
+
+            # Construct Headquarters Address
+            result["headquarters"] = ", ".join(
+                filter(None, [result.get("city"), result.get("state"), result.get("zip_code"), result.get("country")])
+            )
+
+            # Check if we found any meaningful data
+            has_data = any([
+                result.get("company_name"),
+                result.get("category"),
+                result.get("sic_code"),
+                result.get("naics_code"),
+                result.get("headquarters"),
+                result.get("revenue_range"),
+                result.get("years_in_business"),
+                result.get("company_size"),
+            ])
+
+            result['scrape_status'] = 'success' if has_data else 'no_data'
+
+            return idx, result
+
+        except Exception as e:
+            self._logger.error(f"Error parsing SICCode data: {str(e)}")
+            result['scrape_status'] = f'error: {str(e)[:50]}'
+            return idx, result

flowtask/components/CompanyScraper/parsers/visualvisitor.py

@@ -0,0 +1,130 @@
+import re
+from bs4 import BeautifulSoup as bs
+from .base import ScrapperBase
+
+
+class VisualVisitorScrapper(ScrapperBase):
+    """
+    VisualVisitorScrapper Model.
+    """
+    domain: str = 'https://www.visualvisitor.com/'
+    search_term: str = "site:visualvisitor.com '{}'"
+    keywords: list = [
+        ' Phone',
+        ' - Phone',
+    ]
+
+    def define_search_term(self, term: str):
+        standardized_term = term.strip().lower()
+        return self.search_term.format(standardized_term)
+
+    def _extract_codes(self, value):
+        """
+        Extracts NAICS/SIC codes from RocketReach company info.
+        """
+        codes = []
+        for link in value.find_all("a"):  # Iterate over <a> elements
+            match = re.search(r"\b\d+\b", link.text)  # Extract only numbers
+            if match:
+                codes.append(match.group())  # Store only the number part
+        return codes  # Return the list of codes
+
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        """
+        Scrape company information from LeadIQ.
+        Updates the existing row with new data from LeadIQ.
+        """
+        # Start with the existing row data
+        result = row.copy()
+
+        # Actualizamos solo los campos específicos de LeadIQ
+        result.update({
+            'source_platform': 'rocketreach',
+            'scrape_status': 'pending',
+            'search_term': self.search_term_used
+        })
+        try:
+            # Extract `company-header` details
+            company_header = document.select_one(".company-header")
+            if company_header:
+                # Extract company logo
+                img_tag = company_header.select_one(".company-logo")
+                result["logo_url"] = img_tag["src"] if img_tag else None
+
+                # Extract company name
+                title_tag = company_header.select_one(".company-title")
+                if title_tag:
+                    result["company_name"] = title_tag.text.replace(" Information", "").strip()
+
+            # Extract company description from `headline-summary`
+            headline_summary = document.select_one(".headline-summary p")
+            result["company_description"] = headline_summary.text.strip() if headline_summary else None
+
+            # Extract details from the information table
+            info_table = document.select(".headline-summary table tbody tr")
+            for row in info_table:
+                key = row.select_one("td strong")
+                value = row.select_one("td:nth-of-type(2)")
+
+                if key and value:
+                    key_text = key.text.strip().lower()
+                    value_text = value.text.strip()
+
+                    if "website" in key_text:
+                        result["website"] = value.select_one("a")["href"] if value.select_one("a") else value_text
+
+                    elif "ticker" in key_text:
+                        result["stock_symbol"] = value_text
+
+                    elif "revenue" in key_text:
+                        result["revenue_range"] = value_text
+
+                    elif "funding" in key_text:
+                        result["funding"] = value_text
+
+                    elif "employees" in key_text:
+                        result["employee_count"] = value_text.split()[0]
+                        result['number_employees'] = value_text
+
+                    elif "founded" in key_text:
+                        result["founded"] = value_text
+
+                    elif "address" in key_text:
+                        result["headquarters"] = value.select_one("a").text.strip() if value.select_one("a") else value_text
+
+                    elif "phone" in key_text:
+                        result["phone_number"] = value.select_one("a").text.strip() if value.select_one("a") else value_text
+
+                    elif "industry" in key_text:
+                        result["industry"] = [i.strip() for i in value_text.split(",")]
+
+                    elif "keywords" in key_text:
+                        result["keywords"] = [i.strip() for i in value_text.split(",")]
+
+                    elif "sic" in key_text:
+                        result["sic_code"] = self._extract_codes(value)
+
+                    elif "naics" in key_text:
+                        result["naics_code"] = self._extract_codes(value)
+
+            # Validate if any meaningful data was found
+            has_data = any([
+                result.get('company_name'),
+                result.get('logo_url'),
+                result.get('headquarters'),
+                result.get('phone_number'),
+                result.get('website'),
+                result.get('stock_symbol'),
+                result.get('naics_code'),
+                result.get('sic_code'),
+                result.get('employee_count'),
+                result.get('revenue_range'),
+                result.get('company_description')
+            ])
+            result['scrape_status'] = 'success' if has_data else 'no_data'
+            return idx, result
+
+        except Exception as e:
+            self._logger.error(f"Error parsing LeadIQ data: {str(e)}")
+            result['scrape_status'] = f'error: {str(e)[:50]}'
+            return idx, result

flowtask/components/CompanyScraper/parsers/zoominfo.py

@@ -0,0 +1,118 @@
+import time
+from bs4 import BeautifulSoup as bs
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.common.exceptions import (
+    TimeoutException,
+    NoSuchElementException,
+    WebDriverException
+)
+from .base import ScrapperBase
+
+
+class ZoomInfoScrapper(ScrapperBase):
+    """
+    ZoomInfo Model.
+    """
+    domain: str = 'zoominfo.com'
+    search_term: str = 'site:zoominfo.com {} Overview'
+    keywords: list = [
+        ' - Overview, News',
+        'Overview, News'
+    ]
+
+    def define_search_term(self, term: str):
+        cleaned = term.strip().lower()
+        return self.search_term.format(cleaned)
+
+    async def get(self, url, headers: dict):
+        self.use_proxy = True
+        self._free_proxy = False
+        self.use_undetected = True
+        driver = await self.get_driver()
+        try:
+            try:
+                print('URL > ', url)
+                driver.get(url)
+                return driver.page_source
+            except TimeoutException:
+                return None
+        finally:
+            self.close_driver()
+
+    async def scrapping(self, document: bs, idx: int, row: dict):
+        """
+        Scrape company information from Zoominfo.
+        Updates the existing row with new data from Zoominfo.
+        """
+        # Start with the existing row data
+        result = row.copy()
+
+        # Actualizamos solo los campos específicos de Explorium
+        result.update({
+            'source_platform': 'zoominfo',
+            'scrape_status': 'pending',
+            'search_term': self.search_term_used
+        })
+
+        try:
+
+            # Extraer información de la compañía
+            result.update({
+                "company_name": document.select_one("h2#company-description-text-header") and document.select_one("h2#company-description-text-header").text.strip(),
+                "headquarters": document.select_one(".icon-label:-soup-contains('Headquarters') + .content") and document.select_one(".icon-label:-soup-contains('Headquarters') + .content").text.strip(),
+                "phone_number": document.select_one(".icon-label:-soup-contains('Phone Number') + .content") and document.select_one(".icon-label:-soup-contains('Phone Number') + .content").text.strip(),
+                "website": document.select_one(".icon-label:-soup-contains('Website') + a") and document.select_one(".icon-label:-soup-contains('Website') + a")["href"],
+                "revenue_range": document.select_one(".icon-label:-soup-contains('Revenue') + .content") and document.select_one(".icon-label:-soup-contains('Revenue') + .content").text.strip(),
+                "stock_symbol": document.select_one(".icon-label:-soup-contains('Stock Symbol') + .content") and document.select_one(".icon-label:-soup-contains('Stock Symbol') + .content").text.strip(),
+                "industry": [i.text.strip() for i in document.select("#company-chips-wrapper a")],
+                "company_description": document.select_one("#company-description-text-content .company-desc") and document.select_one("#company-description-text-content .company-desc").text.strip(),
+            })  # noqa
+
+            # Extracting NAICS and SIC codes
+            codes_section = document.select("#codes-wrapper .codes-content")
+            result["naics_code"], result["sic_code"] = None, None  # Default to None
+
+            for code in codes_section:
+                text = code.text.strip()
+                if "NAICS Code" in text:
+                    result["naics_code"] = text.replace("NAICS Code", "").strip()
+                elif "SIC Code" in text:
+                    result["sic_code"] = text.replace("SIC Code", "").strip()
+
+            # Extract executives
+            result["executives"] = [
+                {
+                    "name": exec.select_one(".person-name").text.strip(),
+                    "title": exec.select_one(".job-title").text.strip(),
+                    "profile_link": exec.select_one(".person-name")["href"]
+                }
+                for exec in document.select(".org-chart .person-right-content")
+                if exec.select_one(".person-name")
+            ]
+
+            # Verificamos si se encontró algún dato
+            has_data = any([
+                result.get('company_name'),
+                result.get('headquarters'),
+                result.get('country'),
+                result.get('phone_number'),
+                result.get('website'),
+                result.get('stock_symbol'),
+                result.get('naics_code'),
+                result.get('sic_code'),
+                result.get('employee_count'),
+                result.get('revenue_range'),
+                result.get('company_description'),
+            ])
+
+            # Establecemos el estado según si encontramos datos o no
+            result['scrape_status'] = 'success' if has_data else 'no_data'
+
+            # Siempre devolvemos el resultado, tenga datos o no
+            return idx, result
+        except Exception as e:
+            self._logger.error(f"Error parsing Zoominfo data: {str(e)}")
+            result['scrape_status'] = f'error: {str(e)[:50]}'
+            return idx, result