fraudcrawler 0.5.0__py3-none-any.whl → 0.7.22__py3-none-any.whl

fraudcrawler/scraping/url.py

@@ -3,6 +3,7 @@ from typing import List, Set, Tuple
 from urllib.parse import urlparse, parse_qsl, urlencode, quote, urlunparse, ParseResult
 
 from fraudcrawler.settings import KNOWN_TRACKERS
+from fraudcrawler.base.base import ProductItem
 
 logger = logging.getLogger(__name__)
 
@@ -11,11 +12,19 @@ class URLCollector:
     """A class to collect and de-duplicate URLs."""
 
     def __init__(self):
-        self.collected_currently: Set[str] = set()
-        self.collected_previously: Set[str] = set()
+        self._collected_currently: Set[str] = set()
+        self._collected_previously: Set[str] = set()
+
+    def add_previously_collected_urls(self, urls: List[str]) -> None:
+        """Add a set of previously collected URLs to the internal state.
+
+        Args:
+            urls: A set of URLs that have been collected in previous runs.
+        """
+        self._collected_previously.update(urls)
 
     @staticmethod
-    def remove_tracking_parameters(url: str) -> str:
+    def _remove_tracking_parameters(url: str) -> str:
         """Remove tracking parameters from URLs.
 
         Args:
@@ -55,3 +64,33 @@ class URLCollector:
             fragment=parsed_url.fragment,
         )
         return urlunparse(clean_url)
+
+    async def apply(self, product: ProductItem) -> ProductItem:
+        """Manages the collection and deduplication of ProductItems.
+
+        Args:
+            product: The product item to process.
+        """
+        logger.debug(f'Processing product with  url="{product.url}"')
+
+        # Remove tracking parameters from the URL
+        url = self._remove_tracking_parameters(product.url)
+        product.url = url
+
+        # deduplicate on current run
+        if url in self._collected_currently:
+            product.filtered = True
+            product.filtered_at_stage = "URL collection (current run deduplication)"
+            logger.debug(f"URL {url} already collected in current run")
+
+        # deduplicate on previous runs coming from a db
+        elif url in self._collected_previously:
+            product.filtered = True
+            product.filtered_at_stage = "URL collection (previous run deduplication)"
+            logger.debug(f"URL {url} as already collected in previous run")
+
+        # Add to currently collected URLs
+        else:
+            self._collected_currently.add(url)
+
+        return product

fraudcrawler/scraping/zyte.py

@@ -1,12 +1,13 @@
+from base64 import b64decode
 import logging
 from typing import List
-from base64 import b64decode
 
+from bs4 import BeautifulSoup
 import httpx
 from tenacity import RetryCallState
 
 from fraudcrawler.settings import ZYTE_DEFALUT_PROBABILITY_THRESHOLD
-from fraudcrawler.base.base import DomainUtils
+from fraudcrawler.base.base import DomainUtils, ProductItem
 from fraudcrawler.base.retry import get_async_retry
 
 logger = logging.getLogger(__name__)
@@ -61,77 +62,8 @@ class ZyteAPI(DomainUtils):
         else:
             logger.debug(f"retry_state is {retry_state}; not logging before_sleep.")
 
-    async def details(self, url: str) -> dict:
-        """Fetches product details for a single URL.
-
-        Args:
-            url: The URL to fetch product details from.
-
-        Returns:
-            A dictionary containing the product details, fields include:
-            (c.f. https://docs.zyte.com/zyte-api/usage/reference.html#operation/extract/response/200/product)
-            {
-                "url": str,
-                "statusCode": str,
-                "product": {
-                    "name": str,
-                    "price": str,
-                    "mainImage": {"url": str},
-                    "images": [{"url": str}],
-                    "description": str,
-                    "metadata": {
-                        "probability": float,
-                    },
-                },
-                "httpResponseBody": base64
-            }
-        """
-        logger.info(f"Fetching product details by Zyte for URL {url}.")
-
-        # Perform the request and retry if necessary. There is some context aware logging:
-        #  - `before`: before the request is made (and before retrying)
-        #  - `before_sleep`: if the request fails before sleeping
-        retry = get_async_retry()
-        retry.before = lambda retry_state: self._log_before(
-            url=url, retry_state=retry_state
-        )
-        retry.before_sleep = lambda retry_state: self._log_before_sleep(
-            url=url, retry_state=retry_state
-        )
-        async for attempt in retry:
-            with attempt:
-                response = await self._http_client.post(
-                    url=self._endpoint,
-                    json={"url": url, **self._config},
-                    auth=(self._api_key, ""),  # API key as username, empty password
-                )
-                response.raise_for_status()
-
-        details = response.json()
-        return details
-
-    @staticmethod
-    def keep_product(
-        details: dict,
-        threshold: float = ZYTE_DEFALUT_PROBABILITY_THRESHOLD,
-    ) -> bool:
-        """Determines whether to keep the product based on the probability threshold.
-
-        Args:
-            details: A product details data dictionary.
-            threshold: The probability threshold used to filter the products.
-        """
-        try:
-            prob = float(details["product"]["metadata"]["probability"])
-        except KeyError:
-            logger.warning(
-                f"Product with url={details.get('url')} has no probability value - product is ignored"
-            )
-            return False
-        return prob > threshold
-
     @staticmethod
-    def extract_product_name(details: dict) -> str | None:
+    def _extract_product_name(details: dict) -> str | None:
         """Extracts the product name from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -144,7 +76,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("name")
 
     @staticmethod
-    def extract_url_resolved(details: dict) -> str | None:
+    def _extract_url_resolved(details: dict) -> str | None:
         """Extracts the resolved URL from the product data - this is automatically resolved by Zyte.
 
         The input argument is a dictionary of the following structure:
@@ -157,7 +89,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("url")
 
     @staticmethod
-    def extract_product_price(details: dict) -> str | None:
+    def _extract_product_price(details: dict) -> str | None:
         """Extracts the product price from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -170,7 +102,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("price")
 
     @staticmethod
-    def extract_product_description(details: dict) -> str | None:
+    def _extract_product_description(details: dict) -> str | None:
         """Extracts the product description from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -183,7 +115,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("description")
 
     @staticmethod
-    def extract_image_urls(details: dict) -> List[str]:
+    def _extract_image_urls(details: dict) -> List[str]:
         """Extracts the images from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -206,7 +138,28 @@ class ZyteAPI(DomainUtils):
         return images
 
     @staticmethod
-    def extract_probability(details: dict) -> float:
+    def _extract_gtin(details: dict) -> str | None:
+        """Extracts the GTIN from the product data.
+
+        The input argument is a dictionary of the following structure:
+            {
+                "product": {
+                    "gtin": [{"type": str, "value": str}],
+                }
+            }
+        """
+        product = details.get("product", {})
+        gtin_list = product.get("gtin", [])
+        
+        if gtin_list and len(gtin_list) > 0:
+            # Extract the first GTIN value
+            gtin_value = gtin_list[0].get("value")
+            if gtin_value:
+                return gtin_value
+        return None
+
+    @staticmethod
+    def _extract_probability(details: dict) -> float:
         """Extracts the probability from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -223,7 +176,7 @@ class ZyteAPI(DomainUtils):
         )
 
     @staticmethod
-    def extract_html(details: dict) -> str | None:
+    def _extract_html(details: dict) -> str | None:
         """Extracts the HTML from the Zyte API response.
 
         The input argument is a dictionary of the following structure:
@@ -238,7 +191,120 @@ class ZyteAPI(DomainUtils):
         if isinstance(encoded, str):
             decoded_bytes = b64decode(encoded)
 
-            # Convert bytes to string (assuming UTF-8 encoding)
-            decoded_string = decoded_bytes.decode("utf-8")
+            # Convert bytes to string
+            try:
+                decoded_string = decoded_bytes.decode("utf-8")
+            except UnicodeDecodeError:
+                decoded_string = decoded_bytes.decode("iso-8859-1")
             return decoded_string
         return None
+
+    def enrich_context(self, product: ProductItem, details: dict) -> ProductItem:
+        product.product_name = self._extract_product_name(details=details)
+
+        url_resolved = self._extract_url_resolved(details=details)
+        if url_resolved:
+            product.url_resolved = url_resolved
+
+        # If the resolved URL is different from the original URL, we also need to update the domain as
+        # otherwise the unresolved domain will be shown.
+        # For example for an unresolved domain "toppreise.ch" but resolved "digitec.ch
+        if url_resolved and url_resolved != product.url:
+            logger.debug(f"URL resolved for {product.url} is {url_resolved}")
+            product.domain = self._get_domain(url=url_resolved)
+
+        product.product_price = self._extract_product_price(details=details)
+        product.product_description = self._extract_product_description(details=details)
+        product.product_images = self._extract_image_urls(details=details)
+        product.product_gtin = self._extract_gtin(details=details)
+        product.probability = self._extract_probability(details=details)
+        product.html = self._extract_html(details=details)
+        if product.html:
+            soup = BeautifulSoup(product.html, "html.parser")
+            product.html_clean = soup.get_text(separator=" ", strip=True)
+
+        return product
+
+    @staticmethod
+    def keep_product(
+        details: dict,
+        threshold: float = ZYTE_DEFALUT_PROBABILITY_THRESHOLD,
+    ) -> bool:
+        """Determines whether to keep the product based on the probability threshold.
+
+        Args:
+            details: A product details data dictionary.
+            threshold: The probability threshold used to filter the products.
+        """
+        try:
+            prob = float(details["product"]["metadata"]["probability"])
+        except KeyError:
+            logger.warning(
+                f"Product with url={details.get('url')} has no probability value - product is ignored"
+            )
+            return False
+        return prob > threshold
+
+    async def unblock_url_content(self, url: str) -> bytes:
+        """Unblock the content of an URL using Zyte proxy mode.
+
+        Args:
+            url: The URL to fetch using Zyte proxy mode.
+        """
+        logger.debug(f'Unblock URL content using Zyte proxy for url="{url}"')
+        details = await self.details(url)
+
+        if not details or "httpResponseBody" not in details:
+            raise httpx.HTTPError("No httpResponseBody in Zyte response")
+
+        return b64decode(details["httpResponseBody"])
+
+    async def details(self, url: str) -> dict:
+        """Fetches product details for a single URL.
+
+        Args:
+            url: The URL to fetch product details from.
+
+        Returns:
+            A dictionary containing the product details, fields include:
+            (c.f. https://docs.zyte.com/zyte-api/usage/reference.html#operation/extract/response/200/product)
+            {
+                "url": str,
+                "statusCode": str,
+                "product": {
+                    "name": str,
+                    "price": str,
+                    "mainImage": {"url": str},
+                    "images": [{"url": str}],
+                    "description": str,
+                    "gtin": [{"type": str, "value": str}],
+                    "metadata": {
+                        "probability": float,
+                    },
+                },
+                "httpResponseBody": base64
+            }
+        """
+        logger.info(f"Fetching product details by Zyte for URL {url}.")
+
+        # Perform the request and retry if necessary. There is some context aware logging:
+        #  - `before`: before the request is made (and before retrying)
+        #  - `before_sleep`: if the request fails before sleeping
+        retry = get_async_retry()
+        retry.before = lambda retry_state: self._log_before(
+            url=url, retry_state=retry_state
+        )
+        retry.before_sleep = lambda retry_state: self._log_before_sleep(
+            url=url, retry_state=retry_state
+        )
+        async for attempt in retry:
+            with attempt:
+                response = await self._http_client.post(
+                    url=self._endpoint,
+                    json={"url": url, **self._config},
+                    auth=(self._api_key, ""),  # API key as username, empty password
+                )
+                response.raise_for_status()
+
+        details = response.json()
+        return details

fraudcrawler/settings.py

@@ -14,12 +14,22 @@ RETRY_EXP_BASE = 4
 RETRY_JITTER = 1
 RETRY_SKIP_IF_CODE = [400, 401, 403]  # Skip retrying on these HTTP status codes
 
-# Serp settings
+# Search settings
 GOOGLE_LOCATIONS_FILENAME = ROOT_DIR / "fraudcrawler" / "base" / "google-locations.json"
 GOOGLE_LANGUAGES_FILENAME = ROOT_DIR / "fraudcrawler" / "base" / "google-languages.json"
 SEARCH_DEFAULT_COUNTRY_CODES: List[str] = [
     # ".com",
 ]
+TOPPREISE_SEARCH_PATHS = {
+    "de": "produktsuche",
+    "fr": "chercher",
+    "default": "browse",
+}
+TOPPREISE_COMPARISON_PATHS = [
+    "preisvergleich",
+    "comparison-prix",
+    "price-comparison",
+]
 
 # URL De-duplication settings
 KNOWN_TRACKERS = [
@@ -68,17 +78,19 @@ ENRICHMENT_DEFAULT_LIMIT = 10
 # Zyte settings
 ZYTE_DEFALUT_PROBABILITY_THRESHOLD = 0.1
 
-# Processor settings
-PROCESSOR_DEFAULT_MODEL = "gpt-4o"
-PROCESSOR_DEFAULT_IF_MISSING = -1
-PROCESSOR_EMPTY_TOKEN_COUNT = -1
-PROCESSOR_USER_PROMPT_TEMPLATE = "Product Details:\n{product_details}\n\nRelevance:"
-PROCESSOR_PRODUCT_DETAILS_TEMPLATE = "{field_name}:\n{field_value}"
+# Exact match settings
+EXACT_MATCH_PRODUCT_FIELDS = {
+    "url_resolved",
+    "product_name",
+    "product_description",
+    "html",
+}
+EXACT_MATCH_FIELD_SEPARATOR = "\n"
 
 # Async workers settings
-DEFAULT_N_SERP_WKRS = 10
-DEFAULT_N_ZYTE_WKRS = 10
-DEFAULT_N_PROC_WKRS = 10
+DEFAULT_N_SRCH_WKRS = 2
+DEFAULT_N_CNTX_WKRS = 23
+DEFAULT_N_PROC_WKRS = 5
 
 # HTTPX client settings
 DEFAULT_HTTPX_TIMEOUT = {

fraudcrawler-0.7.22.dist-info/METADATA

@@ -0,0 +1,173 @@
+Metadata-Version: 2.1
+Name: fraudcrawler
+Version: 0.7.22
+Summary: Intelligent Market Monitoring
+Home-page: https://github.com/open-veanu/fraudcrawler
+License: MIT
+Author: Domingo Bertus
+Author-email: hello@veanu.ch
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: beautifulsoup4 (>=4.13.4,<5.0.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: openai (>=1.68.2,<2.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: tenacity (>=9.1.2,<10.0.0)
+Project-URL: Repository, https://github.com/open-veanu/fraudcrawler
+Description-Content-Type: text/markdown
+
+# fraudcrawler
+
+![CI Status](https://github.com/open-veanu/fraudcrawler/workflows/CI/badge.svg)
+![Python Version](https://img.shields.io/badge/python-3.11+-blue.svg)
+![License](https://img.shields.io/badge/license-MIT-green.svg)
+![PyPI](https://img.shields.io/pypi/v/fraudcrawler.svg)
+
+Fraudcrawler is an intelligent **market monitoring** tool that searches the web for products, extracts product details, and classifies them using LLMs. It combines search APIs, web scraping, and AI to automate product discovery and relevance assessment.
+
+## Features
+
+- **Asynchronous pipeline** - Products move through search, extraction, and classification stages independently
+- **Multiple search engines** - Google Search, Google Shopping, and more...
+- **Search term enrichment** - Automatically find related terms and expand your search
+- **Product extraction** - Get structured product data via Zyte API
+- **LLM classification** - Assess product relevance using OpenAI API with custom prompts
+- **Marketplace filtering** - Focus searches on specific domains
+- **Deduplication** - Avoid reprocessing previously collected URLs
+- **CSV export** - Results saved with timestamps for easy tracking
+
+## Prerequisites
+
+- Python 3.11 or higher
+- API keys for:
+  - **SerpAPI** - Google search results
+  - **Zyte API** - Product data extraction
+  - **OpenAI API** - Product classification
+  - **DataForSEO** (optional) - Search term enrichment
+
+## Installation
+
+```bash
+python3.11 -m venv .venv
+source .venv/bin/activate
+pip install fraudcrawler
+```
+
+**Using Poetry:**
+```bash
+poetry install
+```
+
+## Configuration
+
+Create a `.env` file with your API credentials (see `.env.example` for template):
+
+```bash
+SERPAPI_KEY=your_serpapi_key
+ZYTEAPI_KEY=your_zyte_key
+OPENAIAPI_KEY=your_openai_key
+DATAFORSEO_USER=your_user  # optional
+DATAFORSEO_PWD=your_pwd    # optional
+```
+
+## Usage
+
+### Basic Configuration
+For a complete working example, see `fraudcrawler/launch_demo_pipeline.py`. After setting up the necessary parameters you can launch and analyse the results with:
+```python
+# Run pipeline
+await client.run(
+    search_term=search_term,
+    search_engines=search_engines,
+    language=language,
+    location=location,
+    deepness=deepness,
+    excluded_urls=excluded_urls,
+)
+
+# Load results
+df = client.load_results()
+print(df.head())
+```
+
+### Advanced Configuration
+
+**Search term enrichment** - Find and search related terms:
+```python
+from fraudcrawler import Enrichment
+
+deepness.enrichment = Enrichment(
+    additional_terms=5,
+    additional_urls_per_term=10
+)
+```
+
+**Marketplace filtering** - Focus on specific domains:
+```python
+from fraudcrawler import Host
+
+marketplaces = [
+    Host(name="International", domains="zavamed.com,apomeds.com"),
+    Host(name="National", domains="netdoktor.ch,nobelpharma.ch"),
+]
+
+await client.run(..., marketplaces=marketplaces)
+```
+
+**Exclude domains** - Exclude specific domains from your results:
+```python
+excluded_urls = [
+    Host(name="Compendium", domains="compendium.ch"),
+]
+
+await client.run(..., excluded_urls=excluded_urls)
+```
+
+**Skip previously collected URLs**:
+```python
+previously_collected_urls = [
+    "https://example.com/product1",
+    "https://example.com/product2",
+]
+
+await client.run(..., previously_collected_urls=previously_collected_urls)
+```
+
+**View all results** from a client instance:
+```python
+client.print_available_results()
+```
+
+## Output
+
+Results are saved as CSV files in `data/results/` with the naming pattern:
+
+```
+<search_term>_<language_code>_<location_code>_<timestamp>.csv
+```
+
+Example: `sildenafil_de_ch_20250115143022.csv`
+
+The CSV includes product details, URLs, and classification scores from your workflows.
+
+## Development
+
+For detailed contribution guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Architecture
+
+Fraudcrawler uses an asynchronous pipeline where products can be at different processing stages simultaneously. Product A might be in classification while Product B is still being scraped. This is enabled by async workers for each stage (Search, Context Extraction, Processing) using `httpx.AsyncClient`.
+
+![Async Setup](https://github.com/open-veanu/fraudcrawler/raw/master/docs/assets/images/Fraudcrawler_Async_Setup.svg)
+
+For more details on the async design, see the [httpx documentation](https://www.python-httpx.org/api/#asyncclient).
+

fraudcrawler-0.7.22.dist-info/RECORD

@@ -0,0 +1,23 @@
+fraudcrawler/__init__.py,sha256=4Xzhj6aS9zmjs8KZS9nhFg9YAWOpCX-TAtJ4s32A5Jk,1191
+fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fraudcrawler/base/base.py,sha256=QchdTXgrabQKnjP28n_QKjpxHChVpOGQA2cfKFaGLAc,6821
+fraudcrawler/base/client.py,sha256=uPaC1uMpHlMR1ThcTkzsoliwmGuTGH3jyTHLyVNYSHk,5608
+fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
+fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
+fraudcrawler/base/orchestrator.py,sha256=BklS4DNzxbp7yvE2NvBWrDDqnvT4YO7Xh_WXstYNWYA,26050
+fraudcrawler/base/retry.py,sha256=bCDd44XO2-lHO8MGvPblD5152-lHt1dOfMAQSmymLO4,1462
+fraudcrawler/launch_demo_pipeline.py,sha256=oZWodtNzA5mhmLNYMS6lglry88NutvH4IxnEWOUtL8M,6179
+fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fraudcrawler/processing/base.py,sha256=UkoYxFNZ3BQkXmgJnTtruz8-eIFCtWiquRN_IoEXfM4,4091
+fraudcrawler/processing/openai.py,sha256=7sbFg2NPsn627VDzsfIkKantE2KahGmVkSZ1R10OrzQ,19050
+fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fraudcrawler/scraping/enrich.py,sha256=dGWi9p0JStQYSGscCnsQPHNlAeqjoL2rXZnHFNmPhaQ,13158
+fraudcrawler/scraping/search.py,sha256=qHeUpzv1IpRhdFvaycGtL3FLOwT8rOiF0PfiOH6BmUA,34561
+fraudcrawler/scraping/url.py,sha256=unUoZ-bThU99ZlLdDUILdPx1kbtwMWPZVPCDqPscqHw,3217
+fraudcrawler/scraping/zyte.py,sha256=xSHGKo09sX2dgQBrPI7oeoHsVL4qZ8voQLBXRU1XBqM,11102
+fraudcrawler/settings.py,sha256=q3je0r_jd30x2dzlgfm8GyKcigFdgteOLa8HX188bho,3768
+fraudcrawler-0.7.22.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.7.22.dist-info/METADATA,sha256=D749e0ZWDZSn8pjxvHj7RUf5m0D1_qHzRlZPRFqTE9A,5303
+fraudcrawler-0.7.22.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fraudcrawler-0.7.22.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.7.22.dist-info/RECORD,,