fraudcrawler 0.6.1__tar.gz → 0.6.2__tar.gz

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/PKG-INFO

@@ -1,9 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: fraudcrawler
-Version: 0.6.1
+Version: 0.6.2
 Summary: Intelligent Market Monitoring
-Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT
+License-File: LICENSE
 Author: Domingo Bertus
 Author-email: hello@veanu.ch
 Requires-Python: >=3.11,<4.0
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: beautifulsoup4 (>=4.13.4,<5.0.0)
 Requires-Dist: httpx (>=0.28.1,<0.29.0)
 Requires-Dist: openai (>=1.68.2,<2.0.0)

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/fraudcrawler/base/base.py

@@ -140,6 +140,8 @@ class ProductItem(BaseModel):
     url_resolved: str
     search_engine_name: str
     domain: str
+    exact_search: bool = False
+    exact_search_match: bool = False
 
     # Context parameters
     product_name: str | None = None

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/fraudcrawler/base/orchestrator.py

@@ -3,10 +3,12 @@ import asyncio
 import logging
 from typing import cast, Dict, List, Self
 
-from bs4 import BeautifulSoup
 import httpx
+import re
 
 from fraudcrawler.settings import (
+    EXACT_MATCH_PRODUCT_FIELDS,
+    EXACT_MATCH_FIELD_SEPARATOR,
     PROCESSOR_DEFAULT_MODEL,
 )
 from fraudcrawler.settings import (
@@ -27,8 +29,8 @@ from fraudcrawler import (
     Searcher,
     SearchEngineName,
     Enricher,
-    URLCollector,
     ZyteAPI,
+    URLCollector,
     Processor,
 )
 
@@ -227,44 +229,29 @@ class Orchestrator(ABC):
 
             if not product.filtered:
                 try:
-                    # Fetch the product context from Zyte API
+                    # Fetch and enrich the product context from Zyte API
                     details = await self._zyteapi.details(url=product.url)
-                    url_resolved = self._zyteapi.extract_url_resolved(details=details)
-                    if url_resolved:
-                        product.url_resolved = url_resolved
-                    product.product_name = self._zyteapi.extract_product_name(
-                        details=details
+                    product = self._zyteapi.enrich_context(
+                        product=product, details=details
                     )
 
-                    # If the resolved URL is different from the original URL, we also need to update the domain as
-                    # otherwise the unresolved domain will be shown.
-                    # For example for an unresolved domain "toppreise.ch" but resolved "digitec.ch
-                    if url_resolved and url_resolved != product.url:
-                        logger.debug(
-                            f"URL resolved for {product.url} is {url_resolved}"
-                        )
-                        product.domain = self._searcher._get_domain(url_resolved)
-
-                    product.product_price = self._zyteapi.extract_product_price(
-                        details=details
-                    )
-                    product.product_description = (
-                        self._zyteapi.extract_product_description(details=details)
-                    )
-                    product.product_images = self._zyteapi.extract_image_urls(
-                        details=details
-                    )
-                    product.probability = self._zyteapi.extract_probability(
-                        details=details
-                    )
-                    product.html = self._zyteapi.extract_html(details=details)
-                    if product.html:
-                        soup = BeautifulSoup(product.html, "html.parser")
-                        product.html_clean = soup.get_text(separator=" ", strip=True)
                     # Filter the product based on the probability threshold
                     if not self._zyteapi.keep_product(details=details):
                         product.filtered = True
-                        product.filtered_at_stage = "Zyte probability threshold"
+                        product.filtered_at_stage = (
+                            "Context (Zyte probability threshold)"
+                        )
+
+                    # Check for exact match inside the full product context
+                    product = self._check_exact_search(product=product)
+                    if (
+                        not product.filtered
+                        and product.exact_search
+                        and not product.exact_search_match
+                    ):
+                        product.filtered = True
+                        product.filtered_at_stage = "Context (exact search)"
+
                 except Exception as e:
                     logger.warning(f"Error executing Zyte API search: {e}.")
             await queue_out.put(product)
@@ -502,6 +489,80 @@ class Orchestrator(ABC):
                         **common_kwargs,  # type: ignore[arg-type]
                     )
 
+    @staticmethod
+    def _is_exact_search(search_term: str) -> bool:
+        """Check if the search term is an exact search (contains double quotation marks).
+
+        Args:
+            search_term: The search term to check.
+
+        Returns:
+            True if the search term contains double quotation marks, False otherwise.
+        """
+        return '"' in search_term
+
+    @staticmethod
+    def _extract_exact_search_terms(search_term: str) -> list[str]:
+        """Extract all exact search terms from within double quotation marks.
+
+        Args:
+            search_term: The search term that may contain double quotation marks.
+
+        Returns:
+            A list of extracted search terms without quotes, or empty list if no quotes found.
+        """
+        # Find all double-quoted strings
+        double_quote_matches = re.findall(r'"([^"]*)"', search_term)
+        return double_quote_matches
+
+    @staticmethod
+    def _check_exact_search_terms_match(
+        product: ProductItem,
+        exact_search_terms: list[str],
+    ) -> bool:
+        """Check if the product, represented by a string of selected attributes, matches ALL of the exact search terms.
+
+        Args:
+            product: The product item.
+            exact_search_terms: List of exact search terms to match against.
+        """
+        field_values = [
+            str(val)
+            for fld in EXACT_MATCH_PRODUCT_FIELDS
+            if (val := getattr(product, fld, None)) is not None
+        ]
+        product_str_lower = EXACT_MATCH_FIELD_SEPARATOR.join(field_values).lower()
+
+        return all(
+            re.search(re.escape(est.lower()), product_str_lower)
+            for est in exact_search_terms
+        )
+
+    def _check_exact_search(self, product: ProductItem) -> ProductItem:
+        """Checks if the search term requests an exact search and if yes, checks for conformity."""
+        # Check for exact search and apply regex matching
+        exact_search = self._is_exact_search(product.search_term)
+        product.exact_search = exact_search
+
+        # Only set exact_search_match if this was an exact search (contains quotes)
+        if exact_search:
+            exact_search_terms = self._extract_exact_search_terms(product.search_term)
+            if exact_search_terms:
+                product.exact_search_match = self._check_exact_search_terms_match(
+                    product=product, exact_search_terms=exact_search_terms
+                )
+                logger.debug(
+                    f"Exact search terms {exact_search_terms} matched: {product.exact_search_match} "
+                    f"for offer with url={product.url}"
+                )
+            else:
+                logger.warning(
+                    f"is_exact_search=True but no exact search terms found in search_term='{product.search_term}' "
+                    f"for offer with url={product.url}"
+                )
+        # If exact_search is False, product.exact_search_match remains False (default value)
+        return product
+
     async def run(
         self,
         search_term: str,

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/fraudcrawler/launch_demo_pipeline.py

@@ -97,4 +97,4 @@ def search(search_term: str):
 
 
 if __name__ == "__main__":
-    search(search_term='Gorenje "R619FEW5"')
+    search(search_term="Kaffeebohnen")

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/fraudcrawler/scraping/zyte.py

@@ -2,11 +2,12 @@ from base64 import b64decode
 import logging
 from typing import List
 
+from bs4 import BeautifulSoup
 import httpx
 from tenacity import RetryCallState
 
 from fraudcrawler.settings import ZYTE_DEFALUT_PROBABILITY_THRESHOLD
-from fraudcrawler.base.base import DomainUtils
+from fraudcrawler.base.base import DomainUtils, ProductItem
 from fraudcrawler.base.retry import get_async_retry
 
 logger = logging.getLogger(__name__)
@@ -61,77 +62,8 @@ class ZyteAPI(DomainUtils):
         else:
             logger.debug(f"retry_state is {retry_state}; not logging before_sleep.")
 
-    async def details(self, url: str) -> dict:
-        """Fetches product details for a single URL.
-
-        Args:
-            url: The URL to fetch product details from.
-
-        Returns:
-            A dictionary containing the product details, fields include:
-            (c.f. https://docs.zyte.com/zyte-api/usage/reference.html#operation/extract/response/200/product)
-            {
-                "url": str,
-                "statusCode": str,
-                "product": {
-                    "name": str,
-                    "price": str,
-                    "mainImage": {"url": str},
-                    "images": [{"url": str}],
-                    "description": str,
-                    "metadata": {
-                        "probability": float,
-                    },
-                },
-                "httpResponseBody": base64
-            }
-        """
-        logger.info(f"Fetching product details by Zyte for URL {url}.")
-
-        # Perform the request and retry if necessary. There is some context aware logging:
-        #  - `before`: before the request is made (and before retrying)
-        #  - `before_sleep`: if the request fails before sleeping
-        retry = get_async_retry()
-        retry.before = lambda retry_state: self._log_before(
-            url=url, retry_state=retry_state
-        )
-        retry.before_sleep = lambda retry_state: self._log_before_sleep(
-            url=url, retry_state=retry_state
-        )
-        async for attempt in retry:
-            with attempt:
-                response = await self._http_client.post(
-                    url=self._endpoint,
-                    json={"url": url, **self._config},
-                    auth=(self._api_key, ""),  # API key as username, empty password
-                )
-                response.raise_for_status()
-
-        details = response.json()
-        return details
-
-    @staticmethod
-    def keep_product(
-        details: dict,
-        threshold: float = ZYTE_DEFALUT_PROBABILITY_THRESHOLD,
-    ) -> bool:
-        """Determines whether to keep the product based on the probability threshold.
-
-        Args:
-            details: A product details data dictionary.
-            threshold: The probability threshold used to filter the products.
-        """
-        try:
-            prob = float(details["product"]["metadata"]["probability"])
-        except KeyError:
-            logger.warning(
-                f"Product with url={details.get('url')} has no probability value - product is ignored"
-            )
-            return False
-        return prob > threshold
-
     @staticmethod
-    def extract_product_name(details: dict) -> str | None:
+    def _extract_product_name(details: dict) -> str | None:
         """Extracts the product name from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -144,7 +76,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("name")
 
     @staticmethod
-    def extract_url_resolved(details: dict) -> str | None:
+    def _extract_url_resolved(details: dict) -> str | None:
         """Extracts the resolved URL from the product data - this is automatically resolved by Zyte.
 
         The input argument is a dictionary of the following structure:
@@ -157,7 +89,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("url")
 
     @staticmethod
-    def extract_product_price(details: dict) -> str | None:
+    def _extract_product_price(details: dict) -> str | None:
         """Extracts the product price from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -170,7 +102,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("price")
 
     @staticmethod
-    def extract_product_description(details: dict) -> str | None:
+    def _extract_product_description(details: dict) -> str | None:
         """Extracts the product description from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -183,7 +115,7 @@ class ZyteAPI(DomainUtils):
         return details.get("product", {}).get("description")
 
     @staticmethod
-    def extract_image_urls(details: dict) -> List[str]:
+    def _extract_image_urls(details: dict) -> List[str]:
         """Extracts the images from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -206,7 +138,7 @@ class ZyteAPI(DomainUtils):
         return images
 
     @staticmethod
-    def extract_probability(details: dict) -> float:
+    def _extract_probability(details: dict) -> float:
         """Extracts the probability from the product data.
 
         The input argument is a dictionary of the following structure:
@@ -223,7 +155,7 @@ class ZyteAPI(DomainUtils):
         )
 
     @staticmethod
-    def extract_html(details: dict) -> str | None:
+    def _extract_html(details: dict) -> str | None:
         """Extracts the HTML from the Zyte API response.
 
         The input argument is a dictionary of the following structure:
@@ -243,6 +175,51 @@ class ZyteAPI(DomainUtils):
             return decoded_string
         return None
 
+    def enrich_context(self, product: ProductItem, details: dict) -> ProductItem:
+        product.product_name = self._extract_product_name(details=details)
+
+        url_resolved = self._extract_url_resolved(details=details)
+        if url_resolved:
+            product.url_resolved = url_resolved
+
+        # If the resolved URL is different from the original URL, we also need to update the domain as
+        # otherwise the unresolved domain will be shown.
+        # For example for an unresolved domain "toppreise.ch" but resolved "digitec.ch
+        if url_resolved and url_resolved != product.url:
+            logger.debug(f"URL resolved for {product.url} is {url_resolved}")
+            product.domain = self._get_domain(url=url_resolved)
+
+        product.product_price = self._extract_product_price(details=details)
+        product.product_description = self._extract_product_description(details=details)
+        product.product_images = self._extract_image_urls(details=details)
+        product.probability = self._extract_probability(details=details)
+        product.html = self._extract_html(details=details)
+        if product.html:
+            soup = BeautifulSoup(product.html, "html.parser")
+            product.html_clean = soup.get_text(separator=" ", strip=True)
+
+        return product
+
+    @staticmethod
+    def keep_product(
+        details: dict,
+        threshold: float = ZYTE_DEFALUT_PROBABILITY_THRESHOLD,
+    ) -> bool:
+        """Determines whether to keep the product based on the probability threshold.
+
+        Args:
+            details: A product details data dictionary.
+            threshold: The probability threshold used to filter the products.
+        """
+        try:
+            prob = float(details["product"]["metadata"]["probability"])
+        except KeyError:
+            logger.warning(
+                f"Product with url={details.get('url')} has no probability value - product is ignored"
+            )
+            return False
+        return prob > threshold
+
     async def unblock_url_content(self, url: str) -> bytes:
         """Unblock the content of an URL using Zyte proxy mode.
 
@@ -256,3 +233,52 @@ class ZyteAPI(DomainUtils):
             raise httpx.HTTPError("No httpResponseBody in Zyte response")
 
         return b64decode(details["httpResponseBody"])
+
+    async def details(self, url: str) -> dict:
+        """Fetches product details for a single URL.
+
+        Args:
+            url: The URL to fetch product details from.
+
+        Returns:
+            A dictionary containing the product details, fields include:
+            (c.f. https://docs.zyte.com/zyte-api/usage/reference.html#operation/extract/response/200/product)
+            {
+                "url": str,
+                "statusCode": str,
+                "product": {
+                    "name": str,
+                    "price": str,
+                    "mainImage": {"url": str},
+                    "images": [{"url": str}],
+                    "description": str,
+                    "metadata": {
+                        "probability": float,
+                    },
+                },
+                "httpResponseBody": base64
+            }
+        """
+        logger.info(f"Fetching product details by Zyte for URL {url}.")
+
+        # Perform the request and retry if necessary. There is some context aware logging:
+        #  - `before`: before the request is made (and before retrying)
+        #  - `before_sleep`: if the request fails before sleeping
+        retry = get_async_retry()
+        retry.before = lambda retry_state: self._log_before(
+            url=url, retry_state=retry_state
+        )
+        retry.before_sleep = lambda retry_state: self._log_before_sleep(
+            url=url, retry_state=retry_state
+        )
+        async for attempt in retry:
+            with attempt:
+                response = await self._http_client.post(
+                    url=self._endpoint,
+                    json={"url": url, **self._config},
+                    auth=(self._api_key, ""),  # API key as username, empty password
+                )
+                response.raise_for_status()
+
+        details = response.json()
+        return details

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/fraudcrawler/settings.py

@@ -78,6 +78,14 @@ ENRICHMENT_DEFAULT_LIMIT = 10
 # Zyte settings
 ZYTE_DEFALUT_PROBABILITY_THRESHOLD = 0.1
 
+# Exact match settings
+EXACT_MATCH_PRODUCT_FIELDS = {
+    "url_resolvedproduct_name",
+    "product_description",
+    "html",
+}
+EXACT_MATCH_FIELD_SEPARATOR = "\n"
+
 # Processor settings
 PROCESSOR_DEFAULT_MODEL = "gpt-4o"
 PROCESSOR_DEFAULT_IF_MISSING = -1

{fraudcrawler-0.6.1 → fraudcrawler-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "fraudcrawler"
-version = "0.6.1"
+version = "0.6.2"
 description = "Intelligent Market Monitoring"
 authors = [
     "Domingo Bertus <hello@veanu.ch>",