fraudcrawler 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

fraudcrawler/__init__.py

@@ -1,5 +1,6 @@
 from fraudcrawler.scraping.serp import SerpApi, SearchEngine
 from fraudcrawler.scraping.enrich import Enricher
+from fraudcrawler.scraping.url import URLCollector
 from fraudcrawler.scraping.zyte import ZyteApi
 from fraudcrawler.processing.processor import Processor
 from fraudcrawler.base.orchestrator import Orchestrator
@@ -18,6 +19,7 @@ __all__ = [
     "SerpApi",
     "SearchEngine",
     "Enricher",
+    "URLCollector",
     "ZyteApi",
     "Processor",
     "Orchestrator",

fraudcrawler/base/orchestrator.py

@@ -1,7 +1,8 @@
 from abc import ABC, abstractmethod
 import asyncio
 import logging
-from typing import Dict, List, Set, cast
+from typing import Dict, List, cast
+
 from bs4 import BeautifulSoup
 
 from fraudcrawler.settings import (
@@ -24,7 +25,14 @@ from fraudcrawler.base.base import (
     Prompt,
     ProductItem,
 )
-from fraudcrawler import SerpApi, SearchEngine, Enricher, ZyteApi, Processor
+from fraudcrawler import (
+    SerpApi,
+    SearchEngine,
+    Enricher,
+    URLCollector,
+    ZyteApi,
+    Processor,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -75,15 +83,12 @@ class Orchestrator(ABC):
             n_zyte_wkrs: Number of async workers for zyte (optional).
             n_proc_wkrs: Number of async workers for the processor (optional).
         """
-        # Setup the variables
-        self._collected_urls_current_run: Set[str] = set()
-        self._collected_urls_previous_runs: Set[str] = set()
-
         # Setup the clients
         self._serpapi = SerpApi(
             api_key=serpapi_key, max_retries=max_retries, retry_delay=retry_delay
         )
         self._enricher = Enricher(user=dataforseo_user, pwd=dataforseo_pwd)
+        self._url_collector = URLCollector()
         self._zyteapi = ZyteApi(
             api_key=zyteapi_key, max_retries=max_retries, retry_delay=retry_delay
         )
@@ -156,16 +161,18 @@ class Orchestrator(ABC):
                 break
 
             if not product.filtered:
-                url = product.url
+                # Clean the URL by removing tracking parameters
+                url = self._url_collector.remove_tracking_parameters(product.url)
+                product.url = url
 
-                if url in self._collected_urls_current_run:
+                if url in self._url_collector.collected_currently:
                     # deduplicate on current run
                     product.filtered = True
                     product.filtered_at_stage = (
                         "URL collection (current run deduplication)"
                     )
                     logger.debug(f"URL {url} already collected in current run")
-                elif url in self._collected_urls_previous_runs:
+                elif url in self._url_collector.collected_previously:
                     # deduplicate on previous runs coming from a db
                     product.filtered = True
                     product.filtered_at_stage = (
@@ -173,7 +180,7 @@ class Orchestrator(ABC):
                     )
                     logger.debug(f"URL {url} as already collected in previous run")
                 else:
-                    self._collected_urls_current_run.add(url)
+                    self._url_collector.collected_currently.add(url)
 
             await queue_out.put(product)
             queue_in.task_done()
@@ -480,7 +487,7 @@ class Orchestrator(ABC):
         #        INITIAL SETUP
         # ---------------------------
         if previously_collected_urls:
-            self._collected_urls_previous_runs = set(self._collected_urls_current_run)
+            self._url_collector.collected_previously = set(previously_collected_urls)
 
         # Setup the async framework
         n_terms_max = 1 + (

fraudcrawler/launch_demo_pipeline.py

@@ -13,7 +13,7 @@ def main():
     client = FraudCrawlerClient()
 
     # Setup the search
-    search_term = "Kühlschrank"
+    search_term = "Medion Kühlbox MD 37454"
     language = Language(name="German")
     location = Location(name="Switzerland")
     deepness = Deepness(num_results=10)

fraudcrawler/scraping/serp.py

@@ -4,10 +4,10 @@ import logging
 from pydantic import BaseModel
 from typing import List
 from urllib.parse import urlparse
+import re
 
 from fraudcrawler.settings import MAX_RETRIES, RETRY_DELAY, SERP_DEFAULT_COUNTRY_CODES
 from fraudcrawler.base.base import Host, Language, Location, AsyncClient
-import re
 
 logger = logging.getLogger(__name__)
 

fraudcrawler/scraping/url.py

@@ -0,0 +1,57 @@
+import logging
+from typing import List, Set, Tuple
+from urllib.parse import urlparse, parse_qsl, urlencode, quote, urlunparse, ParseResult
+
+from fraudcrawler.settings import KNOWN_TRACKERS
+
+logger = logging.getLogger(__name__)
+
+
+class URLCollector:
+    """A class to collect and de-duplicate URLs."""
+
+    def __init__(self):
+        self.collected_currently: Set[str] = set()
+        self.collected_previously: Set[str] = set()
+
+    @staticmethod
+    def remove_tracking_parameters(url: str) -> str:
+        """Remove tracking parameters from URLs.
+
+        Args:
+            url: The URL to clean.
+
+        Returns:
+            The cleaned URL without tracking parameters.
+        """
+        logging.debug(f"Removing tracking parameters from URL: {url}")
+
+        # Parse the url
+        parsed_url = urlparse(url)
+
+        # Parse query parameters
+        queries: List[Tuple[str, str]] = parse_qsl(
+            parsed_url.query, keep_blank_values=True
+        )
+        remove_all = url.startswith(
+            "https://www.ebay"
+        )  # eBay URLs have all query parameters as tracking parameters
+        if remove_all:
+            filtered_queries = []
+        else:
+            filtered_queries = [
+                q
+                for q in queries
+                if not any(q[0].startswith(tracker) for tracker in KNOWN_TRACKERS)
+            ]
+
+        # Rebuild the URL without tracking parameters
+        clean_url = ParseResult(
+            scheme=parsed_url.scheme,
+            netloc=parsed_url.netloc,
+            path=parsed_url.path,
+            params=parsed_url.params,
+            query=urlencode(filtered_queries, quote_via=quote),
+            fragment=parsed_url.fragment,
+        )
+        return urlunparse(clean_url)

fraudcrawler/settings.py

@@ -13,6 +13,18 @@ SERP_DEFAULT_COUNTRY_CODES: List[str] = [
     # ".com",
 ]
 
+# URL De-duplication settings
+KNOWN_TRACKERS = [
+    "srsltid",
+    "utm_source",
+    "utm_medium",
+    "utm_campaign",
+    "utm_term",
+    "utm_content",
+    "ar",
+    "ps",
+]
+
 # Enrichment settings
 ENRICHMENT_DEFAULT_LIMIT = 10
 

{fraudcrawler-0.4.2.dist-info → fraudcrawler-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.4.2
+Version: 0.4.3
 Summary: Intelligent Market Monitoring
 Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT

{fraudcrawler-0.4.2.dist-info → fraudcrawler-0.4.3.dist-info}/RECORD

@@ -1,20 +1,21 @@
-fraudcrawler/__init__.py,sha256=yXFdQzlSLUZV4Oh0wkzghvPlICQO5TnpEtIHZaTay_c,717
+fraudcrawler/__init__.py,sha256=zAqnJ9Mewq0qzSfOjyaICyqDRQZE_Z3FmyF2IPdOhXo,788
 fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/base/base.py,sha256=JWjZ3mpX4caQAsWKYqtHrUqHfHr6GXlAaEjxxHV9ODQ,6020
 fraudcrawler/base/client.py,sha256=FibiYycjUys-c4sv66Y2JqJu5y15be2MYd2_9yB3wG8,4936
 fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
 fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
-fraudcrawler/base/orchestrator.py,sha256=p1gRtj3jVaFmtwPSKruiOixu3QDuSiHjPKFi0KKsgPk,24591
-fraudcrawler/launch_demo_pipeline.py,sha256=zQxKAekJ56iKQ5-NeM0UMS-1Wd3ui0bpeqkH1nM9A4A,4628
+fraudcrawler/base/orchestrator.py,sha256=xOMxA0zPUXSF8AGY5AUqzsOO9LfRIjxI2HuZf__Z_sI,24689
+fraudcrawler/launch_demo_pipeline.py,sha256=CX4A-E63ER7Ip9RNI_IyTAXerYXcQ-NoSvhvLDLdP-s,4640
 fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/processing/processor.py,sha256=An2orst0YRIav7bFuoDMgjwWz2Z9dyjVUbkNAMXNTTo,3748
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=X1BBZshdZqPmbduzhGwH0ULSzq03L_7bf7_UL8yOQ9E,10608
-fraudcrawler/scraping/serp.py,sha256=ix2kCs9Xo694K8vjDL104MDb2Cun1AXfStxMaR-2u7U,17865
+fraudcrawler/scraping/serp.py,sha256=divEp1UBUsws24PWZABhWIxOmaLqLwdeGn4KNrqWkYA,17865
+fraudcrawler/scraping/url.py,sha256=5Z3hPW73E-TLhM-Zha8OTcUOumc_rcx64R0fT9z2Hi8,1748
 fraudcrawler/scraping/zyte.py,sha256=DUF5pIwpZyQw30qURnFxtp8KYpUgBkrXjM7RaVGH92Q,7005
-fraudcrawler/settings.py,sha256=z63Lc8LnmfG7u0F7CVlGOXMMpr7LtJC0BzXDoA8rN7Q,839
-fraudcrawler-0.4.2.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.4.2.dist-info/METADATA,sha256=M1xMdweLHpSbfEceT_5GpcDiLdDHpOHpzQ5w-ZNF4gQ,5931
-fraudcrawler-0.4.2.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fraudcrawler-0.4.2.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.4.2.dist-info/RECORD,,
+fraudcrawler/settings.py,sha256=31jvRFfB-gsVbeidLLl4iQgrFL7GH-824lerIniPI08,1017
+fraudcrawler-0.4.3.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.4.3.dist-info/METADATA,sha256=jlk2WdtXEK0-s6QRQdI96EBpQiyHWKgJiYeW93yiU24,5931
+fraudcrawler-0.4.3.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fraudcrawler-0.4.3.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.4.3.dist-info/RECORD,,