fraudcrawler 0.3.8__tar.gz → 0.3.10__tar.gz

{fraudcrawler-0.3.8 → fraudcrawler-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.3.8
+Version: 0.3.10
 Summary: Intelligent Market Monitoring
 Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT

{fraudcrawler-0.3.8 → fraudcrawler-0.3.10}/fraudcrawler/base/base.py

@@ -6,6 +6,7 @@ from pydantic import (
     model_validator,
 )
 from pydantic_settings import BaseSettings
+import re
 from typing import List
 
 import aiohttp
@@ -48,11 +49,17 @@ class Host(BaseModel):
     name: str
     domains: str | List[str]
 
+    @staticmethod
+    def _normalize_domain(domain: str) -> str:
+        """Make it lowercase and strip 'www.' and 'https?://' prefixes from the domain."""
+        domain = domain.strip().lower()
+        return re.sub(r"^(https?://)?(www\.)?", "", domain)
+
     @field_validator("domains", mode="before")
-    def split_domains_if_str(cls, val):
+    def normalize_domains(cls, val):
         if isinstance(val, str):
             val = val.split(",")
-        return [dom.strip().lower() for dom in val]
+        return [cls._normalize_domain(dom.strip()) for dom in val]
 
 
 class Location(BaseModel):

{fraudcrawler-0.3.8 → fraudcrawler-0.3.10}/fraudcrawler/scraping/serp.py

@@ -4,7 +4,7 @@ from pydantic import BaseModel
 from typing import List
 from urllib.parse import urlparse
 
-from fraudcrawler.settings import MAX_RETRIES, RETRY_DELAY
+from fraudcrawler.settings import MAX_RETRIES, RETRY_DELAY, SERP_DEFAULT_COUNTRY_CODES
 from fraudcrawler.base.base import Host, Language, Location, AsyncClient
 import re
 
@@ -148,17 +148,32 @@ class SerpApi(AsyncClient):
         return urls
 
     @staticmethod
-    def _has_included_country_code(url: str, country_code: str) -> bool:
-        """Determines whether to keep the url based on the country_code.
+    def _relevant_country_code(url: str, country_code: str) -> bool:
+        """Determines whether the url shows relevant country codes.
 
         Args:
             url: The URL to investigate.
             country_code: The country code used to filter the products.
         """
-        return f".{country_code}" in url.lower() or ".com" in url.lower()
+        url = url.lower()
+        country_code_relevance = f".{country_code}" in url
+        default_relevance = any(cc in url for cc in SERP_DEFAULT_COUNTRY_CODES)
+        return country_code_relevance or default_relevance
 
     @staticmethod
-    def _domain_is_present(domain: str, hosts: List[Host]) -> bool:
+    def _domain_in_host(domain: str, host: Host) -> bool:
+        """Checks if the domain is present in the host.
+
+        Args:
+            domain: The domain to check.
+            host: The host to check against.
+        """
+        return any(
+            domain == hst_dom or domain.endswith(f".{hst_dom}")
+            for hst_dom in host.domains
+        )
+
+    def _domain_in_hosts(self, domain: str, hosts: List[Host]) -> bool:
         """Checks if the domain is present in the list of hosts.
 
         Note:
@@ -171,10 +186,7 @@ class SerpApi(AsyncClient):
             domain: The domain to check.
             hosts: The list of hosts to check against.
         """
-        for hst_dom in [dom for hst in hosts for dom in hst.domains]:
-            if domain == hst_dom or domain.endswith(f".{hst_dom}"):
-                return True
-        return False
+        return any(self._domain_in_host(domain=domain, host=hst) for hst in hosts)
 
     def _is_excluded_url(self, domain: str, excluded_urls: List[Host]) -> bool:
         """Checks if the domain is in the excluded URLs.
@@ -183,7 +195,7 @@ class SerpApi(AsyncClient):
             domain: The domain to check.
             excluded_urls: The list of excluded URLs.
         """
-        return self._domain_is_present(domain=domain, hosts=excluded_urls)
+        return self._domain_in_hosts(domain=domain, hosts=excluded_urls)
 
     def _apply_filters(
         self,
@@ -203,13 +215,11 @@ class SerpApi(AsyncClient):
         domain = result.domain
         # Check if the URL is in the marketplaces (if yes, keep the result un-touched)
         if marketplaces:
-            if self._domain_is_present(domain=domain, hosts=marketplaces):
+            if self._domain_in_hosts(domain=domain, hosts=marketplaces):
                 return result
 
-        # Check if the URL has the included country code
-        if not self._has_included_country_code(
-            url=result.url, country_code=location.code
-        ):
+        # Check if the URL has a relevant country_code
+        if not self._relevant_country_code(url=result.url, country_code=location.code):
             result.filtered = True
             result.filtered_at_stage = "SerpAPI (country code filtering)"
             return result
@@ -244,7 +254,9 @@ class SerpApi(AsyncClient):
         if marketplaces:
             try:
                 marketplace_name = next(
-                    mp.name for mp in marketplaces if domain in [d for d in mp.domains]
+                    mp.name
+                    for mp in marketplaces
+                    if self._domain_in_host(domain=domain, host=mp)
                 )
             except StopIteration:
                 logger.warning(f'Failed to find marketplace for domain="{domain}".')

{fraudcrawler-0.3.8 → fraudcrawler-0.3.10}/fraudcrawler/settings.py

@@ -8,6 +8,9 @@ ROOT_DIR = Path(__file__).parents[1]
 # Serp settings
 GOOGLE_LOCATIONS_FILENAME = ROOT_DIR / "fraudcrawler" / "base" / "google-locations.json"
 GOOGLE_LANGUAGES_FILENAME = ROOT_DIR / "fraudcrawler" / "base" / "google-languages.json"
+SERP_DEFAULT_COUNTRY_CODES = [
+    ".com",
+]
 
 # Enrichment settings
 ENRICHMENT_DEFAULT_LIMIT = 10

{fraudcrawler-0.3.8 → fraudcrawler-0.3.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "fraudcrawler"
-version = "0.3.8"
+version = "0.3.10"
 description = "Intelligent Market Monitoring"
 authors = [
     "Domingo Bertus <hello@veanu.ch>",