fraudcrawler 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

fraudcrawler/scraping/serp.py

@@ -196,6 +196,27 @@ class SerpApi(AsyncClient):
             filtered_at_stage=filtered_at_stage,
         )
 
+    @staticmethod
+    def _is_excluded(domain: str, excluded_urls: List[Host]) -> bool:
+        """Checks if the domain is in the excluded URLs.
+
+        Note:
+            By checking `if dom == excl or dom.endswith(f".{excl}")` we also
+            check for subdomains. For example, if the domain is
+            `link.springer.com` and the excluded URL is `springer.com`,
+            it will be excluded.
+
+        Args:
+            domain: The domain to check.
+            excluded_urls: The list of excluded URLs.
+        """
+        dom = domain.lower()
+        excl_doms = [dom.lower() for excl in excluded_urls for dom in excl.domains]
+        for excl in excl_doms:
+            if dom == excl or dom.endswith(f".{excl}"):
+                return True
+        return False
+
     async def apply(
         self,
         search_term: str,
@@ -242,8 +263,11 @@ class SerpApi(AsyncClient):
 
         # Filter out the excluded URLs
         if excluded_urls:
-            excluded = [dom for excl in excluded_urls for dom in excl.domains]
-            results = [res for res in results if res.domain not in excluded]
+            results = [
+                res
+                for res in results
+                if not self._is_excluded(res.domain, excluded_urls)
+            ]
 
         logger.info(
             f'Produced {len(results)} results from SerpApi search with q="{search_string}".'

{fraudcrawler-0.3.5.dist-info → fraudcrawler-0.3.7.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.3.5
+Version: 0.3.7
 Summary: Intelligent Market Monitoring
-Home-page: https://github/open-veanu/fraudcrawler
+Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT
 Author: Domingo Bertus
 Author-email: hello@veanu.ch
@@ -17,7 +17,7 @@ Requires-Dist: openai (>=1.68.2,<2.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Project-URL: Repository, https://github/open-veanu/fraudcrawler
+Project-URL: Repository, https://github.com/open-veanu/fraudcrawler
 Description-Content-Type: text/markdown
 
 # open-veanu/fraudcrawler

{fraudcrawler-0.3.5.dist-info → fraudcrawler-0.3.7.dist-info}/RECORD

@@ -10,11 +10,11 @@ fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 fraudcrawler/processing/processor.py,sha256=IFVKIiNi0QoCAgPFkFtNDgxfhh01iDNUyIBZWACplR8,3993
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=X1BBZshdZqPmbduzhGwH0ULSzq03L_7bf7_UL8yOQ9E,10608
-fraudcrawler/scraping/serp.py,sha256=wT8vhk0EugcrS2CCvMuCCZrlw1MRI-ahtGYKdNUZQo8,8830
+fraudcrawler/scraping/serp.py,sha256=ksN_GgneLLIW0YwAMbBVbWdFD4FvdGnR5Lq_snykLDA,9613
 fraudcrawler/scraping/zyte.py,sha256=ggI4iYG-E_UyiKgUpEFekeUd1giifEfJ_uyFUSJGSLY,6296
 fraudcrawler/settings.py,sha256=yAgGvZ9wAdaYbN5c0SBZoTUkjjLOyU2je1109qcbTzQ,723
-fraudcrawler-0.3.5.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.3.5.dist-info/METADATA,sha256=z0wINs19mCGOFPrXlb4FRSivMQpWG5zTgFmCXu6pIE8,5957
-fraudcrawler-0.3.5.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fraudcrawler-0.3.5.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.3.5.dist-info/RECORD,,
+fraudcrawler-0.3.7.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.3.7.dist-info/METADATA,sha256=po2owcDHfVOmYDr2GnINKbnVlWBrzY77tbB7K_btIFU,5965
+fraudcrawler-0.3.7.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fraudcrawler-0.3.7.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.3.7.dist-info/RECORD,,