fraudcrawler 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

fraudcrawler/base/orchestrator.py

@@ -4,7 +4,12 @@ import logging
 from pydantic import BaseModel, Field
 from typing import Dict, List, Set, cast
 
-from fraudcrawler.settings import PROCESSOR_DEFAULT_MODEL, MAX_RETRIES, RETRY_DELAY
+from fraudcrawler.settings import (
+    PROCESSOR_DEFAULT_MODEL,
+    PROCESSOR_DEFAULT_IF_MISSING,
+    MAX_RETRIES,
+    RETRY_DELAY,
+)
 from fraudcrawler.settings import (
     DEFAULT_N_SERP_WKRS,
     DEFAULT_N_ZYTE_WKRS,
@@ -67,6 +72,7 @@ class Orchestrator(ABC):
         openai_model: str = PROCESSOR_DEFAULT_MODEL,
         max_retries: int = MAX_RETRIES,
         retry_delay: int = RETRY_DELAY,
+        default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
         n_serp_wkrs: int = DEFAULT_N_SERP_WKRS,
         n_zyte_wkrs: int = DEFAULT_N_ZYTE_WKRS,
         n_proc_wkrs: int = DEFAULT_N_PROC_WKRS,
@@ -98,7 +104,11 @@ class Orchestrator(ABC):
         self._zyteapi = ZyteApi(
             api_key=zyteapi_key, max_retries=max_retries, retry_delay=retry_delay
         )
-        self._processor = Processor(api_key=openaiapi_key, model=openai_model)
+        self._processor = Processor(
+            api_key=openaiapi_key,
+            model=openai_model,
+            default_if_missing=default_if_missing,
+        )
 
         # Setup the async framework
         self._n_serp_wkrs = n_serp_wkrs

fraudcrawler/processing/processor.py

@@ -15,15 +15,22 @@ logger = logging.getLogger(__name__)
 class Processor:
     """Processes product data for classification based on a prompt configuration."""
 
-    def __init__(self, api_key: str, model: str):
+    def __init__(
+        self,
+        api_key: str,
+        model: str,
+        default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
+    ):
         """Initializes the Processor.
 
         Args:
             api_key: The OpenAI API key.
             model: The OpenAI model to use.
+            default_if_missing: The default classification to return if error occurs.
         """
         self._client = AsyncOpenAI(api_key=api_key)
         self._model = model
+        self._default_if_missing = default_if_missing
 
     async def _call_openai_api(
         self,
@@ -67,7 +74,7 @@ class Processor:
             logger.warning(
                 f"Missing required fields for classification: name='{name}', description='{description}'"
             )
-            return PROCESSOR_DEFAULT_IF_MISSING
+            return self._default_if_missing
 
         # Substitute placeholders in user_prompt with the relevant arguments
         user_prompt = PROCESSOR_USER_PROMPT_TEMPLATE.format(
@@ -94,7 +101,7 @@ class Processor:
                 logger.warning(
                     f"Classification '{classification}' not in allowed classes {prompt.allowed_classes}"
                 )
-                return PROCESSOR_DEFAULT_IF_MISSING
+                return self._default_if_missing
 
             logger.info(
                 f'Classification for "{name}" (prompt={prompt.name}): {classification}'
@@ -105,4 +112,4 @@ class Processor:
             logger.error(
                 f'Error classifying product "{name}" with prompt "{prompt.name}": {e}'
             )
-            return PROCESSOR_DEFAULT_IF_MISSING
+            return self._default_if_missing

{fraudcrawler-0.3.4.dist-info → fraudcrawler-0.3.6.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.3.4
+Version: 0.3.6
 Summary: Intelligent Market Monitoring
-Home-page: https://github/open-veanu/fraudcrawler
+Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT
 Author: Domingo Bertus
 Author-email: hello@veanu.ch
@@ -17,7 +17,7 @@ Requires-Dist: openai (>=1.68.2,<2.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Project-URL: Repository, https://github/open-veanu/fraudcrawler
+Project-URL: Repository, https://github.com/open-veanu/fraudcrawler
 Description-Content-Type: text/markdown
 
 # open-veanu/fraudcrawler

{fraudcrawler-0.3.4.dist-info → fraudcrawler-0.3.6.dist-info}/RECORD

@@ -4,17 +4,17 @@ fraudcrawler/base/base.py,sha256=KnwOcy35EKyelcgVh95LmOZziWFS6dKlegLK6A96wvg,448
 fraudcrawler/base/client.py,sha256=GcTUMqLfvweLFdHy6CP9tgxsFQiPkc6KyiLcwLnDiw8,4412
 fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
 fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
-fraudcrawler/base/orchestrator.py,sha256=la4BgzhzX0KCkEQhtvueT9iJ4RRZR-YG3BMxG58Ko-I,23780
+fraudcrawler/base/orchestrator.py,sha256=Gmryv8l8nB1QUwwjLoZGop2mwKqWYQQORT_96_w5ptA,23981
 fraudcrawler/launch_demo_pipeline.py,sha256=RIZTtdtZeJPhvSLp1IUjT_nhme_2q6mAGWKoL838E4E,4320
 fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/processing/processor.py,sha256=Z1pr2X7GEpxlB-mmGIIC72qTuRW5YJU1DQpKsQO_lAs,3770
+fraudcrawler/processing/processor.py,sha256=IFVKIiNi0QoCAgPFkFtNDgxfhh01iDNUyIBZWACplR8,3993
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=X1BBZshdZqPmbduzhGwH0ULSzq03L_7bf7_UL8yOQ9E,10608
 fraudcrawler/scraping/serp.py,sha256=wT8vhk0EugcrS2CCvMuCCZrlw1MRI-ahtGYKdNUZQo8,8830
 fraudcrawler/scraping/zyte.py,sha256=ggI4iYG-E_UyiKgUpEFekeUd1giifEfJ_uyFUSJGSLY,6296
 fraudcrawler/settings.py,sha256=yAgGvZ9wAdaYbN5c0SBZoTUkjjLOyU2je1109qcbTzQ,723
-fraudcrawler-0.3.4.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.3.4.dist-info/METADATA,sha256=FslB5x4HVPwRadwg5h83s7dDvJ78lwH4ky62mh1S2cM,5957
-fraudcrawler-0.3.4.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fraudcrawler-0.3.4.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.3.4.dist-info/RECORD,,
+fraudcrawler-0.3.6.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.3.6.dist-info/METADATA,sha256=jM68uPtxj73vhNjFz5NbzfY40yYGTqGL8JHn0W0oVjk,5965
+fraudcrawler-0.3.6.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fraudcrawler-0.3.6.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.3.6.dist-info/RECORD,,