fraudcrawler 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

fraudcrawler/base/base.py

@@ -1,6 +1,10 @@
 import json
 import logging
-from pydantic import BaseModel, field_validator, model_validator
+from pydantic import (
+    BaseModel,
+    field_validator,
+    model_validator,
+)
 from pydantic_settings import BaseSettings
 from typing import List
 
@@ -9,7 +13,6 @@ import aiohttp
 from fraudcrawler.settings import (
     GOOGLE_LANGUAGES_FILENAME,
     GOOGLE_LOCATIONS_FILENAME,
-    PROCESSOR_DEFAULT_IF_MISSING,
 )
 
 logger = logging.getLogger(__name__)
@@ -111,7 +114,13 @@ class Prompt(BaseModel):
     context: str
     system_prompt: str
     allowed_classes: List[int]
-    default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING
+
+    @field_validator("allowed_classes", mode="before")
+    def check_for_positive_value(cls, val):
+        """Check if all values are positive."""
+        if not all(isinstance(i, int) and i >= 0 for i in val):
+            raise ValueError("all values in allowed_classes must be positive integers.")
+        return val
 
 
 class AsyncClient:

fraudcrawler/base/orchestrator.py

@@ -4,13 +4,17 @@ import logging
 from pydantic import BaseModel, Field
 from typing import Dict, List, Set, cast
 
-from fraudcrawler.settings import PROCESSOR_DEFAULT_MODEL, MAX_RETRIES, RETRY_DELAY
+from fraudcrawler.settings import (
+    PROCESSOR_DEFAULT_MODEL,
+    PROCESSOR_DEFAULT_IF_MISSING,
+    MAX_RETRIES,
+    RETRY_DELAY,
+)
 from fraudcrawler.settings import (
     DEFAULT_N_SERP_WKRS,
     DEFAULT_N_ZYTE_WKRS,
     DEFAULT_N_PROC_WKRS,
 )
-from fraudcrawler.settings import PRODUCT_ITEM_DEFAULT_IS_RELEVANT
 from fraudcrawler.base.base import Deepness, Host, Language, Location, Prompt
 from fraudcrawler import SerpApi, Enricher, ZyteApi, Processor
 
@@ -40,7 +44,6 @@ class ProductItem(BaseModel):
     # Filtering parameters
     filtered: bool = False
     filtered_at_stage: str | None = None
-    is_relevant: int = PRODUCT_ITEM_DEFAULT_IS_RELEVANT
 
 
 class Orchestrator(ABC):
@@ -69,6 +72,7 @@ class Orchestrator(ABC):
         openai_model: str = PROCESSOR_DEFAULT_MODEL,
         max_retries: int = MAX_RETRIES,
         retry_delay: int = RETRY_DELAY,
+        default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
         n_serp_wkrs: int = DEFAULT_N_SERP_WKRS,
         n_zyte_wkrs: int = DEFAULT_N_ZYTE_WKRS,
         n_proc_wkrs: int = DEFAULT_N_PROC_WKRS,
@@ -100,7 +104,11 @@ class Orchestrator(ABC):
         self._zyteapi = ZyteApi(
             api_key=zyteapi_key, max_retries=max_retries, retry_delay=retry_delay
         )
-        self._processor = Processor(api_key=openaiapi_key, model=openai_model)
+        self._processor = Processor(
+            api_key=openaiapi_key,
+            model=openai_model,
+            default_if_missing=default_if_missing,
+        )
 
         # Setup the async framework
         self._n_serp_wkrs = n_serp_wkrs

fraudcrawler/processing/processor.py

@@ -3,7 +3,10 @@ import logging
 from openai import AsyncOpenAI
 
 from fraudcrawler.base.base import Prompt
-from fraudcrawler.settings import PROCESSOR_USER_PROMPT_TEMPLATE
+from fraudcrawler.settings import (
+    PROCESSOR_USER_PROMPT_TEMPLATE,
+    PROCESSOR_DEFAULT_IF_MISSING,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -12,15 +15,22 @@ logger = logging.getLogger(__name__)
 class Processor:
     """Processes product data for classification based on a prompt configuration."""
 
-    def __init__(self, api_key: str, model: str):
+    def __init__(
+        self,
+        api_key: str,
+        model: str,
+        default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
+    ):
         """Initializes the Processor.
 
         Args:
             api_key: The OpenAI API key.
             model: The OpenAI model to use.
+            default_if_missing: The default classification to return if error occurs.
         """
         self._client = AsyncOpenAI(api_key=api_key)
         self._model = model
+        self._default_if_missing = default_if_missing
 
     async def _call_openai_api(
         self,
@@ -54,7 +64,7 @@ class Processor:
             description: Product description (often used in the user_prompt).
 
         Note:
-            This method returns `prompt.default_if_missing` if:
+            This method returns `PROCESSOR_DEFAULT_IF_MISSING` if:
                 - 'name' or 'description' is None
                 - an error occurs during the API call
                 - if the response isn't in allowed_classes.
@@ -64,7 +74,7 @@ class Processor:
             logger.warning(
                 f"Missing required fields for classification: name='{name}', description='{description}'"
             )
-            return prompt.default_if_missing
+            return self._default_if_missing
 
         # Substitute placeholders in user_prompt with the relevant arguments
         user_prompt = PROCESSOR_USER_PROMPT_TEMPLATE.format(
@@ -91,7 +101,7 @@ class Processor:
                 logger.warning(
                     f"Classification '{classification}' not in allowed classes {prompt.allowed_classes}"
                 )
-                return prompt.default_if_missing
+                return self._default_if_missing
 
             logger.info(
                 f'Classification for "{name}" (prompt={prompt.name}): {classification}'
@@ -102,4 +112,4 @@ class Processor:
             logger.error(
                 f'Error classifying product "{name}" with prompt "{prompt.name}": {e}'
             )
-            return prompt.default_if_missing
+            return self._default_if_missing

fraudcrawler/settings.py

@@ -22,9 +22,6 @@ PROCESSOR_USER_PROMPT_TEMPLATE = (
     "Context: {context}\n\nProduct Details: {name}\n{description}\\n\nRelevance:"
 )
 
-# Orchestrator settings
-PRODUCT_ITEM_DEFAULT_IS_RELEVANT = -1
-
 # Async settings
 DEFAULT_N_SERP_WKRS = 10
 DEFAULT_N_ZYTE_WKRS = 10

{fraudcrawler-0.3.3.dist-info → fraudcrawler-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.3.3
+Version: 0.3.5
 Summary: Intelligent Market Monitoring
 Home-page: https://github/open-veanu/fraudcrawler
 License: MIT
@@ -68,7 +68,7 @@ The location used in SerpAPI ('gl' parameter). `location=Location('Switzerland')
 Defines the search depth with the number of results to retrieve and optional enrichment parameters.
 
 #### `prompts: List[Prompt]`
-The list of prompts to classify a given product with (multiple) LLM calls. Each prompt object has a `name`, a `context` (used for defining the user prompt), a `system_prompt` (for defining the classification task), `allowed_classes` (a list of possible classes) and optionally `default_if_missing` (a default class if anything goes wrong).
+The list of prompts to classify a given product with (multiple) LLM calls. Each prompt object has a `name`, a `context` (used for defining the user prompt), a `system_prompt` (for defining the classification task), and `allowed_classes` (a list of possible classes).
 
 ```python
 from fraudcrawler import Language, Location, Deepness, Prompt

{fraudcrawler-0.3.3.dist-info → fraudcrawler-0.3.5.dist-info}/RECORD

@@ -1,20 +1,20 @@
 fraudcrawler/__init__.py,sha256=2EgoTb2jNcQt1NxUV8za0154kb7ZnHZ_KeKgx21rdFs,679
 fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/base/base.py,sha256=YgX7cUB3Fta-sXWZu5I-gn85sCfpmoa8M67Whn1m56o,4241
+fraudcrawler/base/base.py,sha256=KnwOcy35EKyelcgVh95LmOZziWFS6dKlegLK6A96wvg,4485
 fraudcrawler/base/client.py,sha256=GcTUMqLfvweLFdHy6CP9tgxsFQiPkc6KyiLcwLnDiw8,4412
 fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
 fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
-fraudcrawler/base/orchestrator.py,sha256=GmJvrP9jKr4FqTKhuU9YMEuZ54gV0asHrSCxhM43onA,23903
+fraudcrawler/base/orchestrator.py,sha256=Gmryv8l8nB1QUwwjLoZGop2mwKqWYQQORT_96_w5ptA,23981
 fraudcrawler/launch_demo_pipeline.py,sha256=RIZTtdtZeJPhvSLp1IUjT_nhme_2q6mAGWKoL838E4E,4320
 fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/processing/processor.py,sha256=sNYK0gY7PsJJS5FMYOiHLXEQJ70buifSokuIiXk5dG4,3715
+fraudcrawler/processing/processor.py,sha256=IFVKIiNi0QoCAgPFkFtNDgxfhh01iDNUyIBZWACplR8,3993
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=X1BBZshdZqPmbduzhGwH0ULSzq03L_7bf7_UL8yOQ9E,10608
 fraudcrawler/scraping/serp.py,sha256=wT8vhk0EugcrS2CCvMuCCZrlw1MRI-ahtGYKdNUZQo8,8830
 fraudcrawler/scraping/zyte.py,sha256=ggI4iYG-E_UyiKgUpEFekeUd1giifEfJ_uyFUSJGSLY,6296
-fraudcrawler/settings.py,sha256=eUb7zd2Q7fYUrLk4cl_d48lZ9zaB8iU7M0zFFuZc_-g,786
-fraudcrawler-0.3.3.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.3.3.dist-info/METADATA,sha256=860K3oeNRoAC2Fmr9D4Gd1DYaXw1KOaWSEBeByL4V-U,6030
-fraudcrawler-0.3.3.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fraudcrawler-0.3.3.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.3.3.dist-info/RECORD,,
+fraudcrawler/settings.py,sha256=yAgGvZ9wAdaYbN5c0SBZoTUkjjLOyU2je1109qcbTzQ,723
+fraudcrawler-0.3.5.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.3.5.dist-info/METADATA,sha256=z0wINs19mCGOFPrXlb4FRSivMQpWG5zTgFmCXu6pIE8,5957
+fraudcrawler-0.3.5.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fraudcrawler-0.3.5.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.3.5.dist-info/RECORD,,