fraudcrawler 0.3.4__tar.gz → 0.3.5__tar.gz

{fraudcrawler-0.3.4 → fraudcrawler-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.3.4
+Version: 0.3.5
 Summary: Intelligent Market Monitoring
 Home-page: https://github/open-veanu/fraudcrawler
 License: MIT

{fraudcrawler-0.3.4 → fraudcrawler-0.3.5}/fraudcrawler/base/orchestrator.py

@@ -4,7 +4,12 @@ import logging
 from pydantic import BaseModel, Field
 from typing import Dict, List, Set, cast
 
-from fraudcrawler.settings import PROCESSOR_DEFAULT_MODEL, MAX_RETRIES, RETRY_DELAY
+from fraudcrawler.settings import (
+    PROCESSOR_DEFAULT_MODEL,
+    PROCESSOR_DEFAULT_IF_MISSING,
+    MAX_RETRIES,
+    RETRY_DELAY,
+)
 from fraudcrawler.settings import (
     DEFAULT_N_SERP_WKRS,
     DEFAULT_N_ZYTE_WKRS,
@@ -67,6 +72,7 @@ class Orchestrator(ABC):
         openai_model: str = PROCESSOR_DEFAULT_MODEL,
         max_retries: int = MAX_RETRIES,
         retry_delay: int = RETRY_DELAY,
+        default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
         n_serp_wkrs: int = DEFAULT_N_SERP_WKRS,
         n_zyte_wkrs: int = DEFAULT_N_ZYTE_WKRS,
         n_proc_wkrs: int = DEFAULT_N_PROC_WKRS,
@@ -98,7 +104,11 @@ class Orchestrator(ABC):
         self._zyteapi = ZyteApi(
             api_key=zyteapi_key, max_retries=max_retries, retry_delay=retry_delay
         )
-        self._processor = Processor(api_key=openaiapi_key, model=openai_model)
+        self._processor = Processor(
+            api_key=openaiapi_key,
+            model=openai_model,
+            default_if_missing=default_if_missing,
+        )
 
         # Setup the async framework
         self._n_serp_wkrs = n_serp_wkrs

{fraudcrawler-0.3.4 → fraudcrawler-0.3.5}/fraudcrawler/processing/processor.py

@@ -15,15 +15,22 @@ logger = logging.getLogger(__name__)
 class Processor:
     """Processes product data for classification based on a prompt configuration."""
 
-    def __init__(self, api_key: str, model: str):
+    def __init__(
+        self,
+        api_key: str,
+        model: str,
+        default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
+    ):
         """Initializes the Processor.
 
         Args:
             api_key: The OpenAI API key.
             model: The OpenAI model to use.
+            default_if_missing: The default classification to return if error occurs.
         """
         self._client = AsyncOpenAI(api_key=api_key)
         self._model = model
+        self._default_if_missing = default_if_missing
 
     async def _call_openai_api(
         self,
@@ -67,7 +74,7 @@ class Processor:
             logger.warning(
                 f"Missing required fields for classification: name='{name}', description='{description}'"
             )
-            return PROCESSOR_DEFAULT_IF_MISSING
+            return self._default_if_missing
 
         # Substitute placeholders in user_prompt with the relevant arguments
         user_prompt = PROCESSOR_USER_PROMPT_TEMPLATE.format(
@@ -94,7 +101,7 @@ class Processor:
                 logger.warning(
                     f"Classification '{classification}' not in allowed classes {prompt.allowed_classes}"
                 )
-                return PROCESSOR_DEFAULT_IF_MISSING
+                return self._default_if_missing
 
             logger.info(
                 f'Classification for "{name}" (prompt={prompt.name}): {classification}'
@@ -105,4 +112,4 @@ class Processor:
             logger.error(
                 f'Error classifying product "{name}" with prompt "{prompt.name}": {e}'
             )
-            return PROCESSOR_DEFAULT_IF_MISSING
+            return self._default_if_missing

{fraudcrawler-0.3.4 → fraudcrawler-0.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "fraudcrawler"
-version = "0.3.4"
+version = "0.3.5"
 description = "Intelligent Market Monitoring"
 authors = [
     "Domingo Bertus <hello@veanu.ch>",