fraudcrawler 0.3.3__tar.gz → 0.3.4__tar.gz

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fraudcrawler
-Version: 0.3.3
+Version: 0.3.4
 Summary: Intelligent Market Monitoring
 Home-page: https://github/open-veanu/fraudcrawler
 License: MIT
@@ -68,7 +68,7 @@ The location used in SerpAPI ('gl' parameter). `location=Location('Switzerland')
 Defines the search depth with the number of results to retrieve and optional enrichment parameters.
 
 #### `prompts: List[Prompt]`
-The list of prompts to classify a given product with (multiple) LLM calls. Each prompt object has a `name`, a `context` (used for defining the user prompt), a `system_prompt` (for defining the classification task), `allowed_classes` (a list of possible classes) and optionally `default_if_missing` (a default class if anything goes wrong).
+The list of prompts to classify a given product with (multiple) LLM calls. Each prompt object has a `name`, a `context` (used for defining the user prompt), a `system_prompt` (for defining the classification task), and `allowed_classes` (a list of possible classes).
 
 ```python
 from fraudcrawler import Language, Location, Deepness, Prompt

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/README.md

@@ -46,7 +46,7 @@ The location used in SerpAPI ('gl' parameter). `location=Location('Switzerland')
 Defines the search depth with the number of results to retrieve and optional enrichment parameters.
 
 #### `prompts: List[Prompt]`
-The list of prompts to classify a given product with (multiple) LLM calls. Each prompt object has a `name`, a `context` (used for defining the user prompt), a `system_prompt` (for defining the classification task), `allowed_classes` (a list of possible classes) and optionally `default_if_missing` (a default class if anything goes wrong).
+The list of prompts to classify a given product with (multiple) LLM calls. Each prompt object has a `name`, a `context` (used for defining the user prompt), a `system_prompt` (for defining the classification task), and `allowed_classes` (a list of possible classes).
 
 ```python
 from fraudcrawler import Language, Location, Deepness, Prompt

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/fraudcrawler/base/base.py

@@ -1,6 +1,10 @@
 import json
 import logging
-from pydantic import BaseModel, field_validator, model_validator
+from pydantic import (
+    BaseModel,
+    field_validator,
+    model_validator,
+)
 from pydantic_settings import BaseSettings
 from typing import List
 
@@ -9,7 +13,6 @@ import aiohttp
 from fraudcrawler.settings import (
     GOOGLE_LANGUAGES_FILENAME,
     GOOGLE_LOCATIONS_FILENAME,
-    PROCESSOR_DEFAULT_IF_MISSING,
 )
 
 logger = logging.getLogger(__name__)
@@ -111,7 +114,13 @@ class Prompt(BaseModel):
     context: str
     system_prompt: str
     allowed_classes: List[int]
-    default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING
+
+    @field_validator("allowed_classes", mode="before")
+    def check_for_positive_value(cls, val):
+        """Check if all values are positive."""
+        if not all(isinstance(i, int) and i >= 0 for i in val):
+            raise ValueError("all values in allowed_classes must be positive integers.")
+        return val
 
 
 class AsyncClient:

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/fraudcrawler/base/orchestrator.py

@@ -10,7 +10,6 @@ from fraudcrawler.settings import (
     DEFAULT_N_ZYTE_WKRS,
     DEFAULT_N_PROC_WKRS,
 )
-from fraudcrawler.settings import PRODUCT_ITEM_DEFAULT_IS_RELEVANT
 from fraudcrawler.base.base import Deepness, Host, Language, Location, Prompt
 from fraudcrawler import SerpApi, Enricher, ZyteApi, Processor
 
@@ -40,7 +39,6 @@ class ProductItem(BaseModel):
     # Filtering parameters
     filtered: bool = False
     filtered_at_stage: str | None = None
-    is_relevant: int = PRODUCT_ITEM_DEFAULT_IS_RELEVANT
 
 
 class Orchestrator(ABC):

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/fraudcrawler/processing/processor.py

@@ -3,7 +3,10 @@ import logging
 from openai import AsyncOpenAI
 
 from fraudcrawler.base.base import Prompt
-from fraudcrawler.settings import PROCESSOR_USER_PROMPT_TEMPLATE
+from fraudcrawler.settings import (
+    PROCESSOR_USER_PROMPT_TEMPLATE,
+    PROCESSOR_DEFAULT_IF_MISSING,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -54,7 +57,7 @@ class Processor:
             description: Product description (often used in the user_prompt).
 
         Note:
-            This method returns `prompt.default_if_missing` if:
+            This method returns `PROCESSOR_DEFAULT_IF_MISSING` if:
                 - 'name' or 'description' is None
                 - an error occurs during the API call
                 - if the response isn't in allowed_classes.
@@ -64,7 +67,7 @@ class Processor:
             logger.warning(
                 f"Missing required fields for classification: name='{name}', description='{description}'"
             )
-            return prompt.default_if_missing
+            return PROCESSOR_DEFAULT_IF_MISSING
 
         # Substitute placeholders in user_prompt with the relevant arguments
         user_prompt = PROCESSOR_USER_PROMPT_TEMPLATE.format(
@@ -91,7 +94,7 @@ class Processor:
                 logger.warning(
                     f"Classification '{classification}' not in allowed classes {prompt.allowed_classes}"
                 )
-                return prompt.default_if_missing
+                return PROCESSOR_DEFAULT_IF_MISSING
 
             logger.info(
                 f'Classification for "{name}" (prompt={prompt.name}): {classification}'
@@ -102,4 +105,4 @@ class Processor:
             logger.error(
                 f'Error classifying product "{name}" with prompt "{prompt.name}": {e}'
             )
-            return prompt.default_if_missing
+            return PROCESSOR_DEFAULT_IF_MISSING

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/fraudcrawler/settings.py

@@ -22,9 +22,6 @@ PROCESSOR_USER_PROMPT_TEMPLATE = (
     "Context: {context}\n\nProduct Details: {name}\n{description}\\n\nRelevance:"
 )
 
-# Orchestrator settings
-PRODUCT_ITEM_DEFAULT_IS_RELEVANT = -1
-
 # Async settings
 DEFAULT_N_SERP_WKRS = 10
 DEFAULT_N_ZYTE_WKRS = 10

{fraudcrawler-0.3.3 → fraudcrawler-0.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "fraudcrawler"
-version = "0.3.3"
+version = "0.3.4"
 description = "Intelligent Market Monitoring"
 authors = [
     "Domingo Bertus <hello@veanu.ch>",