fraudcrawler 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

fraudcrawler/base/base.py

@@ -63,6 +63,14 @@ class Host(BaseModel):
         return [cls._normalize_domain(dom.strip()) for dom in val]
 
 
+class ClassificationResult(BaseModel):
+    """Model for classification results."""
+
+    result: int
+    input_tokens: int
+    output_tokens: int
+
+
 class Location(BaseModel):
     """Model for location details (e.g. `Location(name="Switzerland", code="ch")`)."""
 
@@ -137,6 +145,9 @@ class ProductItem(BaseModel):
     # Processor parameters are set dynamic so we must allow extra fields
     classifications: Dict[str, int] = Field(default_factory=dict)
 
+    # Usage parameters
+    usage: Dict[str, Dict[str, int]] = Field(default_factory=dict)
+
     # Filtering parameters
     filtered: bool = False
     filtered_at_stage: str | None = None

fraudcrawler/base/orchestrator.py

@@ -281,7 +281,13 @@ class Orchestrator(ABC):
                             url=url,
                             product_details=product_details,
                         )
-                        product.classifications[prompt.name] = classification
+                        product.classifications[prompt.name] = int(
+                            classification.result
+                        )
+                        product.usage[prompt.name] = {
+                            "input_tokens": classification.input_tokens,
+                            "output_tokens": classification.output_tokens,
+                        }
                 except Exception as e:
                     logger.warning(f"Error processing product: {e}.")
 

fraudcrawler/processing/processor.py

@@ -2,10 +2,11 @@ import logging
 
 from openai import AsyncOpenAI
 
-from fraudcrawler.base.base import Prompt
+from fraudcrawler.base.base import Prompt, ClassificationResult
 from fraudcrawler.settings import (
     PROCESSOR_USER_PROMPT_TEMPLATE,
     PROCESSOR_DEFAULT_IF_MISSING,
+    PROCESSOR_EMPTY_TOKEN_COUNT,
 )
 
 
@@ -20,6 +21,7 @@ class Processor:
         api_key: str,
         model: str,
         default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING,
+        empty_token_count: int = PROCESSOR_EMPTY_TOKEN_COUNT,
     ):
         """Initializes the Processor.
 
@@ -27,17 +29,22 @@ class Processor:
             api_key: The OpenAI API key.
             model: The OpenAI model to use.
             default_if_missing: The default classification to return if error occurs.
+            empty_token_count: The default value to return as tokensif the classification is empty.
         """
         self._client = AsyncOpenAI(api_key=api_key)
         self._model = model
-        self._default_if_missing = default_if_missing
+        self._error_response = ClassificationResult(
+            result=default_if_missing,
+            input_tokens=empty_token_count,
+            output_tokens=empty_token_count,
+        )
 
     async def _call_openai_api(
         self,
         system_prompt: str,
         user_prompt: str,
         **kwargs,
-    ) -> str:
+    ) -> ClassificationResult:
         """Calls the OpenAI API with the given user prompt."""
         response = await self._client.chat.completions.create(
             model=self._model,
@@ -50,10 +57,24 @@ class Processor:
         content = response.choices[0].message.content
         if not content:
             raise ValueError("Empty response from OpenAI API")
-        return content
 
-    async def classify(self, prompt: Prompt, url: str, product_details: str) -> int:
-        """A generic classification method that classifies a product based on a prompt object.
+        # Convert the content to an integer
+        content = int(content.strip())
+
+        # For tracking consumption we alre return the tokens used
+        classification = ClassificationResult(
+            result=content,
+            input_tokens=response.usage.prompt_tokens,
+            output_tokens=response.usage.completion_tokens,
+        )
+
+        return classification
+
+    async def classify(
+        self, prompt: Prompt, url: str, product_details: str
+    ) -> ClassificationResult:
+        """A generic classification method that classifies a product based on a prompt object and returns
+          the classification, input tokens, and output tokens.
 
         Args:
             prompt: A dictionary with keys "system_prompt", etc.
@@ -69,7 +90,7 @@ class Processor:
         # If required fields are missing, return the prompt's default fallback if provided.
         if not product_details:
             logger.warning("Missing required product_details for classification.")
-            return self._default_if_missing
+            return self._error_response
 
         # Substitute placeholders in user_prompt with the relevant arguments
         user_prompt = PROCESSOR_USER_PROMPT_TEMPLATE.format(
@@ -81,22 +102,21 @@ class Processor:
             logger.debug(
                 f'Calling OpenAI API for classification (url="{url}", prompt="{prompt.name}")'
             )
-            content = await self._call_openai_api(
+            classification = await self._call_openai_api(
                 system_prompt=prompt.system_prompt,
                 user_prompt=user_prompt,
                 max_tokens=1,
             )
-            classification = int(content.strip())
 
             # Enforce that the classification is in the allowed classes
-            if classification not in prompt.allowed_classes:
+            if classification.result not in prompt.allowed_classes:
                 logger.warning(
-                    f"Classification '{classification}' not in allowed classes {prompt.allowed_classes}"
+                    f"Classification '{classification.result}' not in allowed classes {prompt.allowed_classes}"
                 )
-                return self._default_if_missing
+                return self._error_response
 
             logger.info(
-                f'Classification for url="{url}" (prompt={prompt.name}): {classification}'
+                f'Classification for url="{url}" (prompt={prompt.name}): {classification.result} and total tokens used: {classification.input_tokens + classification.output_tokens}'
             )
             return classification
 
@@ -104,4 +124,4 @@ class Processor:
             logger.error(
                 f'Error classifying product at url="{url}" with prompt "{prompt.name}": {e}'
             )
-            return self._default_if_missing
+            return self._error_response

fraudcrawler/settings.py

@@ -15,14 +15,43 @@ SERP_DEFAULT_COUNTRY_CODES: List[str] = [
 
 # URL De-duplication settings
 KNOWN_TRACKERS = [
-    "srsltid",
-    "utm_source",
-    "utm_medium",
-    "utm_campaign",
-    "utm_term",
-    "utm_content",
-    "ar",
-    "ps",
+    "srsltid",        # Search result click ID (used by some search engines)
+    "utm_source",     # UTM: Source of the traffic (e.g., Google, Newsletter)
+    "utm_medium",     # UTM: Medium such as CPC, email, social
+    "utm_campaign",   # UTM: Campaign name (e.g., summer_sale)
+    "utm_term",       # UTM: Keyword term (used in paid search)
+    "utm_content",    # UTM: Used to differentiate similar links or ads
+    "ar",             # Often used for ad region or targeting info
+    "ps",             # Could refer to promotion source or partner segment
+    "gclid",          # Google Ads click ID (auto-tagging)
+    "gclsrc",         # Source of the GCLID (e.g., ads, search)
+    "sku",            # Product SKU identifier, often used in ecommerce links
+    "ref",            # Referrer username or source (e.g., GitHub ref links)
+    "referral",       # Alternate form of referrer, often human-readable
+    "aff_id",         # Affiliate identifier (ID-based)
+    "aff",            # Short form for affiliate tag
+    "affiliate",      # Affiliate tracking parameter (human-readable)
+    "partner",        # Indicates marketing or distribution partner
+    "fbclid",         # Facebook Click Identifier
+    "msclkid",        # Microsoft/Bing Ads click identifier
+    "twclid",         # Twitter Ads click identifier
+    "variant",        # A/B test variant (used to test versions of pages)
+    "session_id",     # Session tracking ID, should not persist across URLs
+    "track",          # Generic flag used to enable/disable tracking
+    "cid",            # Campaign ID (used in ads or emails)
+    "campaignid",     # Alternate or long-form campaign ID
+    "adgroup",        # Ad group identifier for campaigns
+    "bannerid",       # Specific banner ad ID (for display ad tracking)
+    "token",          # Often used to identify users or temporary sessions
+    "tag",            # Affiliate or marketing tag (used for tracking)
+    "hash",           # Generic hash identifier, often for state or cache
+    "user",           # User ID or identifier passed in URL (should be avoided)
+    "src",            # Generic source indicator, less formal than `utm_source`
+    "selsort",        # Sorting parameter for search results
+    "shid",           # Shop ID (used in ecommerce)
+    "shoparea",       # Shop area (used in ecommerce)
+    "shopid",         # Shop ID (used in ecommerce)
+    "shoparea",       # Shop area (used in ecommerce)
 ]
 
 # Enrichment settings
@@ -34,6 +63,7 @@ ZYTE_DEFALUT_PROBABILITY_THRESHOLD = 0.1
 # Processor settings
 PROCESSOR_DEFAULT_MODEL = "gpt-4o"
 PROCESSOR_DEFAULT_IF_MISSING = -1
+PROCESSOR_EMPTY_TOKEN_COUNT = -1
 PROCESSOR_USER_PROMPT_TEMPLATE = "Product Details:\n{product_details}\n\nRelevance:"
 PROCESSOR_PRODUCT_DETAILS_TEMPLATE = "{field_name}:\n{field_value}"
 

{fraudcrawler-0.4.3.dist-info → fraudcrawler-0.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: fraudcrawler
-Version: 0.4.3
+Version: 0.4.5
 Summary: Intelligent Market Monitoring
 Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT
@@ -11,7 +11,6 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.11.14,<4.0.0)
 Requires-Dist: beautifulsoup4 (>=4.13.4,<5.0.0)
 Requires-Dist: openai (>=1.68.2,<2.0.0)

{fraudcrawler-0.4.3.dist-info → fraudcrawler-0.4.5.dist-info}/RECORD

@@ -1,21 +1,21 @@
 fraudcrawler/__init__.py,sha256=zAqnJ9Mewq0qzSfOjyaICyqDRQZE_Z3FmyF2IPdOhXo,788
 fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/base/base.py,sha256=JWjZ3mpX4caQAsWKYqtHrUqHfHr6GXlAaEjxxHV9ODQ,6020
+fraudcrawler/base/base.py,sha256=pYGdRV_Ssw5fA6tLVhlZwAO0OLQl6qn6LgJPCzOCrpc,6258
 fraudcrawler/base/client.py,sha256=FibiYycjUys-c4sv66Y2JqJu5y15be2MYd2_9yB3wG8,4936
 fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
 fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
-fraudcrawler/base/orchestrator.py,sha256=xOMxA0zPUXSF8AGY5AUqzsOO9LfRIjxI2HuZf__Z_sI,24689
+fraudcrawler/base/orchestrator.py,sha256=cum23GYZHzaivFi8ZAIvbzCkpwwdvyhxBROAaDpTEeM,24984
 fraudcrawler/launch_demo_pipeline.py,sha256=CX4A-E63ER7Ip9RNI_IyTAXerYXcQ-NoSvhvLDLdP-s,4640
 fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/processing/processor.py,sha256=An2orst0YRIav7bFuoDMgjwWz2Z9dyjVUbkNAMXNTTo,3748
+fraudcrawler/processing/processor.py,sha256=LM9aSJiVskrwqdoI7FnEglM6zK8fW6e5bhp8ecUuD0E,4633
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=X1BBZshdZqPmbduzhGwH0ULSzq03L_7bf7_UL8yOQ9E,10608
 fraudcrawler/scraping/serp.py,sha256=divEp1UBUsws24PWZABhWIxOmaLqLwdeGn4KNrqWkYA,17865
 fraudcrawler/scraping/url.py,sha256=5Z3hPW73E-TLhM-Zha8OTcUOumc_rcx64R0fT9z2Hi8,1748
 fraudcrawler/scraping/zyte.py,sha256=DUF5pIwpZyQw30qURnFxtp8KYpUgBkrXjM7RaVGH92Q,7005
-fraudcrawler/settings.py,sha256=31jvRFfB-gsVbeidLLl4iQgrFL7GH-824lerIniPI08,1017
-fraudcrawler-0.4.3.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.4.3.dist-info/METADATA,sha256=jlk2WdtXEK0-s6QRQdI96EBpQiyHWKgJiYeW93yiU24,5931
-fraudcrawler-0.4.3.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fraudcrawler-0.4.3.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.4.3.dist-info/RECORD,,
+fraudcrawler/settings.py,sha256=XL4hnTO-Ks4BY_SRl8zdSUg5aUN3ZXyDg5kHx_xflJo,3365
+fraudcrawler-0.4.5.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.4.5.dist-info/METADATA,sha256=N9Kjjf5YwqzqP6jyq-onZkE_rEuTcmdiUawj6baHEHk,5880
+fraudcrawler-0.4.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fraudcrawler-0.4.5.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.4.5.dist-info/RECORD,,

{fraudcrawler-0.4.3.dist-info → fraudcrawler-0.4.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.0
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any