fraudcrawler 0.5.0__py3-none-any.whl → 0.7.22__py3-none-any.whl

fraudcrawler/base/retry.py

@@ -1,4 +1,3 @@
-from httpx import HTTPStatusError
 from tenacity import (
     AsyncRetrying,
     retry_if_exception,
@@ -17,24 +16,39 @@ from fraudcrawler.settings import (
 
 
 def _is_retryable_exception(err: BaseException) -> bool:
-    if (
-        isinstance(err, HTTPStatusError)
-        and err.response.status_code in RETRY_SKIP_IF_CODE
-    ):
+    """Checks if failing HTTP connection is worth to be re-tried."""
+
+    # Get status_code from err
+    response = getattr(err, "response", None)
+    if response is not None:
+        status_code = getattr(response, "status_code", None)
+    else:
+        status_code = getattr(err, "status_code", None)
+
+    # Check if we skip retry
+    if status_code is not None and status_code in RETRY_SKIP_IF_CODE:
         return False
+
+    # Else we do try it again
     return True
 
 
-def get_async_retry() -> AsyncRetrying:
+def get_async_retry(
+    stop_after: int = RETRY_STOP_AFTER_ATTEMPT,
+    initial_delay: int = RETRY_INITIAL_DELAY,
+    max_delay: int = RETRY_MAX_DELAY,
+    exp_base: int = RETRY_EXP_BASE,
+    jitter: int = RETRY_JITTER,
+) -> AsyncRetrying:
     """returns the retry configuration for async operations."""
     return AsyncRetrying(
         retry=retry_if_exception(_is_retryable_exception),
-        stop=stop_after_attempt(RETRY_STOP_AFTER_ATTEMPT),
+        stop=stop_after_attempt(stop_after),
         wait=wait_exponential_jitter(
-            initial=RETRY_INITIAL_DELAY,
-            max=RETRY_MAX_DELAY,
-            exp_base=RETRY_EXP_BASE,
-            jitter=RETRY_JITTER,
+            initial=initial_delay,
+            max=max_delay,
+            exp_base=exp_base,
+            jitter=jitter,
         ),
         reraise=True,
     )

fraudcrawler/launch_demo_pipeline.py

@@ -1,55 +1,84 @@
+import asyncio
 import logging
+from typing import Sequence
 
-from fraudcrawler import FraudCrawlerClient, Language, Location, Deepness, Prompt
+from fraudcrawler.base.base import Setup
+from fraudcrawler import (
+    FraudCrawlerClient,
+    HttpxAsyncClient,
+    Searcher,
+    Enricher,
+    URLCollector,
+    ZyteAPI,
+    SearchEngineName,
+    Language,
+    Location,
+    Deepness,
+    Processor,
+    Workflow,
+    OpenAIClassification,
+)
 
 LOG_FMT = "%(asctime)s | %(name)s | %(funcName)s | %(levelname)s | %(message)s"
 LOG_LVL = "INFO"
 DATE_FMT = "%Y-%m-%d %H:%M:%S"
+SETUP = Setup()  # type: ignore[call-arg]
 logging.basicConfig(format=LOG_FMT, level=LOG_LVL, datefmt=DATE_FMT)
 
 
-def search(search_term: str):
-    # Setup the client
-    client = FraudCrawlerClient()
-
-    # Setup the search
-    language = Language(name="German")
-    location = Location(name="Switzerland")
-    deepness = Deepness(num_results=10)
-    prompts = [
-        Prompt(
+def _setup_workflows(http_client: HttpxAsyncClient) -> Sequence[Workflow]:
+    """Sets up the set of workflows to be run iteratively."""
+    _AVAILABILITY_SYSTEM_PROMPT = (
+        "You are a helpful and intelligent assistant helping an organization that is interested in checking the availability of certain products."
+        "Your task is to classify any given product as either available (1) or not available (0), strictly based on the context and product details provided by the user. "
+        "You must consider all aspects of the given context and make a binary decision accordingly. "
+        "If the product can be purchased, added to a shopping basket, delivered, or is listed as available in any form, classify it as 1 (available); "
+        "if there is any mention of out of stock, not available, no longer shippable, or similar, classify it as 0 (not available). "
+        "Respond only with the number 1 or 0."
+    )
+    _SERIOUSNESS_SYSTEM_PROMPT = (
+        "You are a helpful and intelligent assistant helping an organization that is interested in checking the energy efficiency of certain devices. "
+        "Your task is to classify each item as either a product for sale (1) or not a product for sale (0). To make this distinction, consider the following criteria: \n"
+        "    1 Product for Sale (1): Classify as 1 if the result clearly indicates an item available for purchase, typically found  "
+        "within an online shop or marketplace.\n"
+        "    2 Not a Product for Sale (0): Classify as 0 if the result is unrelated to a direct purchase of a product. This includes items such as: \n"
+        "        - Books and Videos: These may be available for sale, but if they are about or related to the searched product rather than being the "
+        "exact product itself, classify as 0.\n"
+        "        - Advertisements: Promotional content that doesn't directly sell a product.\n"
+        "        - Companies and Services: Names and descriptions of companies or services related to the product but not the product itself.\n"
+        "        - Related Topics/Content: Any text or media that discusses or elaborates on the topic without offering a tangible product for sale.\n"
+        "Make your decision based solely on the context and details provided in the search result. Respond only with the number 1 or 0."
+    )
+    return [
+        OpenAIClassification(
+            http_client=http_client,
             name="availability",
-            system_prompt=(
-                "You are a helpful and intelligent assistant helping an organization that is interested in checking the availability of certain products."
-                "Your task is to classify any given product as either available (1) or not available (0), strictly based on the context and product details provided by the user. "
-                "You must consider all aspects of the given context and make a binary decision accordingly. "
-                "If the product can be purchased, added to a shopping basket, delivered, or is listed as available in any form, classify it as 1 (available); "
-                "if there is any mention of out of stock, not available, no longer shippable, or similar, classify it as 0 (not available). "
-                "Respond only with the number 1 or 0."
-            ),
+            api_key=SETUP.openaiapi_key,
+            model="gpt-4o",
             product_item_fields=["product_name", "html_clean"],
+            system_prompt=_AVAILABILITY_SYSTEM_PROMPT,
+            allowed_classes=[0, 1],
+        ),
+        OpenAIClassification(
+            http_client=http_client,
+            name="seriousness",
+            api_key=SETUP.openaiapi_key,
+            model="gpt-4o",
+            product_item_fields=["product_name", "product_description"],
+            system_prompt=_SERIOUSNESS_SYSTEM_PROMPT,
             allowed_classes=[0, 1],
         ),
-        # Prompt(
-        #     name="seriousness",
-        #     system_prompt=(
-        #         "You are a helpful and intelligent assistant helping an organization that is interested in checking the energy efficiency of certain devices. "
-        #         "Your task is to classify each item as either a product for sale (1) or not a product for sale (0). To make this distinction, consider the following criteria: \n"
-        #         "    1 Product for Sale (1): Classify as 1 if the result clearly indicates an item available for purchase, typically found  "
-        #         "within an online shop or marketplace.\n"
-        #         "    2 Not a Product for Sale (0): Classify as 0 if the result is unrelated to a direct purchase of a product. This includes items such as: \n"
-        #         "        - Books and Videos: These may be available for sale, but if they are about or related to the searched product rather than being the "
-        #         "exact product itself, classify as 0.\n"
-        #         "        - Advertisements: Promotional content that doesn't directly sell a product.\n"
-        #         "        - Companies and Services: Names and descriptions of companies or services related to the product but not the product itself.\n"
-        #         "        - Related Topics/Content: Any text or media that discusses or elaborates on the topic without offering a tangible product for sale.\n"
-        #         "Make your decision based solely on the context and details provided in the search result. Respond only with the number 1 or 0."
-        #     ),
-        #     product_item_fields=["product_name", "product_description"],
-        #     allowed_classes=[0, 1],
-        # ),
     ]
-    # # Optional: Add tern ENRICHEMENT
+
+
+async def run(http_client: HttpxAsyncClient, search_term: str):
+    # Setup the search
+    search_engines = list(SearchEngineName)
+    language = Language(name="German")
+    location = Location(name="Switzerland")
+    deepness = Deepness(num_results=10)
+
+    # # Optional: Add term enrichment
     # from fraudcrawler import Enrichment
 
     # deepness.enrichment = Enrichment(additional_terms=10, additional_urls_per_term=20)
@@ -66,13 +95,41 @@ def search(search_term: str):
         Host(name="Brack", domains="brack.ch"),
     ]
 
+    # Setup clients
+    searcher = Searcher(
+        http_client=http_client,
+        serpapi_key=SETUP.serpapi_key,
+        zyteapi_key=SETUP.zyteapi_key,
+    )
+    enricher = Enricher(
+        http_client=http_client,
+        user=SETUP.dataforseo_user,
+        pwd=SETUP.dataforseo_pwd,
+    )
+    url_collector = URLCollector()
+    zyteapi = ZyteAPI(
+        http_client=http_client,
+        api_key=SETUP.zyteapi_key,
+    )
+    workflows = _setup_workflows(http_client=http_client)
+    processor = Processor(workflows=workflows)
+
+    # Setup the client
+    client = FraudCrawlerClient(
+        searcher=searcher,
+        enricher=enricher,
+        url_collector=url_collector,
+        zyteapi=zyteapi,
+        processor=processor,
+    )
+
     # Execute the pipeline
-    client.execute(
+    await client.run(
         search_term=search_term,
+        search_engines=search_engines,
         language=language,
         location=location,
         deepness=deepness,
-        prompts=prompts,
         # marketplaces=marketplaces,
         excluded_urls=excluded_urls,
     )
@@ -96,5 +153,10 @@ def search(search_term: str):
     print()
 
 
+async def main(search_term: str):
+    async with HttpxAsyncClient() as http_client:
+        await run(http_client=http_client, search_term=search_term)
+
+
 if __name__ == "__main__":
-    search(search_term='Liebherr "TP1410"')
+    asyncio.run(main(search_term="Kaffeebohnen"))

fraudcrawler/processing/base.py

@@ -0,0 +1,129 @@
+from abc import ABC, abstractmethod
+import logging
+from pydantic import BaseModel
+from typing import Any, Dict, List, Sequence, TypeAlias
+
+from fraudcrawler.base.base import ProductItem
+
+logger = logging.getLogger(__name__)
+
+
+UserInputs: TypeAlias = Dict[str, List[str]]
+
+
+class ClassificationResult(BaseModel):
+    """Model for classification results."""
+
+    result: int
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+class TmpResult(BaseModel):
+    """Model for tmp results."""
+
+    result: Any
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+WorkflowResult: TypeAlias = ClassificationResult | TmpResult | None
+
+
+class Workflow(ABC):
+    """Abstract base class for independent processing workflows."""
+
+    def __init__(
+        self,
+        name: str,
+    ):
+        """Abstract base class for defining a classification workflow.
+
+        Args:
+            name: Name of the classification workflow.
+        """
+        self.name = name
+
+    @abstractmethod
+    async def run(self, product: ProductItem) -> WorkflowResult:
+        """Runs the workflow."""
+        pass
+
+
+class Processor:
+    """Processing product items for a set of classification workflows."""
+
+    def __init__(self, workflows: Sequence[Workflow]):
+        """Initializes the Processor.
+
+        Args:
+            workflows: Sequence of workflows for classification of product items.
+        """
+        if not self._are_unique(workflows=workflows):
+            raise ValueError(
+                f"Workflow names are not unique: {[wf.name for wf in workflows]}"
+            )
+        self._workflows = workflows
+
+    @staticmethod
+    def _are_unique(workflows: Sequence[Workflow]) -> bool:
+        """Tests if the workflows have unique names."""
+        return len(workflows) == len(set([wf.name for wf in workflows]))
+
+    async def run(self, product: ProductItem) -> ProductItem:
+        """Run the processing step for multiple workflows and return all results together with workflow.name.
+
+        Args:
+            product: The product item to process.
+        """
+        for wf in self._workflows:
+            try:
+                logger.info(
+                    f'Running workflow="{wf.name}" for product with url="{product.url_resolved}".'
+                )
+                res = await wf.run(product=product)
+            except Exception:
+                logger.error(
+                    f'Error while running workflow="{wf.name}" for product with url="{product.url_resolved}"',
+                    exc_info=True,
+                )
+                continue
+
+            # Update the product item
+            inp_tok = out_tok = 0
+            if isinstance(res, ClassificationResult):
+                logger.debug(
+                    f'result from workflow="{wf.name}" added to product.classifications'
+                )
+                product.classifications[wf.name] = int(res.result)
+                inp_tok = res.input_tokens
+                out_tok = res.output_tokens
+
+            elif isinstance(res, TmpResult):
+                logger.debug(f'result from workflow="{wf.name}" added to product.tmp')
+                product.tmp[wf.name] = res
+                inp_tok = res.input_tokens
+                out_tok = res.output_tokens
+
+            elif res is None:
+                logger.debug(
+                    f'result from workflow="{wf.name}" is `None` and therefore not stored'
+                )
+
+            else:
+                logger.warning(
+                    f'result from workflow="{wf.name}" return type={type(res)} is not allowed; '
+                    f"must either be of type `ClassificationResult`, "
+                    f"`TmpResult`, or `None`; not type={type(res)}"
+                )
+
+            if inp_tok > 0 or out_tok > 0:
+                logger.debug(
+                    f'result from workflow="{wf.name}" used input_tokens={inp_tok}, output_tokens={out_tok}'
+                )
+                product.usage[wf.name] = {
+                    "input_tokens": inp_tok,
+                    "output_tokens": out_tok,
+                }
+
+        return product