fraudcrawler 0.3.3__py3-none-any.whl

fraudcrawler/__init__.py

@@ -0,0 +1,30 @@
+from fraudcrawler.scraping.serp import SerpApi
+from fraudcrawler.scraping.enrich import Enricher
+from fraudcrawler.scraping.zyte import ZyteApi
+from fraudcrawler.processing.processor import Processor
+from fraudcrawler.base.orchestrator import Orchestrator, ProductItem
+from fraudcrawler.base.client import FraudCrawlerClient
+from fraudcrawler.base.base import (
+    Deepness,
+    Enrichment,
+    Host,
+    Language,
+    Location,
+    Prompt,
+)
+
+__all__ = [
+    "SerpApi",
+    "Enricher",
+    "ZyteApi",
+    "Processor",
+    "Orchestrator",
+    "ProductItem",
+    "FraudCrawlerClient",
+    "Language",
+    "Location",
+    "Host",
+    "Deepness",
+    "Enrichment",
+    "Prompt",
+]

fraudcrawler/base/base.py

@@ -0,0 +1,145 @@
+import json
+import logging
+from pydantic import BaseModel, field_validator, model_validator
+from pydantic_settings import BaseSettings
+from typing import List
+
+import aiohttp
+
+from fraudcrawler.settings import (
+    GOOGLE_LANGUAGES_FILENAME,
+    GOOGLE_LOCATIONS_FILENAME,
+    PROCESSOR_DEFAULT_IF_MISSING,
+)
+
+logger = logging.getLogger(__name__)
+
+# Load google locations and languages
+with open(GOOGLE_LOCATIONS_FILENAME, "r") as gfile:
+    _locs = json.load(gfile)
+_LOCATION_CODES = {loc["name"]: loc["country_code"].lower() for loc in _locs}
+with open(GOOGLE_LANGUAGES_FILENAME, "r") as gfile:
+    _langs = json.load(gfile)
+_LANGUAGE_CODES = {lang["language_name"]: lang["language_code"] for lang in _langs}
+
+
+# Base classes
+class Setup(BaseSettings):
+    """Class for loading environment variables."""
+
+    # Crawler ENV variables
+    serpapi_key: str
+    dataforseo_user: str
+    dataforseo_pwd: str
+    zyteapi_key: str
+    openaiapi_key: str
+
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+
+
+class Host(BaseModel):
+    """Model for host details (e.g. `Host(name="Galaxus", domains="galaxus.ch, digitec.ch")`)."""
+
+    name: str
+    domains: str | List[str]
+
+    @field_validator("domains", mode="before")
+    def split_domains_if_str(cls, val):
+        if isinstance(val, str):
+            return [dom.strip() for dom in val.split(",")]
+        return val
+
+
+class Location(BaseModel):
+    """Model for location details (e.g. `Location(name="Switzerland", code="ch")`)."""
+
+    name: str
+    code: str = ""
+
+    @model_validator(mode="before")
+    def set_code(cls, values):
+        """Set the location code if not provided and make it lower case."""
+        name = values.get("name")
+        code = values.get("code")
+        if code is None or not len(code):
+            code = _LOCATION_CODES.get(name)
+            if code is None:
+                raise ValueError(f'Location code not found for location name="{name}"')
+        code = code.lower()
+        return {"name": name, "code": code}
+
+
+class Language(BaseModel):
+    """Model for language details (e.g. `Language(name="German", code="de")`)."""
+
+    name: str
+    code: str = ""
+
+    @model_validator(mode="before")
+    def set_code(cls, values):
+        """Set the language code if not provided and make it lower case."""
+        name = values.get("name")
+        code = values.get("code")
+        if code is None or not len(code):
+            code = _LANGUAGE_CODES.get(name)
+            if code is None:
+                raise ValueError(f'Language code not found for language name="{name}"')
+        code = code.lower()
+        return {"name": name, "code": code}
+
+
+class Enrichment(BaseModel):
+    """Model for enriching initial search_term with alternative ones."""
+
+    additional_terms: int
+    additional_urls_per_term: int
+
+
+class Deepness(BaseModel):
+    """Model for search depth."""
+
+    num_results: int
+    enrichment: Enrichment | None = None
+
+
+class Prompt(BaseModel):
+    """Model for prompts."""
+
+    name: str
+    context: str
+    system_prompt: str
+    allowed_classes: List[int]
+    default_if_missing: int = PROCESSOR_DEFAULT_IF_MISSING
+
+
+class AsyncClient:
+    """Base class for sub-classes using async HTTP requests."""
+
+    @staticmethod
+    async def get(
+        url: str,
+        headers: dict | None = None,
+        params: dict | None = None,
+    ) -> dict:
+        """Async GET request of a given URL returning the data."""
+        async with aiohttp.ClientSession(headers=headers) as session:
+            async with session.get(url=url, params=params) as response:
+                response.raise_for_status()
+                json_ = await response.json()
+        return json_
+
+    @staticmethod
+    async def post(
+        url: str,
+        headers: dict | None = None,
+        data: List[dict] | dict | None = None,
+        auth: aiohttp.BasicAuth | None = None,
+    ) -> dict:
+        """Async POST request of a given URL returning the data."""
+        async with aiohttp.ClientSession(headers=headers) as session:
+            async with session.post(url=url, json=data, auth=auth) as response:
+                response.raise_for_status()
+                json_ = await response.json()
+        return json_

fraudcrawler/base/client.py

@@ -0,0 +1,134 @@
+import asyncio
+import csv
+from datetime import datetime
+import logging
+from pathlib import Path
+from pydantic import BaseModel
+from typing import List
+
+import pandas as pd
+
+from fraudcrawler.settings import ROOT_DIR
+from fraudcrawler.base.base import Setup, Language, Location, Deepness, Host, Prompt
+from fraudcrawler.base.orchestrator import Orchestrator, ProductItem
+
+logger = logging.getLogger(__name__)
+
+_RESULTS_DIR = ROOT_DIR / "data" / "results"
+
+
+class Results(BaseModel):
+    """The results of the product search."""
+
+    search_term: str
+    filename: Path | None = None
+
+
+class FraudCrawlerClient(Orchestrator):
+    """The main client for FraudCrawler."""
+
+    _filename_template = "{search_term}_{language}_{location}_{timestamp}.csv"
+
+    def __init__(self):
+        setup = Setup()
+        super().__init__(
+            serpapi_key=setup.serpapi_key,
+            dataforseo_user=setup.dataforseo_user,
+            dataforseo_pwd=setup.dataforseo_pwd,
+            zyteapi_key=setup.zyteapi_key,
+            openaiapi_key=setup.openaiapi_key,
+        )
+
+        self._results_dir = _RESULTS_DIR
+        if not self._results_dir.exists():
+            self._results_dir.mkdir(parents=True)
+        self._results: List[Results] = []
+
+    async def _collect_results(
+        self, queue_in: asyncio.Queue[ProductItem | None]
+    ) -> None:
+        """Collects the results from the given queue_in and saves it as csv.
+
+        Args:
+            queue_in: The input queue containing the results.
+        """
+        products = []
+        while True:
+            product = await queue_in.get()
+            if product is None:
+                queue_in.task_done()
+                break
+
+            products.append(product.model_dump())
+            queue_in.task_done()
+
+        # Convert the list of products to a DataFrame
+        df = pd.json_normalize(products)
+        cols = [c.split(".")[-1] for c in df.columns]
+        if len(cols) != len(set(cols)):
+            logger.error("Duplicate columns after json_normalize.")
+        else:
+            df.columns = cols
+
+        # Save the DataFrame to a CSV file
+        filename = self._results[-1].filename
+        df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
+        logger.info(f"Results saved to {filename}")
+
+    def execute(
+        self,
+        search_term: str,
+        language: Language,
+        location: Location,
+        deepness: Deepness,
+        prompts: List[Prompt],
+        marketplaces: List[Host] | None = None,
+        excluded_urls: List[Host] | None = None,
+    ) -> None:
+        """Runs the pipeline steps: serp, enrich, zyte, process, and collect the results.
+
+        Args:
+            search_term: The search term for the query.
+            language: The language to use for the query.
+            location: The location to use for the query.
+            deepness: The search depth and enrichment details.
+            prompts: The list of prompts to use for classification.
+            marketplaces: The marketplaces to include in the search.
+            excluded_urls: The URLs to exclude from the search.
+        """
+        timestamp = datetime.today().strftime("%Y%m%d%H%M%S")
+        filename = self._results_dir / self._filename_template.format(
+            search_term=search_term,
+            language=language.code,
+            location=location.code,
+            timestamp=timestamp,
+        )
+        self._results.append(Results(search_term=search_term, filename=filename))
+
+        asyncio.run(
+            super().run(
+                search_term=search_term,
+                language=language,
+                location=location,
+                deepness=deepness,
+                prompts=prompts,
+                marketplaces=marketplaces,
+                excluded_urls=excluded_urls,
+            )
+        )
+
+    def load_results(self, index: int = -1) -> pd.DataFrame:
+        """Loads the results from the saved .csv files.
+
+        Args:
+            index: The index of the results to load (`incex=-1` are the results for the most recent run).
+        """
+
+        results = self._results[index]
+        return pd.read_csv(results.filename)
+
+    def print_available_results(self) -> None:
+        """Prints the available results."""
+        n_res = len(self._results)
+        for i, res in enumerate(self._results):
+            print(f"index={-n_res + i}: {res.search_term} - {res.filename}")