fraudcrawler 0.5.0__py3-none-any.whl → 0.7.26__py3-none-any.whl

fraudcrawler/scraping/search.py

@@ -6,12 +6,18 @@ from typing import Dict, List
 from urllib.parse import quote_plus
 
 from bs4 import BeautifulSoup
+from bs4.element import Tag
 import httpx
 from tenacity import RetryCallState
 
-from fraudcrawler.settings import SEARCH_DEFAULT_COUNTRY_CODES
+from fraudcrawler.settings import (
+    SEARCH_DEFAULT_COUNTRY_CODES,
+    TOPPREISE_SEARCH_PATHS,
+    TOPPREISE_COMPARISON_PATHS,
+)
 from fraudcrawler.base.base import Host, Language, Location, DomainUtils
 from fraudcrawler.base.retry import get_async_retry
+from fraudcrawler.scraping.zyte import ZyteAPI
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +45,14 @@ class SearchEngine(ABC, DomainUtils):
 
     _hostname_pattern = r"^(?:https?:\/\/)?([^\/:?#]+)"
 
+    def __init__(self, http_client: httpx.AsyncClient):
+        """Initializes the SearchEngine with the given HTTP client.
+
+        Args:
+            http_client: An httpx.AsyncClient to use for the async requests.
+        """
+        self._http_client = http_client
+
     @property
     @abstractmethod
     def _search_engine_name(self) -> str:
@@ -50,45 +64,81 @@ class SearchEngine(ABC, DomainUtils):
         """Apply the search with the given parameters and return results."""
         pass
 
+    def _create_search_result(self, url: str) -> SearchResult:
+        """From a given url it creates the class:`SearchResult` instance."""
+        # Get marketplace name
+        domain = self._get_domain(url=url)
+
+        # Create and return the SearchResult object
+        result = SearchResult(
+            url=url,
+            domain=domain,
+            search_engine_name=self._search_engine_name,
+        )
+        return result
+
     @classmethod
     def _log_before(
-        cls, search_string: str, retry_state: RetryCallState | None
+        cls, url: str, params: dict | None, retry_state: RetryCallState | None
     ) -> None:
-        """Context aware logging before the request is made."""
+        """Context aware logging before HTTP request is made."""
         if retry_state:
             logger.debug(
-                f'Performing search in {cls.__name__} with q="{search_string}" '
-                f"(attempt {retry_state.attempt_number})."
+                f'Performing HTTP request in {cls.__name__} to url="{url}" '
+                f"with params={params} (attempt {retry_state.attempt_number})."
             )
         else:
             logger.debug(f"retry_state is {retry_state}; not logging before.")
 
     @classmethod
     def _log_before_sleep(
-        cls, search_string: str, retry_state: RetryCallState | None
+        cls, url: str, params: dict | None, retry_state: RetryCallState | None
     ) -> None:
-        """Context aware logging before sleeping after a failed request."""
+        """Context aware logging before sleeping after a failed HTTP request."""
         if retry_state and retry_state.outcome:
             logger.warning(
-                f'Attempt {retry_state.attempt_number} of {cls.__name__} search with q="{search_string}" '
+                f"Attempt {retry_state.attempt_number} of {cls.__name__} HTTP request "
+                f'to url="{url}" with params="{params}" '
                 f"failed with error: {retry_state.outcome.exception()}. "
                 f"Retrying in {retry_state.upcoming_sleep:.0f} seconds."
             )
         else:
             logger.debug(f"retry_state is {retry_state}; not logging before_sleep.")
 
-    def _create_search_result(self, url: str) -> SearchResult:
-        """From a given url it creates the class:`SearchResult` instance."""
-        # Get marketplace name
-        domain = self._get_domain(url=url)
+    async def http_client_get(
+        self, url: str, params: dict | None = None, headers: dict | None = None
+    ) -> httpx.Response:
+        """Performs a GET request with retries.
 
-        # Create and return the SearchResult object
-        result = SearchResult(
-            url=url,
-            domain=domain,
-            search_engine_name=self._search_engine_name,
+        Args:
+            retry: The retry strategy to use.
+            url: The URL to request.
+            params: Query parameters for the request.
+            headers: HTTP headers to use for the request.
+        """
+        # Perform the request and retry if necessary. There is some context aware logging:
+        #  - `before`: before the request is made (and before retrying)
+        #  - `before_sleep`: if the request fails before sleeping
+        retry = get_async_retry()
+        retry.before = lambda retry_state: self._log_before(
+            url=url, params=params, retry_state=retry_state
+        )
+        retry.before_sleep = lambda retry_state: self._log_before_sleep(
+            url=url, params=params, retry_state=retry_state
         )
-        return result
+
+        async for attempt in retry:
+            with attempt:
+                response = await self._http_client.get(
+                    url=url,
+                    params=params,
+                    headers=headers,
+                )
+                response.raise_for_status()
+                return response
+
+        # In case of not entering the for loop (for some strange reason)
+        raise RuntimeError("Retry exhausted without success")
 
 
 class SerpAPI(SearchEngine):
@@ -103,7 +153,7 @@ class SerpAPI(SearchEngine):
             http_client: An httpx.AsyncClient to use for the async requests.
             api_key: The API key for SerpAPI.
         """
-        self._http_client = http_client
+        super().__init__(http_client=http_client)
         self._api_key = api_key
 
     @property
@@ -131,6 +181,17 @@ class SerpAPI(SearchEngine):
             search_string += " site:" + " OR site:".join(s for s in sites)
         return search_string
 
+    @staticmethod
+    def _get_google_domain(location: Location) -> str:
+        """Gets the Google domain for the given location if they do not use the default pattern google.tld"""
+        if location.name == "Brazil":
+            return "google.com.br"
+        elif location.name == "United Kingdom":
+            return "google.co.uk"
+        elif location.name == "Argentina":
+            return "google.com.ar"
+        return f"google.{location.code}"
+
     async def _search(
         self,
         search_string: str,
@@ -169,38 +230,29 @@ class SerpAPI(SearchEngine):
             f"num_results={num_results}."
         )
 
-        # Setup the parameters
+        # Get Google domain and country code
+        google_domain = self._get_google_domain(location)
+        country_code = location.code
+
         params: Dict[str, str | int] = {
             "engine": engine,
             "q": search_string,
-            "google_domain": f"google.{location.code}",
+            "google_domain": google_domain,
             "location_requested": location.name,
             "location_used": location.name,
-            "tbs": f"ctr:{location.code.upper()}",
-            "cr": f"country{location.code.upper()}",
-            "gl": location.code,
+            "tbs": f"ctr:{country_code.upper()}",
+            "cr": f"country{country_code.upper()}",
+            "gl": country_code,
             "hl": language.code,
             "num": num_results,
             "api_key": self._api_key,
         }
         logger.debug(f"SerpAPI search with params: {params}")
 
-        # Perform the request and retry if necessary. There is some context aware logging:
-        #  - `before`: before the request is made (and before retrying)
-        #  - `before_sleep`: if the request fails before sleeping
-        retry = get_async_retry()
-        retry.before = lambda retry_state: self._log_before(
-            search_string=search_string, retry_state=retry_state
+        # Perform the search request
+        response: httpx.Response = await self.http_client_get(
+            url=self._endpoint, params=params
         )
-        retry.before_sleep = lambda retry_state: self._log_before_sleep(
-            search_string=search_string, retry_state=retry_state
-        )
-        async for attempt in retry:
-            with attempt:
-                response = await self._http_client.get(
-                    url=self._endpoint, params=params
-                )
-                response.raise_for_status()
 
         # Extract the URLs from the response
         data = response.json()
@@ -316,7 +368,21 @@ class SerpAPIGoogleShopping(SerpAPI):
         """
         results = data.get("shopping_results")
         if results is not None:
-            return [url for res in results if (url := res.get("product_link"))]
+            # return [url for res in results if (url := res.get("product_link"))]   # c.f. https://github.com/serpapi/public-roadmap/issues/3045
+            return [
+                url
+                for res in results
+                if (url := res.get("serpapi_immersive_product_api"))
+            ]
+        return []
+
+    @staticmethod
+    def _extract_product_urls_from_immersive_product_api(data: dict) -> List[str]:
+        """Extracts product urls from the serpapi immersive product API data."""
+        if results := data.get("product_results"):
+            stores = results.get("stores", [])
+            urls = [url for sre in stores if (url := sre.get("link"))]
+            return list(set(urls))
         return []
 
     async def search(
@@ -329,6 +395,9 @@ class SerpAPIGoogleShopping(SerpAPI):
     ) -> List[SearchResult]:
         """Performs a google shopping search using SerpApi and returns SearchResults.
 
+        Similar to Toppreise, this method extracts merchant URLs from Google Shopping product pages
+        and creates multiple SearchResult objects for each merchant URL found.
+
         Args:
             search_term: The search term to use for the query.
             language: The language to use for the query ('hl' parameter).
@@ -342,7 +411,7 @@ class SerpAPIGoogleShopping(SerpAPI):
             marketplaces=marketplaces,
         )
 
-        # Perform the search
+        # Perform the search to get Google Shopping URLs
         urls = await self._search(
             search_string=search_string,
             language=language,
@@ -355,10 +424,10 @@ class SerpAPIGoogleShopping(SerpAPI):
         # and Google Shopping searches (see https://github.com/serpapi/public-roadmap/issues/1858)
         urls = urls[:num_results]
 
-        # Create and return SearchResult objects from the URLs
+        # Create SearchResult objects from merchant URLs (similar to Toppreise pattern)
         results = [self._create_search_result(url=url) for url in urls]
         logger.debug(
-            f'Produced {len(results)} results from SerpAPI with engine="{self._engine}" and q="{search_string}".'
+            f'Produced {len(results)} results from Google Shopping search with q="{search_string}".'
         )
         return results
 
@@ -366,36 +435,77 @@ class SerpAPIGoogleShopping(SerpAPI):
 class Toppreise(SearchEngine):
     """Search engine for toppreise.ch."""
 
-    _endpoint = "https://www.toppreise.ch/produktsuche"
-    _headers = {
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.5",
-        "Accept-Encoding": "gzip, deflate",
-        "Connection": "keep-alive",
-        "Upgrade-Insecure-Requests": "1",
-    }
+    _endpoint = "https://www.toppreise.ch/"
 
-    def __init__(self, http_client: httpx.AsyncClient):
+    def __init__(self, http_client: httpx.AsyncClient, zyteapi_key: str):
         """Initializes the Toppreise client.
 
         Args:
             http_client: An httpx.AsyncClient to use for the async requests.
+            zyteapi_key: ZyteAPI key for fallback when direct access fails.
         """
-        self._http_client = http_client
+        super().__init__(http_client=http_client)
+        self._zyteapi = ZyteAPI(http_client=http_client, api_key=zyteapi_key)
 
-    @property
-    def _search_engine_name(self) -> str:
-        """The name of the search engine."""
-        return SearchEngineName.TOPPREISE.value
+    async def http_client_get_with_fallback(self, url: str) -> bytes:
+        """Performs a GET request with retries.
+
+        If direct access fails (e.g. 403 Forbidden), it will attempt to unblock the URL
+        content using Zyte proxy mode.
+
+        Args:
+            url: The URL to request.
+        """
+        # Try to access the URL directly
+        try:
+            response: httpx.Response = await self.http_client_get(
+                url=url, headers=self._headers
+            )
+            content = response.content
+
+        # If we get a 403 Error (can happen depending on IP/location of deployment),
+        # we try to unblock the URL using Zyte proxy mode
+        except httpx.HTTPStatusError as err_direct:
+            if err_direct.response.status_code == 403:
+                logger.warning(
+                    f"Received 403 Forbidden for {url}, attempting to unblock with Zyte proxy"
+                )
+                try:
+                    content = await self._zyteapi.unblock_url_content(url)
+                except Exception as err_resolve:
+                    msg = f'Error unblocking URL="{url}" with Zyte proxy: {err_resolve}'
+                    logger.error(msg, exc_info=True)
+                    raise httpx.HTTPError(msg) from err_resolve
+            else:
+                raise err_direct
+        return content
+
+    @classmethod
+    def _get_search_endpoint(cls, language: Language) -> str:
+        """Get the search endpoint based on the language."""
+        search_path = TOPPREISE_SEARCH_PATHS.get(
+            language.code, TOPPREISE_SEARCH_PATHS["default"]
+        )
+        return f"{cls._endpoint}{search_path}"
 
     @staticmethod
-    def _get_external_product_urls(content: bytes) -> List[str]:
-        """Extracts external product URLs from the Toppreise search results page."""
+    def _extract_links(
+        element: Tag, ext_products: bool = True, comp_products: bool = True
+    ) -> List[str]:
+        """Extracts all relevant product URLs from a BeautifulSoup object of a Toppreise page.
 
-        # Parse the HTML
-        soup = BeautifulSoup(content, "html.parser")
-        links = soup.find_all("a", href=True)
+        Note:
+            Depending on the arguments, it extracts:
+                - product comparison URLs (i.e. https://www.toppreise.ch/preisvergleich/...)
+                - external product URLs (i.e. https://www.example.com/ext_...).
+
+        Args:
+            tag: BeautifulSoup Tag object containing the HTML to parse.
+            ext_products: Whether to extract external product URLs.
+            comp_products: Whether to extract product comparison URLs.
+        """
+        # Find all links in the page
+        links = element.find_all("a", href=True)
 
         # Filter links to only include external product links
         hrefs = [
@@ -406,7 +516,15 @@ class Toppreise(SearchEngine):
                 and (href := link.get("href"))  # Ensure href is not None
                 and not href.startswith("javascript:")  # Skip javascript links
                 and isinstance(href, str)  # Ensure href is a string
-                and "ext_" in href  # Skip links that are not external product link
+                # Make sure the link is either an external product link (href contains 'ext_')
+                # or is a search result link (href contains 'preisvergleich', 'comparison-prix', or 'price-comparison')
+                and (
+                    ("ext_" in href and ext_products)
+                    or (
+                        any(pth in href for pth in TOPPREISE_COMPARISON_PATHS)
+                        and comp_products
+                    )
+                )
             )
         ]
 
@@ -421,44 +539,67 @@ class Toppreise(SearchEngine):
 
         # Return deduplicated urls
         urls = list(set(urls))
+        return urls
+
+    def _extract_product_urls_from_search_page(self, content: bytes) -> List[str]:
+        """Extracts product urls from a Toppreise search page (i.e. https://www.toppreise.ch/produktsuche)."""
+
+        # Parse the HTML
+        soup = BeautifulSoup(content, "html.parser")
+        main = soup.find("div", id="Page_Browsing")
+        if not isinstance(main, Tag):
+            logger.warning("No main content found in Toppreise search page.")
+            return []
+
+        # Extract links (external product links and comparison links)
+        urls = self._extract_links(element=main)
+
+        logger.debug(f"Found {len(urls)} product URLs from Toppreise search results.")
+        return urls
+
+    def _extract_product_urls_from_comparison_page(self, content: bytes) -> List[str]:
+        """Extracts product urls from a Toppreise product comparison page (i.e. https://www.toppreise.ch/preisvergleich/...)."""
+
+        # Parse the HTML
+        soup = BeautifulSoup(content, "html.parser")
+
+        # Extract links (external product links only)
+        urls = self._extract_links(element=soup, comp_products=False)
+
         logger.debug(
-            f"Found {len(urls)} external product URLs from Toppreise search results."
+            f"Found {len(urls)} external product URLs from Toppreise comparison page."
         )
         return urls
 
-    async def _search(self, search_string: str, num_results: int) -> List[str]:
+    @property
+    def _search_engine_name(self) -> str:
+        """The name of the search engine."""
+        return SearchEngineName.TOPPREISE.value
+
+    async def _search(
+        self, search_string: str, language: Language, num_results: int
+    ) -> List[str]:
         """Performs a search on Toppreise and returns the URLs of the results.
 
+        If direct access fails (e.g. 403 Forbidden), it will attempt to unblock the URL
+        content using Zyte proxy mode.
+
         Args:
             search_string: The search string to use for the query.
+            language: The language to use for the query.
             num_results: Max number of results to return.
         """
         # Build the search URL for Toppreise
+        endpoint = self._get_search_endpoint(language=language)
         encoded_search = quote_plus(search_string)
-        url = f"{self._endpoint}?q={encoded_search}"
+        url = f"{endpoint}?q={encoded_search}"
         logger.debug(f"Toppreise search URL: {url}")
 
-        # Perform the request and retry if necessary. There is some context aware logging:
-        #  - `before`: before the request is made (and before retrying)
-        #  - `before_sleep`: if the request fails before sleeping
-        retry = get_async_retry()
-        retry.before = lambda retry_state: self._log_before(
-            search_string=search_string, retry_state=retry_state
-        )
-        retry.before_sleep = lambda retry_state: self._log_before_sleep(
-            search_string=search_string, retry_state=retry_state
-        )
-        async for attempt in retry:
-            with attempt:
-                response = await self._http_client.get(
-                    url=url,
-                    headers=self._headers,
-                )
-                response.raise_for_status()
+        # Perform the request with fallback if necessary
+        content = await self.http_client_get_with_fallback(url=url)
 
         # Get external product urls from the content
-        content = response.content
-        urls = self._get_external_product_urls(content=content)
+        urls = self._extract_product_urls_from_search_page(content=content)
         urls = urls[:num_results]  # Limit to num_results if needed
 
         return urls
@@ -466,17 +607,20 @@ class Toppreise(SearchEngine):
     async def search(
         self,
         search_term: str,
+        language: Language,
         num_results: int,
     ) -> List[SearchResult]:
         """Performs a Toppreise search and returns SearchResults.
 
         Args:
             search_term: The search term to use for the query.
+            language: The language to use for the search.
             num_results: Max number of results to return.
         """
         # Perform the search
         urls = await self._search(
             search_string=search_term,
+            language=language,
             num_results=num_results,
         )
 
@@ -488,21 +632,121 @@ class Toppreise(SearchEngine):
         return results
 
 
-class Search(DomainUtils):
+class Searcher(DomainUtils):
     """Class to perform searches using different search engines."""
 
-    def __init__(self, http_client: httpx.AsyncClient, serpapi_key: str):
+    _post_search_retry_stop_after = 3
+
+    def __init__(
+        self, http_client: httpx.AsyncClient, serpapi_key: str, zyteapi_key: str
+    ):
         """Initializes the Search class with the given SerpAPI key.
 
         Args:
             http_client: An httpx.AsyncClient to use for the async requests.
             serpapi_key: The API key for SERP API.
+            zyteapi_key: ZyteAPI key for fallback when direct access fails.
         """
+        self._http_client = http_client
         self._google = SerpAPIGoogle(http_client=http_client, api_key=serpapi_key)
         self._google_shopping = SerpAPIGoogleShopping(
-            http_client=http_client, api_key=serpapi_key
+            http_client=http_client,
+            api_key=serpapi_key,
+        )
+        self._toppreise = Toppreise(
+            http_client=http_client,
+            zyteapi_key=zyteapi_key,
+        )
+
+    async def _post_search_google_shopping_immersive(self, url: str) -> List[str]:
+        """Post-search for product URLs from a Google Shopping immersive product page.
+
+        Args:
+            url: The URL of the Google Shopping product page.
+        """
+        # Add SerpAPI key to the url
+        sep = "&" if "?" in url else "?"
+        url = f"{url}{sep}api_key={self._google_shopping._api_key}"
+
+        # Fetch the content of the Google Shopping product page
+        response = await self._google_shopping.http_client_get(url=url)
+
+        # Get external product urls from the data
+        data = response.json()
+        urls = self._google_shopping._extract_product_urls_from_immersive_product_api(
+            data=data
+        )
+        return urls
+
+    async def _post_search_toppreise_comparison(self, url: str) -> List[str]:
+        """Post-search for product URLs from a Toppreise product comparison page.
+
+        Note:
+            In comparison to the function Toppreise._search, here we extract the urls from
+            product comparison pages (f.e. https://www.toppreise.ch/preisvergleich/). These
+            pages can also be found in the results of a google search.
+
+        Args:
+            url: The URL of the Toppreise product listing page.
+        """
+        # Perform the request with fallback if necessary
+        content = await self._toppreise.http_client_get_with_fallback(url=url)
+
+        # Get external product urls from the content
+        urls = self._toppreise._extract_product_urls_from_comparison_page(
+            content=content
         )
-        self._toppreise = Toppreise(http_client=http_client)
+        return urls
+
+    async def _post_search(self, results: List[SearchResult]) -> List[SearchResult]:
+        """Post-search for additional embedded product URLs from the obtained results.
+
+        Note:
+            This function is used to extract embedded product URLs from
+            product listing pages (e.g. Toppreise, Google Shopping) if needed.
+
+        Args:
+            results: The list of SearchResult objects obtained from the search.
+        """
+        post_search_results: List[SearchResult] = []
+        for res in results:
+            url = res.url
+            post_search_urls: List[str] = []
+
+            # Extract embedded product URLs from the Google Shopping immersive product page
+            if "engine=google_immersive_product" in url:
+                logger.debug(
+                    f'Extracting embedded product URLs from url="{url}" found by search_engine="{res.search_engine_name}"'
+                )
+                post_search_urls = await self._post_search_google_shopping_immersive(
+                    url=url
+                )
+                logger.debug(
+                    f'Extracted {len(post_search_urls)} embedded product URLs from url="{url}".'
+                )
+
+            # Extract embedded product URLs from the Toppreise product listing page
+            elif any(pth in url for pth in TOPPREISE_COMPARISON_PATHS):
+                logger.debug(
+                    f'Extracting embedded product URLs from url="{url}" found by search_engine="{res.search_engine_name}"'
+                )
+                post_search_urls = await self._post_search_toppreise_comparison(url=url)
+                logger.debug(
+                    f'Extracted {len(post_search_urls)} embedded product URLs from url="{url}".'
+                )
+
+            # Add the extracted product URLs as SearchResult objects
+            psr = [
+                SearchResult(
+                    url=psu,
+                    domain=self._get_domain(url=psu),
+                    search_engine_name=res.search_engine_name,
+                )
+                for psu in post_search_urls
+            ]
+            post_search_results.extend(psr)
+
+        return post_search_results
 
     @staticmethod
     def _domain_in_host(domain: str, host: Host) -> bool:
@@ -592,63 +836,77 @@ class Search(DomainUtils):
     async def apply(
         self,
         search_term: str,
+        search_engine: SearchEngineName | str,
         language: Language,
         location: Location,
         num_results: int,
         marketplaces: List[Host] | None = None,
         excluded_urls: List[Host] | None = None,
-        search_engines: List[SearchEngineName | str] | None = None,
     ) -> List[SearchResult]:
         """Performs a search and returns SearchResults.
 
         Args:
             search_term: The search term to use for the query.
+            search_engine: The search engine to use for the search.
             language: The language to use for the query ('hl' parameter).
             location: The location to use for the query ('gl' parameter).
             num_results: Max number of results per search engine.
             marketplaces: The marketplaces to include in the search.
             excluded_urls: The URLs to exclude from the search.
-            search_engines: The list of search engines to use for the search.
         """
-        if search_engines is None:
-            search_engines = list(SearchEngineName)
-        else:
-            search_engines = [
-                SearchEngineName(sen) if isinstance(sen, str) else sen
-                for sen in search_engines
-            ]
-        results: List[SearchResult] = []
+        logger.info(
+            f'Performing search for term="{search_term}" using engine="{search_engine}".'
+        )
+
+        # -------------------------------
+        # SEARCH
+        # -------------------------------
+        # Map string to SearchEngineName if needed
+        if isinstance(search_engine, str):
+            search_engine = SearchEngineName(search_engine)
 
         # Make SerpAPI google search
-        if SearchEngineName.GOOGLE in search_engines:
-            res = await self._google.search(
+        if search_engine == SearchEngineName.GOOGLE:
+            results = await self._google.search(
                 search_term=search_term,
                 language=language,
                 location=location,
                 num_results=num_results,
                 marketplaces=marketplaces,
             )
-            results.extend(res)
 
         # Make SerpAPI google shopping search
-        if SearchEngineName.GOOGLE_SHOPPING in search_engines:
-            res = await self._google_shopping.search(
+        elif search_engine == SearchEngineName.GOOGLE_SHOPPING:
+            results = await self._google_shopping.search(
                 search_term=search_term,
                 language=language,
                 location=location,
                 num_results=num_results,
                 marketplaces=marketplaces,
             )
-            results.extend(res)
 
         # Make Toppreise search
-        if SearchEngineName.TOPPREISE in search_engines:
-            res = await self._toppreise.search(
+        elif search_engine == SearchEngineName.TOPPREISE:
+            results = await self._toppreise.search(
                 search_term=search_term,
+                language=language,
                 num_results=num_results,
             )
-            results.extend(res)
 
+        # Other search engines can be added here (raise unknown engine error otherwise)
+        else:
+            raise ValueError(f"Unknown search engine: {search_engine}")
+
+        # -------------------------------
+        # POST-SEARCH URL EXTRACTION
+        # -------------------------------
+        post_search_results = await self._post_search(results=results)
+        post_search_results = post_search_results[:num_results]
+        results.extend(post_search_results)
+
+        # -------------------------------
+        # FILTERS
+        # -------------------------------
         # Apply filters
         results = [
             self._apply_filters(
@@ -660,5 +918,7 @@ class Search(DomainUtils):
             for res in results
         ]
 
-        logger.debug(f"Search produced a total of {len(results)} results.")
+        logger.info(
+            f'Search for term="{search_term}" using engine="{search_engine}" produced {len(results)} results.'
+        )
         return results