cs2tracker 2.1.12__py3-none-any.whl → 2.1.14__py3-none-any.whl

cs2tracker/scraper/scraper.py

@@ -1,44 +1,63 @@
 import time
 from datetime import datetime
-from urllib.parse import unquote
 
-from bs4 import BeautifulSoup
-from bs4.element import Tag
 from currency_converter import CurrencyConverter
-from requests import RequestException, Session
+from requests import RequestException
 from requests.adapters import HTTPAdapter, Retry
+from requests_cache import CachedSession
 from tenacity import RetryError, retry, stop_after_attempt
 
-from cs2tracker.constants import AUTHOR_STRING, BANNER, CAPSULE_INFO, CASE_HREFS
+from cs2tracker.constants import AUTHOR_STRING, BANNER
 from cs2tracker.scraper.discord_notifier import DiscordNotifier
-from cs2tracker.util import PaddedConsole, PriceLogs, ValidatedConfig
-
-MAX_LINE_LEN = 72
-SEPARATOR = "-"
-PRICE_INFO = "Owned: {:<10}  Steam market price: ${:<10}  Total: ${:<10}\n"
+from cs2tracker.scraper.parsers import CSGOTrader, PriceSource
+from cs2tracker.util import PriceLogs, get_config, get_console
 
 HTTP_PROXY_URL = "http://{}:@smartproxy.crawlbase.com:8012"
 HTTPS_PROXY_URL = "http://{}:@smartproxy.crawlbase.com:8012"
 
-console = PaddedConsole()
+console = get_console()
+config = get_config()
+
+
+class ConfigError:
+    def __init__(self):
+        self.message = "Invalid configuration. Please fix the config file before running."
+
+
+class ParsingError:
+    def __init__(self, message):
+        self.message = message
+
+
+class RequestLimitExceededError:
+    def __init__(self):
+        self.message = "Too many requests. Consider using proxies to prevent rate limiting."
+
+
+class PageLoadError:
+    def __init__(self, status_code):
+        self.message = f"Failed to load page: {status_code}. Retrying..."
+
+
+class UnexpectedError:
+    def __init__(self, error):
+        self.message = f"An unexpected error occurred: {error}"
 
 
 class Scraper:
     def __init__(self):
         """Initialize the Scraper class."""
-        self.load_config()
         self._start_session()
+        self._add_parser(CSGOTrader)
 
-        self.usd_total = 0
-        self.eur_total = 0
-
-    def load_config(self):
-        """Load the configuration file and validate its contents."""
-        self.config = ValidatedConfig()
+        self.error_stack = []
+        self.totals = {
+            price_source: {"usd": 0.0, "eur": 0.0} for price_source in self.parser.SOURCES
+        }
 
     def _start_session(self):
         """Start a requests session with custom headers and retry logic."""
-        self.session = Session()
+        self.session = CachedSession("scraper_cache", backend="memory")
         self.session.headers.update(
             {
                 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
@@ -48,6 +67,16 @@ class Scraper:
         self.session.mount("http://", HTTPAdapter(max_retries=retries))
         self.session.mount("https://", HTTPAdapter(max_retries=retries))
 
+    def _add_parser(self, parser):
+        """Add a parser for a specific page where item prices should be scraped."""
+        self.parser = parser
+
+    def _print_error(self):
+        """Print the last error message from the error stack, if any."""
+        last_error = self.error_stack[-1] if self.error_stack else None
+        if last_error:
+            console.error(f"{last_error.message}")
+
     def scrape_prices(self, update_sheet_callback=None):
         """
         Scrape prices for capsules and cases, calculate totals in USD and EUR, and
@@ -56,62 +85,73 @@ class Scraper:
         :param update_sheet_callback: Optional callback function to update a tksheet
             that is displayed in the GUI with the latest scraper price calculation.
         """
-        if not self.config.valid:
-            console.print(
-                "[bold red][!] Invalid configuration. Please fix the config file before running."
-            )
+        if not config.valid:
+            self.error_stack.append(ConfigError())
+            self._print_error()
             return
 
-        capsule_usd_total = self._scrape_capsule_section_prices(update_sheet_callback)
-        case_usd_total = self._scrape_case_prices(update_sheet_callback)
-        custom_item_usd_total = self._scrape_custom_item_prices(update_sheet_callback)
+        # Reset totals from the previous run and clear the error stack
+        self.error_stack.clear()
+        self.totals = {
+            price_source: {"usd": 0.0, "eur": 0.0} for price_source in self.parser.SOURCES
+        }
 
-        self.usd_total += capsule_usd_total
-        self.usd_total += case_usd_total
-        self.usd_total += custom_item_usd_total
-        self.eur_total = CurrencyConverter().convert(self.usd_total, "USD", "EUR")
+        for section in config.sections():
+            if section in ("User Settings", "App Settings"):
+                continue
+            self._scrape_item_prices(section, update_sheet_callback)
+
+        for price_source, totals in self.totals.items():
+            usd_total = totals["usd"]
+            eur_total = CurrencyConverter().convert(usd_total, "USD", "EUR")
+            self.totals.update({price_source: {"usd": usd_total, "eur": eur_total}})  # type: ignore
 
         if update_sheet_callback:
-            update_sheet_callback(["", "", "", ""])
-            update_sheet_callback(
-                [
-                    f"[{datetime.now().strftime('%Y-%m-%d')}] Total:",
-                    f"${self.usd_total:.2f}",
-                    f"€{self.eur_total:.2f}",
-                    "",
-                ]
-            )
+            update_sheet_callback(["", ""] + ["", ""] * len(self.parser.SOURCES))
+            for price_source, totals in self.totals.items():
+                usd_total = totals["usd"]
+                eur_total = totals["eur"]
+                update_sheet_callback(
+                    [
+                        f"[{datetime.now().strftime('%Y-%m-%d')}] {price_source.value.title()} Total:",
+                        f"${usd_total:.2f}",
+                        f"€{eur_total:.2f}",
+                        "",
+                    ]
+                )
 
         self._print_total()
-        PriceLogs.save(self.usd_total, self.eur_total)
         self._send_discord_notification()
 
-        # Reset totals for next run
-        self.usd_total, self.eur_total = 0, 0
+        # TODO: modify price logs, charts etc for multiple sources (only use steam as source for now)
+        steam_usd_total = self.totals[PriceSource.STEAM]["usd"]
+        steam_eur_total = self.totals[PriceSource.STEAM]["eur"]
+        PriceLogs.save(steam_usd_total, steam_eur_total)
 
     def _print_total(self):
         """Print the total prices in USD and EUR, formatted with titles and
         separators.
         """
-        usd_title = "USD Total".center(MAX_LINE_LEN, SEPARATOR)
-        console.print(f"[bold green]{usd_title}")
-        console.print(f"${self.usd_total:.2f}")
+        console.title("USD Total", "green")
+        for price_source, totals in self.totals.items():
+            usd_total = totals.get("usd")
+            console.print(f"{price_source.value.title():<10}: ${usd_total:.2f}")
 
-        eur_title = "EUR Total".center(MAX_LINE_LEN, SEPARATOR)
-        console.print(f"[bold green]{eur_title}")
-        console.print(f"€{self.eur_total:.2f}")
+        console.title("EUR Total", "green")
+        for price_source, totals in self.totals.items():
+            eur_total = totals.get("eur")
+            console.print(f"{price_source.value.title():<10}: €{eur_total:.2f}")
 
-        end_string = SEPARATOR * MAX_LINE_LEN
-        console.print(f"[bold green]{end_string}\n")
+        console.separator("green")
 
     def _send_discord_notification(self):
         """Send a message to a Discord webhook if notifications are enabled in the
         config file and a webhook URL is provided.
         """
-        discord_notifications = self.config.getboolean(
+        discord_notifications = config.getboolean(
             "App Settings", "discord_notifications", fallback=False
         )
-        webhook_url = self.config.get("User Settings", "discord_webhook_url", fallback=None)
+        webhook_url = config.get("User Settings", "discord_webhook_url", fallback=None)
 
         if discord_notifications and webhook_url:
             DiscordNotifier.notify(webhook_url)
@@ -127,8 +167,8 @@ class Scraper:
         :raises RequestException: If the request fails.
         :raises RetryError: If the retry limit is reached.
         """
-        use_proxy = self.config.getboolean("App Settings", "use_proxy", fallback=False)
-        proxy_api_key = self.config.get("User Settings", "proxy_api_key", fallback=None)
+        use_proxy = config.getboolean("App Settings", "use_proxy", fallback=False)
+        proxy_api_key = config.get("User Settings", "proxy_api_key", fallback=None)
 
         if use_proxy and proxy_api_key:
             page = self.session.get(
@@ -143,203 +183,85 @@ class Scraper:
             page = self.session.get(url)
 
         if not page.ok or not page.content:
-            console.print(f"[bold red][!] Failed to load page ({page.status_code}). Retrying...\n")
+            self.error_stack.append(PageLoadError(page.status_code))
+            self._print_error()
             raise RequestException(f"Failed to load page: {url}")
 
         return page
 
-    def _parse_item_price(self, item_page, item_href):
-        """
-        Parse the price of an item from the given steamcommunity market page and item
-        href.
-
-        :param item_page: The HTTP response object containing the item page content.
-        :param item_href: The href of the item listing to find the price for.
-        :return: The price of the item as a float.
-        :raises ValueError: If the item listing or price span cannot be found.
-        """
-        item_soup = BeautifulSoup(item_page.content, "html.parser")
-        item_listing = item_soup.find("a", attrs={"href": f"{item_href}"})
-        if not isinstance(item_listing, Tag):
-            raise ValueError(f"Failed to find item listing: {item_href}")
-
-        item_price_span = item_listing.find("span", attrs={"class": "normal_price"})
-        if not isinstance(item_price_span, Tag):
-            raise ValueError(f"Failed to find price span in item listing: {item_href}")
-
-        price_str = item_price_span.text.split()[2]
-        price = float(price_str.replace("$", ""))
-
-        return price
-
-    def _scrape_capsule_prices(self, capsule_section, capsule_info, update_sheet_callback=None):
-        """
-        Scrape prices for a specific capsule section, printing the details to the
-        console.
-
-        :param capsule_section: The section name in the config for the capsule.
-        :param capsule_info: A dictionary containing information about the capsule page,
-            hrefs, and names.
-        :param update_sheet_callback: Optional callback function to update a tksheet
-            that is displayed in the GUI with the latest scraper price calculation.
-        """
-        capsule_title = capsule_section.center(MAX_LINE_LEN, SEPARATOR)
-        console.print(f"[bold magenta]{capsule_title}\n")
-
-        capsule_usd_total = 0
-        try:
-            capsule_page = self._get_page(capsule_info["page"])
-            for capsule_name, capsule_href in zip(capsule_info["names"], capsule_info["items"]):
-                config_capsule_name = capsule_name.replace(" ", "_").lower()
-                owned = self.config.getint(capsule_section, config_capsule_name, fallback=0)
-                if owned == 0:
-                    continue
-
-                price_usd = self._parse_item_price(capsule_page, capsule_href)
-                price_usd_owned = round(float(owned * price_usd), 2)
-
-                console.print(f"[bold deep_sky_blue4]{capsule_name}")
-                console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
-                if update_sheet_callback:
-                    update_sheet_callback([capsule_name, owned, price_usd, price_usd_owned])
-                capsule_usd_total += price_usd_owned
-        except (RetryError, ValueError):
-            console.print(
-                "[bold red][!] Too many requests. (Consider using proxies to prevent rate limiting)\n"
-            )
-        except Exception as error:
-            console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
-
-        return capsule_usd_total
-
-    def _scrape_capsule_section_prices(self, update_sheet_callback=None):
+    def _scrape_prices_from_all_sources(self, item_href, owned):
         """
-        Scrape prices for all capsule sections defined in the configuration.
+        For a given item href and owned count, scrape the item's price from all sources
+        available to the currently registered parser.
 
-        :param update_sheet_callback: Optional callback function to update a tksheet
-            that is displayed in the GUI with the latest scraper price calculation.
-        """
-        capsule_usd_total = 0
-        for capsule_section, capsule_info in CAPSULE_INFO.items():
-            # Only scrape capsule sections where the user owns at least one item
-            if any(int(owned) > 0 for _, owned in self.config.items(capsule_section)):
-                capsule_usd_total += self._scrape_capsule_prices(
-                    capsule_section, capsule_info, update_sheet_callback
-                )
-
-        return capsule_usd_total
-
-    def _market_page_from_href(self, item_href):
-        """
-        Convert an href of a Steam Community Market item to a market page URL.
-
-        :param item_href: The href of the item listing, typically ending with the item's
-            name.
-        :return: A URL string for the Steam Community Market page of the item.
-        """
-        url_encoded_name = item_href.split("/")[-1]
-        page_url = f"https://steamcommunity.com/market/search?q={url_encoded_name}"
-
-        return page_url
-
-    def _scrape_case_prices(self, update_sheet_callback=None):
-        """
-        Scrape prices for all cases defined in the configuration.
-
-        For each case, it prints the case name, owned count, price per item, and total
-        price for owned items.
-
-        :param update_sheet_callback: Optional callback function to update a tksheet
-            that is displayed in the GUI with the latest scraper price calculation.
+        :param item_href: The url of the steamcommunity market listing of the item
+        :param owned: How many of this item the user owns
+        :return: A list of item prices for the different sources
+        :raises RequestException: If the request fails.
+        :raises RetryError: If the retry limit is reached.
+        :raises ValueError: If the parser could not find the item
         """
-        case_usd_total = 0
-        for case_index, (config_case_name, owned) in enumerate(self.config.items("Cases")):
-            if int(owned) == 0:
-                continue
-
-            case_name = config_case_name.replace("_", " ").title()
-            case_title = case_name.center(MAX_LINE_LEN, SEPARATOR)
-            console.print(f"[bold magenta]{case_title}\n")
-
+        prices = []
+        for price_source in self.parser.SOURCES:
             try:
-                case_page_url = self._market_page_from_href(CASE_HREFS[case_index])
-                case_page = self._get_page(case_page_url)
-                price_usd = self._parse_item_price(case_page, CASE_HREFS[case_index])
-                price_usd_owned = round(float(int(owned) * price_usd), 2)
-
-                console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
-                if update_sheet_callback:
-                    update_sheet_callback([case_name, owned, price_usd, price_usd_owned])
-                case_usd_total += price_usd_owned
+                item_page_url = self.parser.get_item_page_url(item_href, price_source)
+                item_page = self._get_page(item_page_url)
+                price_usd = self.parser.parse_item_price(item_page, item_href, price_source)
 
-                if not self.config.getboolean("App Settings", "use_proxy", fallback=False):
-                    time.sleep(1)
-            except (RetryError, ValueError):
-                console.print(
-                    "[bold red][!] Too many requests. (Consider using proxies to prevent rate limiting)\n"
+                price_usd_owned = round(float(int(owned) * price_usd), 2)
+                self.totals[price_source]["usd"] += price_usd_owned
+
+                prices += [price_usd, price_usd_owned]
+                console.price(
+                    self.parser.PRICE_INFO,
+                    owned,
+                    price_source.value.title(),
+                    price_usd,
+                    price_usd_owned,
                 )
-            except Exception as error:
-                console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
+            except ValueError as error:
+                prices += [0.0, 0.0]
+                self.error_stack.append(ParsingError(error))
+                self._print_error()
 
-        return case_usd_total
+        return prices
 
-    def _scrape_custom_item_prices(self, update_sheet_callback=None):
+    def _scrape_item_prices(self, section, update_sheet_callback=None):
         """
-        Scrape prices for custom items defined in the configuration.
+        Scrape prices for all items defined in a configuration section that uses hrefs
+        as option keys.
 
-        For each custom item, it prints the item name, owned count, price per item, and
-        total price for owned items.
+        For each item, it prints the item name, owned count, price per item, and total
+        price for owned items.
 
         :param update_sheet_callback: Optional callback function to update a tksheet
             that is displayed in the GUI with the latest scraper price calculation.
         """
-        custom_item_usd_total = 0
-        for custom_item_href, owned in self.config.items("Custom Items"):
+        for item_href, owned in config.items(section):
+            if self.error_stack and isinstance(self.error_stack[-1], RequestLimitExceededError):
+                break
             if int(owned) == 0:
                 continue
 
-            custom_item_name = unquote(custom_item_href.split("/")[-1])
-            custom_item_title = custom_item_name.center(MAX_LINE_LEN, SEPARATOR)
-            console.print(f"[bold magenta]{custom_item_title}\n")
-
+            item_name = config.option_to_name(item_href, href=True)
+            console.title(item_name, "magenta")
             try:
-                custom_item_page_url = self._market_page_from_href(custom_item_href)
-                custom_item_page = self._get_page(custom_item_page_url)
-                price_usd = self._parse_item_price(custom_item_page, custom_item_href)
-                price_usd_owned = round(float(int(owned) * price_usd), 2)
+                prices = self._scrape_prices_from_all_sources(item_href, owned)
 
-                console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
                 if update_sheet_callback:
-                    update_sheet_callback([custom_item_name, owned, price_usd, price_usd_owned])
-                custom_item_usd_total += price_usd_owned
+                    update_sheet_callback([item_name, owned] + prices)
 
-                if not self.config.getboolean("App Settings", "use_proxy", fallback=False):
+                if (
+                    not config.getboolean("App Settings", "use_proxy", fallback=False)
+                    and self.parser.NEEDS_TIMEOUT
+                ):
                     time.sleep(1)
-            except (RetryError, ValueError):
-                console.print(
-                    "[bold red][!] Too many requests. (Consider using proxies to prevent rate limiting)\n"
-                )
+            except RetryError:
+                self.error_stack.append(RequestLimitExceededError())
+                self._print_error()
             except Exception as error:
-                console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
-
-        return custom_item_usd_total
-
-    def toggle_use_proxy(self, enabled: bool):
-        """
-        Toggle the use of proxies for requests. This will update the configuration file.
-
-        :param enabled: If True, proxies will be used; if False, they will not be used.
-        """
-        self.config.toggle_use_proxy(enabled)
-
-    def toggle_discord_webhook(self, enabled: bool):
-        """
-        Toggle the use of a Discord webhook to notify users of price calculations.
-
-        :param enabled: If True, the webhook will be used; if False, it will not be
-            used.
-        """
-        self.config.toggle_discord_webhook(enabled)
+                self.error_stack.append(UnexpectedError(error))
+                self._print_error()
 
 
 if __name__ == "__main__":

cs2tracker/util/__init__.py

@@ -1,9 +1,9 @@
 from cs2tracker.util.padded_console import (  # noqa: F401 # pylint:disable=unused-import
-    PaddedConsole,
+    get_console,
 )
 from cs2tracker.util.price_logs import (  # noqa: F401 # pylint:disable=unused-import
     PriceLogs,
 )
 from cs2tracker.util.validated_config import (  # noqa: F401 # pylint:disable=unused-import
-    ValidatedConfig,
+    get_config,
 )

cs2tracker/util/padded_console.py

@@ -7,6 +7,10 @@ PADDING_LEFT = 4
 PADDING_RIGHT = 0
 
 
+MAX_LINE_LEN = 72
+SEPARATOR = "-"
+
+
 class PaddedConsole:
     def __init__(self, padding=(PADDING_TOP, PADDING_RIGHT, PADDING_BOTTOM, PADDING_LEFT)):
         """Initialize a PaddedConsole with specified padding."""
@@ -17,6 +21,34 @@ class PaddedConsole:
         """Print text with padding to the console."""
         self.console.print(Padding(text, self.padding))
 
+    def error(self, text):
+        """Print error text with padding to the console."""
+        text = "[bold red][!] " + text
+        self.print(text)
+
+    def title(self, text, color):
+        """Print the given text as a title."""
+        title = text.center(MAX_LINE_LEN, SEPARATOR)
+        console.print(f"\n[bold {color}]{title}\n")
+
+    def separator(self, color):
+        """Print a separator line."""
+        separator = SEPARATOR * MAX_LINE_LEN
+        console.print(f"[bold {color}]{separator}")
+
+    def price(self, price_str, price_source, owned, steam_market_price, total_owned):
+        # pylint: disable=too-many-arguments,too-many-positional-arguments
+        """Print price information."""
+        console.print(price_str.format(price_source, owned, steam_market_price, total_owned))
+
     def __getattr__(self, attr):
         """Ensure console methods can be called directly on PaddedConsole."""
         return getattr(self.console, attr)
+
+
+console = PaddedConsole()
+
+
+def get_console():
+    """Get the PaddedConsole instance."""
+    return console