cs2tracker 2.1.13__py3-none-any.whl → 2.1.15__py3-none-any.whl

cs2tracker/scraper/scraper.py

@@ -1,21 +1,18 @@
 import time
 from datetime import datetime
-from urllib.parse import unquote
 
-from bs4 import BeautifulSoup
-from bs4.element import Tag
-from currency_converter import CurrencyConverter
-from requests import RequestException, Session
+from requests import RequestException
 from requests.adapters import HTTPAdapter, Retry
+from requests_cache import CachedSession
 from tenacity import RetryError, retry, stop_after_attempt
 
-from cs2tracker.constants import AUTHOR_STRING, BANNER, CAPSULE_INFO, CASE_HREFS
+from cs2tracker.config import get_config
+from cs2tracker.constants import AUTHOR_STRING, BANNER
+from cs2tracker.logs import PriceLogs
 from cs2tracker.scraper.discord_notifier import DiscordNotifier
-from cs2tracker.util import PriceLogs, get_config, get_console
-
-MAX_LINE_LEN = 72
-SEPARATOR = "-"
-PRICE_INFO = "Owned: {:<10}  Steam market price: ${:<10}  Total: ${:<10}\n"
+from cs2tracker.scraper.parser import Parser
+from cs2tracker.util.currency_conversion import convert, to_symbol
+from cs2tracker.util.padded_console import get_console
 
 HTTP_PROXY_URL = "http://{}:@smartproxy.crawlbase.com:8012"
 HTTPS_PROXY_URL = "http://{}:@smartproxy.crawlbase.com:8012"
@@ -29,6 +26,11 @@ class ConfigError:
         self.message = "Invalid configuration. Please fix the config file before running."
 
 
+class ParsingError:
+    def __init__(self, message):
+        self.message = message
+
+
 class RequestLimitExceededError:
     def __init__(self):
         self.message = "Too many requests. Consider using proxies to prevent rate limiting."
@@ -44,17 +46,33 @@ class UnexpectedError:
         self.message = f"An unexpected error occurred: {error}"
 
 
+class SheetNotFoundError:
+    def __init__(self):
+        self.message = "Could not find sheet to update."
+
+
 class Scraper:
     def __init__(self):
         """Initialize the Scraper class."""
         self._start_session()
         self.error_stack = []
-        self.usd_total = 0
-        self.eur_total = 0
+
+        # We set the conversion currency as an attribute of the Scraper instance
+        # and only update it from the config at the start of the scraping process.
+        # This allows us to use the same conversion currency throughout the scraping
+        # process and prevents issues with changing the conversion currency while scraping.
+        self.conversion_currency = config.conversion_currency
+        self.totals = {
+            price_source: {
+                "USD": 0.0,
+                self.conversion_currency: 0.0,
+            }
+            for price_source in Parser.SOURCES
+        }
 
     def _start_session(self):
         """Start a requests session with custom headers and retry logic."""
-        self.session = Session()
+        self.session = CachedSession("scraper_cache", backend="memory")
         self.session.headers.update(
             {
                 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
@@ -64,79 +82,112 @@ class Scraper:
         self.session.mount("http://", HTTPAdapter(max_retries=retries))
         self.session.mount("https://", HTTPAdapter(max_retries=retries))
 
-    def _print_error(self):
-        """Print the last error message from the error stack, if any."""
-        last_error = self.error_stack[-1] if self.error_stack else None
-        if last_error:
-            console.error(f"{last_error.message}\n")
+    def _error(self, error):
+        """Add an error to the error stack and print the last error message from the
+        error stack.
+        """
+        self.error_stack.append(error)
+        console.error(f"{error.message}")
+
+    def _prepare_new_run(self):
+        """
+        Reset totals for the next run and get the most recent conversion currency from
+        the config.
+
+        This way, we don't have to create a new Scraper instance for each run.
+        """
+        self.error_stack.clear()
+        self.conversion_currency = config.conversion_currency
+        self.totals = {
+            price_source: {
+                "USD": 0.0,
+                self.conversion_currency: 0.0,
+            }
+            for price_source in Parser.SOURCES
+        }
 
     def scrape_prices(self, update_sheet_callback=None):
         """
-        Scrape prices for capsules and cases, calculate totals in USD and EUR, and
-        print/save the results.
+        Scrape prices for capsules and cases, calculate totals in USD and conversion
+        currency, and print/save the results.
 
         :param update_sheet_callback: Optional callback function to update a tksheet
             that is displayed in the GUI with the latest scraper price calculation.
         """
         if not config.valid:
-            self.error_stack.append(ConfigError())
-            self._print_error()
+            self._error(ConfigError())
             return
 
-        # Reset totals from the previous run and clear the error stack
-        self.usd_total, self.eur_total = 0, 0
-        self.error_stack.clear()
+        self._prepare_new_run()
 
-        capsule_usd_total = self._scrape_capsule_section_prices(update_sheet_callback)
-        case_usd_total = self._scrape_case_prices(update_sheet_callback)
-        custom_item_usd_total = self._scrape_custom_item_prices(update_sheet_callback)
-
-        self.usd_total += capsule_usd_total
-        self.usd_total += case_usd_total
-        self.usd_total += custom_item_usd_total
-        self.eur_total = CurrencyConverter().convert(self.usd_total, "USD", "EUR")
-
-        if update_sheet_callback:
-            update_sheet_callback(["", "", "", ""])
-            update_sheet_callback(
-                [
-                    f"[{datetime.now().strftime('%Y-%m-%d')}] Total:",
-                    f"${self.usd_total:.2f}",
-                    f"€{self.eur_total:.2f}",
-                    "",
-                ]
-            )
+        for section in config.sections():
+            if section in ("User Settings", "App Settings"):
+                continue
+            self._scrape_item_prices(section, update_sheet_callback)
 
-        self._print_total()
-        PriceLogs.save(self.usd_total, self.eur_total)
+        self._convert_totals()
+        self._print_totals(update_sheet_callback)
         self._send_discord_notification()
 
-    def _print_total(self):
-        """Print the total prices in USD and EUR, formatted with titles and
-        separators.
+        usd_totals = [self.totals[price_source]["USD"] for price_source in Parser.SOURCES]
+        PriceLogs.save(usd_totals)
+
+    def _convert_totals(self):
         """
-        usd_title = "USD Total".center(MAX_LINE_LEN, SEPARATOR)
-        console.print(f"[bold green]{usd_title}")
-        console.print(f"${self.usd_total:.2f}")
+        Convert the total prices from USD to the configured conversion currency and
+        update the totals dictionary.
 
-        eur_title = "EUR Total".center(MAX_LINE_LEN, SEPARATOR)
-        console.print(f"[bold green]{eur_title}")
-        console.print(f"€{self.eur_total:.2f}")
+        with the converted totals.
+        """
+        for price_source, totals in self.totals.items():
+            usd_total = totals["USD"]
+            converted_total = convert(usd_total, "USD", self.conversion_currency)
+            self.totals.update({price_source: {"USD": usd_total, self.conversion_currency: converted_total}})  # type: ignore
 
-        end_string = SEPARATOR * MAX_LINE_LEN
-        console.print(f"[bold green]{end_string}\n")
+    def _print_totals(self, update_sheet_callback=None):
+        """
+        Print the total prices in USD and converted currency, formatted with titles and
+        separators.
+
+        :param update_sheet_callback: Optional callback function to update a tksheet
+            with the final totals.
+        """
+        console.title("USD Total", "green")
+        for price_source, totals in self.totals.items():
+            usd_total = totals["USD"]
+            console.print(f"{price_source.name.title():<10}: ${usd_total:.2f}")
+
+        console.title(f"{self.conversion_currency} Total", "green")
+        for price_source, totals in self.totals.items():
+            converted_total = totals[self.conversion_currency]
+            console.print(
+                f"{price_source.name.title():<10}: {to_symbol(self.conversion_currency)}{converted_total:.2f}"
+            )
+
+        if update_sheet_callback and not (
+            self.error_stack and isinstance(self.error_stack[-1], SheetNotFoundError)
+        ):
+            update_sheet_callback(["", ""] + ["", ""] * len(Parser.SOURCES))
+            for price_source, totals in self.totals.items():
+                usd_total = totals["USD"]
+                converted_total = totals[self.conversion_currency]
+                update_sheet_callback(
+                    [
+                        f"[{datetime.now().strftime('%Y-%m-%d')}] {price_source.name.title()} Total:",
+                        f"${usd_total:.2f}",
+                        f"{to_symbol(self.conversion_currency)}{converted_total:.2f}",
+                        "",
+                    ]
+                )
 
     def _send_discord_notification(self):
         """Send a message to a Discord webhook if notifications are enabled in the
         config file and a webhook URL is provided.
         """
-        discord_notifications = config.getboolean(
-            "App Settings", "discord_notifications", fallback=False
-        )
-        webhook_url = config.get("User Settings", "discord_webhook_url", fallback=None)
+        discord_webhook_url = config.discord_webhook_url
 
-        if discord_notifications and webhook_url:
-            DiscordNotifier.notify(webhook_url)
+        if config.discord_notifications and discord_webhook_url:
+            DiscordNotifier.notify(discord_webhook_url)
 
     @retry(stop=stop_after_attempt(10))
     def _get_page(self, url):
@@ -149,10 +200,9 @@ class Scraper:
         :raises RequestException: If the request fails.
         :raises RetryError: If the retry limit is reached.
         """
-        use_proxy = config.getboolean("App Settings", "use_proxy", fallback=False)
-        proxy_api_key = config.get("User Settings", "proxy_api_key", fallback=None)
+        proxy_api_key = config.proxy_api_key
 
-        if use_proxy and proxy_api_key:
+        if config.use_proxy and proxy_api_key:
             page = self.session.get(
                 url=url,
                 proxies={
@@ -165,213 +215,83 @@ class Scraper:
             page = self.session.get(url)
 
         if not page.ok or not page.content:
-            self.error_stack.append(PageLoadError(page.status_code))
-            self._print_error()
+            self._error(PageLoadError(page.status_code))
             raise RequestException(f"Failed to load page: {url}")
 
         return page
 
-    def _print_item_title(self, raw_item_str, from_config=False, from_href=False):
-        """
-        Print the title for a case, capsule, or custom item.
-
-        :param raw_item_str: The raw string to convert into an item name and title.
-        :param from_config: Whether the raw item string is from the config file.
-        :param from_href: Whether the raw item string is an href.
-        :return: The formatted item name.
-        """
-        if from_config:
-            item_name = raw_item_str.replace("_", " ").title()
-        elif from_href:
-            item_name = unquote(raw_item_str.split("/")[-1])
-        else:
-            item_name = raw_item_str
-
-        item_title = item_name.center(MAX_LINE_LEN, SEPARATOR)
-        console.print(f"[bold magenta]{item_title}\n")
-        return item_name
-
-    def _parse_item_price(self, item_page, item_href):
-        """
-        Parse the price of an item from the given steamcommunity market page and item
-        href.
-
-        :param item_page: The HTTP response object containing the item page content.
-        :param item_href: The href of the item listing to find the price for.
-        :return: The price of the item as a float.
-        :raises ValueError: If the item listing or price span cannot be found.
+    def _scrape_prices_from_all_sources(self, item_href, owned):
         """
-        item_soup = BeautifulSoup(item_page.content, "html.parser")
-        item_listing = item_soup.find("a", attrs={"href": f"{item_href}"})
-        if not isinstance(item_listing, Tag):
-            raise ValueError(f"Failed to find item listing: {item_href}")
-
-        item_price_span = item_listing.find("span", attrs={"class": "normal_price"})
-        if not isinstance(item_price_span, Tag):
-            raise ValueError(f"Failed to find price span in item listing: {item_href}")
-
-        price_str = item_price_span.text.split()[2]
-        price = float(price_str.replace("$", ""))
+        For a given item href and owned count, scrape the item's price from all sources
+        available to the currently registered parser.
 
-        return price
-
-    def _scrape_capsule_prices(self, capsule_section, capsule_info, update_sheet_callback=None):
-        """
-        Scrape prices for a specific capsule section, printing the details to the
-        console.
-
-        :param capsule_section: The section name in the config for the capsule.
-        :param capsule_info: A dictionary containing information about the capsule page,
-            hrefs, and names.
-        :param update_sheet_callback: Optional callback function to update a tksheet
-            that is displayed in the GUI with the latest scraper price calculation.
-        """
-        self._print_item_title(capsule_section)
-        capsule_usd_total = 0
-        try:
-            capsule_page = self._get_page(capsule_info["page"])
-            for capsule_href in capsule_info["items"]:
-                capsule_name = unquote(capsule_href.split("/")[-1])
-                config_capsule_name = capsule_name.replace(" ", "_").lower()
-                owned = config.getint(capsule_section, config_capsule_name, fallback=0)
-                if owned == 0:
-                    continue
-
-                price_usd = self._parse_item_price(capsule_page, capsule_href)
-                price_usd_owned = round(float(owned * price_usd), 2)
-
-                console.print(f"[bold deep_sky_blue4]{capsule_name}")
-                console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
-                capsule_usd_total += price_usd_owned
-
-                if update_sheet_callback:
-                    update_sheet_callback([capsule_name, owned, price_usd, price_usd_owned])
-        except (RetryError, ValueError):
-            self.error_stack.append(RequestLimitExceededError())
-            self._print_error()
-        except Exception as error:
-            self.error_stack.append(UnexpectedError(error))
-            self._print_error()
-
-        return capsule_usd_total
-
-    def _scrape_capsule_section_prices(self, update_sheet_callback=None):
-        """
-        Scrape prices for all capsule sections defined in the configuration.
-
-        :param update_sheet_callback: Optional callback function to update a tksheet
-            that is displayed in the GUI with the latest scraper price calculation.
+        :param item_href: The url of the steamcommunity market listing of the item
+        :param owned: How many of this item the user owns
+        :return: A list of item prices for the different sources
+        :raises RequestException: If the request fails.
+        :raises RetryError: If the retry limit is reached.
+        :raises ValueError: If the parser could not find the item
         """
-        capsule_usd_total = 0
-        for capsule_section, capsule_info in CAPSULE_INFO.items():
-            if self.error_stack:
-                break
+        prices = []
+        for price_source in Parser.SOURCES:
+            try:
+                item_page_url = Parser.get_item_page_url(item_href, price_source)
+                item_page = self._get_page(item_page_url)
+                price_usd = Parser.parse_item_price(item_page, item_href, price_source)
 
-            # Only scrape capsule sections where the user owns at least one item
-            if any(int(owned) > 0 for _, owned in config.items(capsule_section)):
-                capsule_usd_total += self._scrape_capsule_prices(
-                    capsule_section, capsule_info, update_sheet_callback
+                price_usd_owned = round(float(int(owned) * price_usd), 2)
+                self.totals[price_source]["USD"] += price_usd_owned
+
+                prices += [price_usd, price_usd_owned]
+                console.price(
+                    Parser.PRICE_INFO,
+                    owned,
+                    price_source.name.title(),
+                    price_usd,
+                    price_usd_owned,
                 )
+            except ValueError as error:
+                prices += [0.0, 0.0]
+                self._error(ParsingError(error))
 
-                if not config.getboolean("App Settings", "use_proxy", fallback=False):
-                    time.sleep(1)
-
-        return capsule_usd_total
+        return prices
 
-    def _market_page_from_href(self, item_href):
+    def _scrape_item_prices(self, section, update_sheet_callback=None):
         """
-        Convert an href of a Steam Community Market item to a market page URL.
+        Scrape prices for all items defined in a configuration section that uses hrefs
+        as option keys.
 
-        :param item_href: The href of the item listing, typically ending with the item's
-            name.
-        :return: A URL string for the Steam Community Market page of the item.
-        """
-        url_encoded_name = item_href.split("/")[-1]
-        page_url = f"https://steamcommunity.com/market/search?q={url_encoded_name}"
-
-        return page_url
-
-    def _scrape_case_prices(self, update_sheet_callback=None):
-        """
-        Scrape prices for all cases defined in the configuration.
-
-        For each case, it prints the case name, owned count, price per item, and total
+        For each item, it prints the item name, owned count, price per item, and total
         price for owned items.
 
         :param update_sheet_callback: Optional callback function to update a tksheet
             that is displayed in the GUI with the latest scraper price calculation.
         """
-        case_usd_total = 0
-        for case_index, (config_case_name, owned) in enumerate(config.items("Cases")):
-            if self.error_stack:
-                break
-            if int(owned) == 0:
-                continue
-
-            case_name = self._print_item_title(config_case_name, from_config=True)
-            try:
-                case_page_url = self._market_page_from_href(CASE_HREFS[case_index])
-                case_page = self._get_page(case_page_url)
-                price_usd = self._parse_item_price(case_page, CASE_HREFS[case_index])
-                price_usd_owned = round(float(int(owned) * price_usd), 2)
-
-                console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
-                case_usd_total += price_usd_owned
-
-                if update_sheet_callback:
-                    update_sheet_callback([case_name, owned, price_usd, price_usd_owned])
-
-                if not config.getboolean("App Settings", "use_proxy", fallback=False):
-                    time.sleep(1)
-            except (RetryError, ValueError):
-                self.error_stack.append(RequestLimitExceededError())
-                self._print_error()
-            except Exception as error:
-                self.error_stack.append(UnexpectedError(error))
-                self._print_error()
-
-        return case_usd_total
-
-    def _scrape_custom_item_prices(self, update_sheet_callback=None):
-        """
-        Scrape prices for custom items defined in the configuration.
-
-        For each custom item, it prints the item name, owned count, price per item, and
-        total price for owned items.
-
-        :param update_sheet_callback: Optional callback function to update a tksheet
-            that is displayed in the GUI with the latest scraper price calculation.
-        """
-        custom_item_usd_total = 0
-        for custom_item_href, owned in config.items("Custom Items"):
-            if self.error_stack:
+        for item_href, owned in config.items(section):
+            if self.error_stack and isinstance(
+                self.error_stack[-1], (RequestLimitExceededError, SheetNotFoundError)
+            ):
                 break
             if int(owned) == 0:
                 continue
 
-            custom_item_name = self._print_item_title(custom_item_href, from_href=True)
+            item_name = config.option_to_name(item_href, href=True)
+            console.title(item_name, "magenta")
             try:
-                custom_item_page_url = self._market_page_from_href(custom_item_href)
-                custom_item_page = self._get_page(custom_item_page_url)
-                price_usd = self._parse_item_price(custom_item_page, custom_item_href)
-                price_usd_owned = round(float(int(owned) * price_usd), 2)
-
-                console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
-                custom_item_usd_total += price_usd_owned
+                prices = self._scrape_prices_from_all_sources(item_href, owned)
 
                 if update_sheet_callback:
-                    update_sheet_callback([custom_item_name, owned, price_usd, price_usd_owned])
+                    try:
+                        update_sheet_callback([item_name, owned] + prices)
+                    except Exception:
+                        self._error(SheetNotFoundError())
 
-                if not config.getboolean("App Settings", "use_proxy", fallback=False):
+                if not config.use_proxy and Parser.NEEDS_TIMEOUT:
                     time.sleep(1)
-            except (RetryError, ValueError):
-                self.error_stack.append(RequestLimitExceededError())
-                self._print_error()
+            except RetryError:
+                self._error(RequestLimitExceededError())
             except Exception as error:
-                self.error_stack.append(UnexpectedError(error))
-                self._print_error()
-
-        return custom_item_usd_total
+                self._error(UnexpectedError(error))
 
 
 if __name__ == "__main__":

cs2tracker/util/__init__.py

@@ -1,9 +0,0 @@
-from cs2tracker.util.padded_console import (  # noqa: F401 # pylint:disable=unused-import
-    get_console,
-)
-from cs2tracker.util.price_logs import (  # noqa: F401 # pylint:disable=unused-import
-    PriceLogs,
-)
-from cs2tracker.util.validated_config import (  # noqa: F401 # pylint:disable=unused-import
-    get_config,
-)

cs2tracker/util/currency_conversion.py

@@ -0,0 +1,84 @@
+from currency_converter import CurrencyConverter
+
+from cs2tracker.config import get_config
+
+config = get_config()
+
+
+converter = CurrencyConverter()
+CURRENCY_SYMBOLS = {
+    "EUR": "€",
+    "KRW": "₩",
+    "ISK": "kr",
+    "HKD": "HK$",
+    "SKK": "Sk",
+    "SEK": "kr",
+    "NOK": "kr",
+    "HUF": "Ft",
+    "LTL": "Lt",
+    "ZAR": "R",
+    "PHP": "₱",
+    "GBP": "£",
+    "MXN": "$",
+    "CYP": "£",
+    "LVL": "Ls",
+    "DKK": "kr",
+    "NZD": "NZ$",
+    "TRY": "₺",
+    "USD": "$",
+    "RON": "lei",
+    "RUB": "₽",
+    "EEK": "kr",
+    "CHF": "CHF",
+    "MYR": "RM",
+    "ILS": "₪",
+    "PLN": "zł",
+    "BRL": "R$",
+    "BGN": "лв",
+    "THB": "฿",
+    "INR": "₹",
+    "ROL": "lei",
+    "AUD": "A$",
+    "CNY": "¥",
+    "HRK": "kn",
+    "MTL": "Lm",
+    "IDR": "Rp",
+    "JPY": "¥",
+    "CAD": "C$",
+}
+CURRENCY_SYMBOLS = {currency: symbol for currency, symbol in CURRENCY_SYMBOLS.items() if currency in converter.currencies}  # type: ignore
+
+
+def convert(amount, source_currency, target_currency):
+    """
+    Convert an amount from source currency to target currency.
+
+    :param amount: The amount to convert.
+    :param source_currency: The currency to convert from.
+    :param target_currency: The currency to convert to.
+    :return: The converted amount in the target currency.
+    """
+    try:
+        if target_currency == "EUR":
+            converted_amount = converter.convert(amount, source_currency, target_currency)
+        else:
+            # The currency converter always needs the target or origin currency to be EUR
+            # Therefore we need an intermediate conversion step if the target currency is not EUR
+            intermediate_amount = converter.convert(amount, source_currency, "EUR")
+            converted_amount = converter.convert(intermediate_amount, "EUR", target_currency)
+
+        return round(converted_amount, 2)
+    except Exception:
+        return 0.0
+
+
+def to_symbol(currency):
+    """
+    Convert a currency code to its symbol.
+
+    :param currency: The currency code to convert.
+    :return: The symbol of the currency.
+    """
+    if currency in CURRENCY_SYMBOLS:
+        return CURRENCY_SYMBOLS[currency]
+    return currency

cs2tracker/util/padded_console.py

@@ -7,6 +7,10 @@ PADDING_LEFT = 4
 PADDING_RIGHT = 0
 
 
+MAX_LINE_LEN = 72
+SEPARATOR = "-"
+
+
 class PaddedConsole:
     def __init__(self, padding=(PADDING_TOP, PADDING_RIGHT, PADDING_BOTTOM, PADDING_LEFT)):
         """Initialize a PaddedConsole with specified padding."""
@@ -17,11 +21,31 @@ class PaddedConsole:
         """Print text with padding to the console."""
         self.console.print(Padding(text, self.padding))
 
+    def info(self, text):
+        """Print info text with padding to the console."""
+        text = "[bold green][+] " + text
+        self.print(text)
+
     def error(self, text):
         """Print error text with padding to the console."""
         text = "[bold red][!] " + text
         self.print(text)
 
+    def title(self, text, color):
+        """Print the given text as a title."""
+        title = text.center(MAX_LINE_LEN, SEPARATOR)
+        console.print(f"\n[bold {color}]{title}\n")
+
+    def separator(self, color):
+        """Print a separator line."""
+        separator = SEPARATOR * MAX_LINE_LEN
+        console.print(f"[bold {color}]{separator}")
+
+    def price(self, price_str, price_source, owned, steam_market_price, total_owned):
+        # pylint: disable=too-many-arguments,too-many-positional-arguments
+        """Print price information."""
+        console.print(price_str.format(price_source, owned, steam_market_price, total_owned))
+
     def __getattr__(self, attr):
         """Ensure console methods can be called directly on PaddedConsole."""
         return getattr(self.console, attr)