cs2tracker 2.1.9__py3-none-any.whl → 2.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cs2tracker might be problematic. Click here for more details.
- cs2tracker/_version.py +2 -2
- cs2tracker/application.py +341 -95
- cs2tracker/background_task.py +109 -0
- cs2tracker/constants.py +2 -3
- cs2tracker/data/config.ini +155 -156
- cs2tracker/discord_notifier.py +87 -0
- cs2tracker/price_logs.py +100 -0
- cs2tracker/scraper.py +45 -365
- cs2tracker/validated_config.py +117 -0
- {cs2tracker-2.1.9.dist-info → cs2tracker-2.1.10.dist-info}/METADATA +7 -4
- cs2tracker-2.1.10.dist-info/RECORD +20 -0
- cs2tracker-2.1.9.dist-info/RECORD +0 -16
- {cs2tracker-2.1.9.dist-info → cs2tracker-2.1.10.dist-info}/WHEEL +0 -0
- {cs2tracker-2.1.9.dist-info → cs2tracker-2.1.10.dist-info}/entry_points.txt +0 -0
- {cs2tracker-2.1.9.dist-info → cs2tracker-2.1.10.dist-info}/licenses/LICENSE.md +0 -0
- {cs2tracker-2.1.9.dist-info → cs2tracker-2.1.10.dist-info}/top_level.txt +0 -0
cs2tracker/scraper.py
CHANGED
|
@@ -1,9 +1,5 @@
|
|
|
1
|
-
import csv
|
|
2
|
-
import os
|
|
3
1
|
import time
|
|
4
|
-
from
|
|
5
|
-
from datetime import datetime
|
|
6
|
-
from subprocess import DEVNULL, call
|
|
2
|
+
from urllib.parse import unquote
|
|
7
3
|
|
|
8
4
|
from bs4 import BeautifulSoup
|
|
9
5
|
from bs4.element import Tag
|
|
@@ -12,21 +8,11 @@ from requests import RequestException, Session
|
|
|
12
8
|
from requests.adapters import HTTPAdapter, Retry
|
|
13
9
|
from tenacity import RetryError, retry, stop_after_attempt
|
|
14
10
|
|
|
15
|
-
from cs2tracker.constants import
|
|
16
|
-
|
|
17
|
-
BANNER,
|
|
18
|
-
BATCH_FILE,
|
|
19
|
-
CAPSULE_INFO,
|
|
20
|
-
CASE_HREFS,
|
|
21
|
-
CONFIG_FILE,
|
|
22
|
-
OS,
|
|
23
|
-
OUTPUT_FILE,
|
|
24
|
-
PROJECT_DIR,
|
|
25
|
-
PYTHON_EXECUTABLE,
|
|
26
|
-
RUNNING_IN_EXE,
|
|
27
|
-
OSType,
|
|
28
|
-
)
|
|
11
|
+
from cs2tracker.constants import AUTHOR_STRING, BANNER, CAPSULE_INFO, CASE_HREFS
|
|
12
|
+
from cs2tracker.discord_notifier import DiscordNotifier
|
|
29
13
|
from cs2tracker.padded_console import PaddedConsole
|
|
14
|
+
from cs2tracker.price_logs import PriceLogs
|
|
15
|
+
from cs2tracker.validated_config import ValidatedConfig
|
|
30
16
|
|
|
31
17
|
MAX_LINE_LEN = 72
|
|
32
18
|
SEPARATOR = "-"
|
|
@@ -35,97 +21,21 @@ PRICE_INFO = "Owned: {:<10} Steam market price: ${:<10} Total: ${:<10}\n"
|
|
|
35
21
|
HTTP_PROXY_URL = "http://{}:@smartproxy.crawlbase.com:8012"
|
|
36
22
|
HTTPS_PROXY_URL = "http://{}:@smartproxy.crawlbase.com:8012"
|
|
37
23
|
|
|
38
|
-
|
|
39
|
-
DC_WEBHOOK_AVATAR_URL = "https://img.icons8.com/?size=100&id=uWQJp2tLXUH6&format=png&color=000000"
|
|
40
|
-
DC_RECENT_HISTORY_LIMIT = 5
|
|
41
|
-
|
|
42
|
-
WIN_BACKGROUND_TASK_NAME = "CS2Tracker Daily Calculation"
|
|
43
|
-
WIN_BACKGROUND_TASK_SCHEDULE = "DAILY"
|
|
44
|
-
WIN_BACKGROUND_TASK_TIME = "12:00"
|
|
45
|
-
WIN_BACKGROUND_TASK_CMD = (
|
|
46
|
-
f"powershell -WindowStyle Hidden -Command \"Start-Process '{BATCH_FILE}' -WindowStyle Hidden\""
|
|
47
|
-
)
|
|
24
|
+
console = PaddedConsole()
|
|
48
25
|
|
|
49
26
|
|
|
50
27
|
class Scraper:
|
|
51
28
|
def __init__(self):
|
|
52
29
|
"""Initialize the Scraper class."""
|
|
53
|
-
self.
|
|
54
|
-
self.parse_config()
|
|
30
|
+
self.load_config()
|
|
55
31
|
self._start_session()
|
|
56
32
|
|
|
57
33
|
self.usd_total = 0
|
|
58
34
|
self.eur_total = 0
|
|
59
35
|
|
|
60
|
-
def
|
|
61
|
-
"""
|
|
62
|
-
|
|
63
|
-
raise ValueError("Missing 'User Settings' section in the configuration file.")
|
|
64
|
-
if not self.config.has_section("App Settings"):
|
|
65
|
-
raise ValueError("Missing 'App Settings' section in the configuration file.")
|
|
66
|
-
if not self.config.has_section("Custom Items"):
|
|
67
|
-
raise ValueError("Missing 'Custom Items' section in the configuration file.")
|
|
68
|
-
if not self.config.has_section("Cases"):
|
|
69
|
-
raise ValueError("Missing 'Cases' section in the configuration file.")
|
|
70
|
-
for capsule_section in CAPSULE_INFO:
|
|
71
|
-
if not self.config.has_section(capsule_section):
|
|
72
|
-
raise ValueError(f"Missing '{capsule_section}' section in the configuration file.")
|
|
73
|
-
|
|
74
|
-
def _validate_config_values(self):
|
|
75
|
-
"""Validate that the configuration file has valid values for all sections."""
|
|
76
|
-
try:
|
|
77
|
-
for custom_item_name, custom_item_owned in self.config.items("Custom Items"):
|
|
78
|
-
if " " not in custom_item_owned:
|
|
79
|
-
raise ValueError(
|
|
80
|
-
f"Invalid custom item format (<item_name> = <owned_count> <item_url>): {custom_item_name} = {custom_item_owned}"
|
|
81
|
-
)
|
|
82
|
-
owned, _ = custom_item_owned.split(" ", 1)
|
|
83
|
-
if int(owned) < 0:
|
|
84
|
-
raise ValueError(
|
|
85
|
-
f"Invalid value in 'Custom Items' section: {custom_item_name} = {custom_item_owned}"
|
|
86
|
-
)
|
|
87
|
-
for case_name, case_owned in self.config.items("Cases"):
|
|
88
|
-
if int(case_owned) < 0:
|
|
89
|
-
raise ValueError(
|
|
90
|
-
f"Invalid value in 'Cases' section: {case_name} = {case_owned}"
|
|
91
|
-
)
|
|
92
|
-
for capsule_section in CAPSULE_INFO:
|
|
93
|
-
for capsule_name, capsule_owned in self.config.items(capsule_section):
|
|
94
|
-
if int(capsule_owned) < 0:
|
|
95
|
-
raise ValueError(
|
|
96
|
-
f"Invalid value in '{capsule_section}' section: {capsule_name} = {capsule_owned}"
|
|
97
|
-
)
|
|
98
|
-
except ValueError as error:
|
|
99
|
-
if "Invalid " in str(error):
|
|
100
|
-
raise
|
|
101
|
-
raise ValueError("Invalid value type. All values must be integers.") from error
|
|
102
|
-
|
|
103
|
-
def _validate_config(self):
|
|
104
|
-
"""
|
|
105
|
-
Validate the configuration file to ensure all required sections exist with the
|
|
106
|
-
right values.
|
|
107
|
-
|
|
108
|
-
:raises ValueError: If any required section is missing or if any value is
|
|
109
|
-
invalid.
|
|
110
|
-
"""
|
|
111
|
-
self._validate_config_sections()
|
|
112
|
-
self._validate_config_values()
|
|
113
|
-
|
|
114
|
-
def parse_config(self):
|
|
115
|
-
"""
|
|
116
|
-
Parse the configuration file to read settings and user-owned items.
|
|
117
|
-
|
|
118
|
-
Sets self.valid_config to True if the configuration is valid, and False if it is
|
|
119
|
-
not.
|
|
120
|
-
"""
|
|
121
|
-
self.config = ConfigParser(interpolation=None)
|
|
122
|
-
self.config.read(CONFIG_FILE)
|
|
123
|
-
try:
|
|
124
|
-
self._validate_config()
|
|
125
|
-
self.valid_config = True
|
|
126
|
-
except ValueError as error:
|
|
127
|
-
self.console.print(f"[bold red][!] Configuration error: {error}")
|
|
128
|
-
self.valid_config = False
|
|
36
|
+
def load_config(self):
|
|
37
|
+
"""Load the configuration file and validate its contents."""
|
|
38
|
+
self.config = ValidatedConfig()
|
|
129
39
|
|
|
130
40
|
def _start_session(self):
|
|
131
41
|
"""Start a requests session with custom headers and retry logic."""
|
|
@@ -143,8 +53,8 @@ class Scraper:
|
|
|
143
53
|
"""Scrape prices for capsules and cases, calculate totals in USD and EUR, and
|
|
144
54
|
print/save the results.
|
|
145
55
|
"""
|
|
146
|
-
if not self.
|
|
147
|
-
|
|
56
|
+
if not self.config.valid:
|
|
57
|
+
console.print(
|
|
148
58
|
"[bold red][!] Invalid configuration. Please fix the config file before running."
|
|
149
59
|
)
|
|
150
60
|
return
|
|
@@ -159,7 +69,7 @@ class Scraper:
|
|
|
159
69
|
self.eur_total = CurrencyConverter().convert(self.usd_total, "USD", "EUR")
|
|
160
70
|
|
|
161
71
|
self._print_total()
|
|
162
|
-
self.
|
|
72
|
+
PriceLogs.save(self.usd_total, self.eur_total)
|
|
163
73
|
self._send_discord_notification()
|
|
164
74
|
|
|
165
75
|
# Reset totals for next run
|
|
@@ -170,125 +80,15 @@ class Scraper:
|
|
|
170
80
|
separators.
|
|
171
81
|
"""
|
|
172
82
|
usd_title = "USD Total".center(MAX_LINE_LEN, SEPARATOR)
|
|
173
|
-
|
|
174
|
-
|
|
83
|
+
console.print(f"[bold green]{usd_title}")
|
|
84
|
+
console.print(f"${self.usd_total:.2f}")
|
|
175
85
|
|
|
176
86
|
eur_title = "EUR Total".center(MAX_LINE_LEN, SEPARATOR)
|
|
177
|
-
|
|
178
|
-
|
|
87
|
+
console.print(f"[bold green]{eur_title}")
|
|
88
|
+
console.print(f"€{self.eur_total:.2f}")
|
|
179
89
|
|
|
180
90
|
end_string = SEPARATOR * MAX_LINE_LEN
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
def _save_price_log(self):
|
|
184
|
-
"""
|
|
185
|
-
Save the current date and total prices in USD and EUR to a CSV file.
|
|
186
|
-
|
|
187
|
-
This will append a new entry to the output file if no entry has been made for
|
|
188
|
-
today.
|
|
189
|
-
|
|
190
|
-
:raises FileNotFoundError: If the output file does not exist.
|
|
191
|
-
:raises IOError: If there is an error writing to the output file.
|
|
192
|
-
"""
|
|
193
|
-
with open(OUTPUT_FILE, "r", encoding="utf-8") as price_logs:
|
|
194
|
-
price_logs_reader = csv.reader(price_logs)
|
|
195
|
-
rows = list(price_logs_reader)
|
|
196
|
-
last_log_date, _, _ = rows[-1] if rows else ("", "", "")
|
|
197
|
-
|
|
198
|
-
today = datetime.now().strftime("%Y-%m-%d")
|
|
199
|
-
if last_log_date != today:
|
|
200
|
-
# Append first price calculation of the day
|
|
201
|
-
with open(OUTPUT_FILE, "a", newline="", encoding="utf-8") as price_logs:
|
|
202
|
-
price_logs_writer = csv.writer(price_logs)
|
|
203
|
-
price_logs_writer.writerow(
|
|
204
|
-
[today, f"{self.usd_total:.2f}$", f"{self.eur_total:.2f}€"]
|
|
205
|
-
)
|
|
206
|
-
else:
|
|
207
|
-
# Replace the last calculation of today with the most recent one of today
|
|
208
|
-
with open(OUTPUT_FILE, "r+", newline="", encoding="utf-8") as price_logs:
|
|
209
|
-
price_logs_reader = csv.reader(price_logs)
|
|
210
|
-
rows = list(price_logs_reader)
|
|
211
|
-
rows_without_today = rows[:-1]
|
|
212
|
-
price_logs.seek(0)
|
|
213
|
-
price_logs.truncate()
|
|
214
|
-
|
|
215
|
-
price_logs_writer = csv.writer(price_logs)
|
|
216
|
-
price_logs_writer.writerows(rows_without_today)
|
|
217
|
-
price_logs_writer.writerow(
|
|
218
|
-
[today, f"{self.usd_total:.2f}$", f"{self.eur_total:.2f}€"]
|
|
219
|
-
)
|
|
220
|
-
|
|
221
|
-
def read_price_log(self):
|
|
222
|
-
"""
|
|
223
|
-
Parse the output file to extract dates, dollar prices, and euro prices. This
|
|
224
|
-
data is used for drawing the plot of past prices.
|
|
225
|
-
|
|
226
|
-
:return: A tuple containing three lists: dates, dollar prices, and euro prices.
|
|
227
|
-
:raises FileNotFoundError: If the output file does not exist.
|
|
228
|
-
:raises IOError: If there is an error reading the output file.
|
|
229
|
-
"""
|
|
230
|
-
dates, dollars, euros = [], [], []
|
|
231
|
-
with open(OUTPUT_FILE, "r", encoding="utf-8") as price_logs:
|
|
232
|
-
price_logs_reader = csv.reader(price_logs)
|
|
233
|
-
for row in price_logs_reader:
|
|
234
|
-
date, price_usd, price_eur = row
|
|
235
|
-
date = datetime.strptime(date, "%Y-%m-%d")
|
|
236
|
-
price_usd = float(price_usd.rstrip("$"))
|
|
237
|
-
price_eur = float(price_eur.rstrip("€"))
|
|
238
|
-
|
|
239
|
-
dates.append(date)
|
|
240
|
-
dollars.append(price_usd)
|
|
241
|
-
euros.append(price_eur)
|
|
242
|
-
|
|
243
|
-
return dates, dollars, euros
|
|
244
|
-
|
|
245
|
-
def _construct_recent_calculations_embeds(self):
|
|
246
|
-
"""
|
|
247
|
-
Construct the embeds for the Discord message that will be sent after a price
|
|
248
|
-
calculation has been made.
|
|
249
|
-
|
|
250
|
-
:return: A list of embeds for the Discord message.
|
|
251
|
-
"""
|
|
252
|
-
dates, usd_logs, eur_logs = self.read_price_log()
|
|
253
|
-
dates, usd_logs, eur_logs = reversed(dates), reversed(usd_logs), reversed(eur_logs)
|
|
254
|
-
|
|
255
|
-
date_history, usd_history, eur_history = [], [], []
|
|
256
|
-
for date, usd_log, eur_log in zip(dates, usd_logs, eur_logs):
|
|
257
|
-
if len(date_history) >= DC_RECENT_HISTORY_LIMIT:
|
|
258
|
-
break
|
|
259
|
-
date_history.append(date.strftime("%Y-%m-%d"))
|
|
260
|
-
usd_history.append(f"${usd_log:.2f}")
|
|
261
|
-
eur_history.append(f"€{eur_log:.2f}")
|
|
262
|
-
|
|
263
|
-
date_history = "\n".join(date_history)
|
|
264
|
-
usd_history = "\n".join(usd_history)
|
|
265
|
-
eur_history = "\n".join(eur_history)
|
|
266
|
-
|
|
267
|
-
embeds = [
|
|
268
|
-
{
|
|
269
|
-
"title": "📊 Recent Price History",
|
|
270
|
-
"color": 5814783,
|
|
271
|
-
"fields": [
|
|
272
|
-
{
|
|
273
|
-
"name": "Date",
|
|
274
|
-
"value": date_history,
|
|
275
|
-
"inline": True,
|
|
276
|
-
},
|
|
277
|
-
{
|
|
278
|
-
"name": "USD Total",
|
|
279
|
-
"value": usd_history,
|
|
280
|
-
"inline": True,
|
|
281
|
-
},
|
|
282
|
-
{
|
|
283
|
-
"name": "EUR Total",
|
|
284
|
-
"value": eur_history,
|
|
285
|
-
"inline": True,
|
|
286
|
-
},
|
|
287
|
-
],
|
|
288
|
-
}
|
|
289
|
-
]
|
|
290
|
-
|
|
291
|
-
return embeds
|
|
91
|
+
console.print(f"[bold green]{end_string}\n")
|
|
292
92
|
|
|
293
93
|
def _send_discord_notification(self):
|
|
294
94
|
"""Send a message to a Discord webhook if notifications are enabled in the
|
|
@@ -298,25 +98,9 @@ class Scraper:
|
|
|
298
98
|
"App Settings", "discord_notifications", fallback=False
|
|
299
99
|
)
|
|
300
100
|
webhook_url = self.config.get("User Settings", "discord_webhook_url", fallback=None)
|
|
301
|
-
webhook_url = None if webhook_url in ("None", "") else webhook_url
|
|
302
101
|
|
|
303
102
|
if discord_notifications and webhook_url:
|
|
304
|
-
|
|
305
|
-
try:
|
|
306
|
-
response = self.session.post(
|
|
307
|
-
url=webhook_url,
|
|
308
|
-
json={
|
|
309
|
-
"embeds": embeds,
|
|
310
|
-
"username": DC_WEBHOOK_USERNAME,
|
|
311
|
-
"avatar_url": DC_WEBHOOK_AVATAR_URL,
|
|
312
|
-
},
|
|
313
|
-
)
|
|
314
|
-
response.raise_for_status()
|
|
315
|
-
self.console.print("[bold steel_blue3][+] Discord notification sent.\n")
|
|
316
|
-
except RequestException as error:
|
|
317
|
-
self.console.print(f"[bold red][!] Failed to send Discord notification: {error}\n")
|
|
318
|
-
except Exception as error:
|
|
319
|
-
self.console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
|
|
103
|
+
DiscordNotifier.notify(webhook_url)
|
|
320
104
|
|
|
321
105
|
@retry(stop=stop_after_attempt(10))
|
|
322
106
|
def _get_page(self, url):
|
|
@@ -330,15 +114,14 @@ class Scraper:
|
|
|
330
114
|
:raises RetryError: If the retry limit is reached.
|
|
331
115
|
"""
|
|
332
116
|
use_proxy = self.config.getboolean("App Settings", "use_proxy", fallback=False)
|
|
333
|
-
|
|
334
|
-
api_key = None if api_key in ("None", "") else api_key
|
|
117
|
+
proxy_api_key = self.config.get("User Settings", "proxy_api_key", fallback=None)
|
|
335
118
|
|
|
336
|
-
if use_proxy and
|
|
119
|
+
if use_proxy and proxy_api_key:
|
|
337
120
|
page = self.session.get(
|
|
338
121
|
url=url,
|
|
339
122
|
proxies={
|
|
340
|
-
"http": HTTP_PROXY_URL.format(
|
|
341
|
-
"https": HTTPS_PROXY_URL.format(
|
|
123
|
+
"http": HTTP_PROXY_URL.format(proxy_api_key),
|
|
124
|
+
"https": HTTPS_PROXY_URL.format(proxy_api_key),
|
|
342
125
|
},
|
|
343
126
|
verify=False,
|
|
344
127
|
)
|
|
@@ -346,9 +129,7 @@ class Scraper:
|
|
|
346
129
|
page = self.session.get(url)
|
|
347
130
|
|
|
348
131
|
if not page.ok or not page.content:
|
|
349
|
-
|
|
350
|
-
f"[bold red][!] Failed to load page ({page.status_code}). Retrying...\n"
|
|
351
|
-
)
|
|
132
|
+
console.print(f"[bold red][!] Failed to load page ({page.status_code}). Retrying...\n")
|
|
352
133
|
raise RequestException(f"Failed to load page: {url}")
|
|
353
134
|
|
|
354
135
|
return page
|
|
@@ -391,7 +172,7 @@ class Scraper:
|
|
|
391
172
|
hrefs, and names.
|
|
392
173
|
"""
|
|
393
174
|
capsule_title = capsule_section.center(MAX_LINE_LEN, SEPARATOR)
|
|
394
|
-
|
|
175
|
+
console.print(f"[bold magenta]{capsule_title}\n")
|
|
395
176
|
|
|
396
177
|
capsule_usd_total = 0
|
|
397
178
|
try:
|
|
@@ -405,17 +186,15 @@ class Scraper:
|
|
|
405
186
|
price_usd = self._parse_item_price(capsule_page, capsule_href)
|
|
406
187
|
price_usd_owned = round(float(owned * price_usd), 2)
|
|
407
188
|
|
|
408
|
-
|
|
409
|
-
|
|
189
|
+
console.print(f"[bold deep_sky_blue4]{capsule_name}")
|
|
190
|
+
console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
|
|
410
191
|
capsule_usd_total += price_usd_owned
|
|
411
192
|
except (RetryError, ValueError):
|
|
412
|
-
|
|
413
|
-
"[bold red][!]
|
|
193
|
+
console.print(
|
|
194
|
+
"[bold red][!] Too many requests. (Consider using proxies to prevent rate limiting)\n"
|
|
414
195
|
)
|
|
415
196
|
except Exception as error:
|
|
416
|
-
|
|
417
|
-
f"[bold red][!] An unexpected error occurred while scraping capsule prices: {error}\n"
|
|
418
|
-
)
|
|
197
|
+
console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
|
|
419
198
|
|
|
420
199
|
return capsule_usd_total
|
|
421
200
|
|
|
@@ -456,7 +235,7 @@ class Scraper:
|
|
|
456
235
|
|
|
457
236
|
case_name = config_case_name.replace("_", " ").title()
|
|
458
237
|
case_title = case_name.center(MAX_LINE_LEN, SEPARATOR)
|
|
459
|
-
|
|
238
|
+
console.print(f"[bold magenta]{case_title}\n")
|
|
460
239
|
|
|
461
240
|
try:
|
|
462
241
|
case_page_url = self._market_page_from_href(CASE_HREFS[case_index])
|
|
@@ -464,19 +243,17 @@ class Scraper:
|
|
|
464
243
|
price_usd = self._parse_item_price(case_page, CASE_HREFS[case_index])
|
|
465
244
|
price_usd_owned = round(float(int(owned) * price_usd), 2)
|
|
466
245
|
|
|
467
|
-
|
|
246
|
+
console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
|
|
468
247
|
case_usd_total += price_usd_owned
|
|
469
248
|
|
|
470
249
|
if not self.config.getboolean("App Settings", "use_proxy", fallback=False):
|
|
471
250
|
time.sleep(1)
|
|
472
251
|
except (RetryError, ValueError):
|
|
473
|
-
|
|
474
|
-
"[bold red][!]
|
|
252
|
+
console.print(
|
|
253
|
+
"[bold red][!] Too many requests. (Consider using proxies to prevent rate limiting)\n"
|
|
475
254
|
)
|
|
476
255
|
except Exception as error:
|
|
477
|
-
|
|
478
|
-
f"[bold red][!] An unexpected error occurred while scraping case prices: {error}\n"
|
|
479
|
-
)
|
|
256
|
+
console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
|
|
480
257
|
|
|
481
258
|
return case_usd_total
|
|
482
259
|
|
|
@@ -488,14 +265,13 @@ class Scraper:
|
|
|
488
265
|
total price for owned items.
|
|
489
266
|
"""
|
|
490
267
|
custom_item_usd_total = 0
|
|
491
|
-
for
|
|
492
|
-
owned, custom_item_href = owned_and_href.split(" ", 1)
|
|
268
|
+
for custom_item_href, owned in self.config.items("Custom Items"):
|
|
493
269
|
if int(owned) == 0:
|
|
494
270
|
continue
|
|
495
271
|
|
|
496
|
-
custom_item_name =
|
|
272
|
+
custom_item_name = unquote(custom_item_href.split("/")[-1])
|
|
497
273
|
custom_item_title = custom_item_name.center(MAX_LINE_LEN, SEPARATOR)
|
|
498
|
-
|
|
274
|
+
console.print(f"[bold magenta]{custom_item_title}\n")
|
|
499
275
|
|
|
500
276
|
try:
|
|
501
277
|
custom_item_page_url = self._market_page_from_href(custom_item_href)
|
|
@@ -503,117 +279,27 @@ class Scraper:
|
|
|
503
279
|
price_usd = self._parse_item_price(custom_item_page, custom_item_href)
|
|
504
280
|
price_usd_owned = round(float(int(owned) * price_usd), 2)
|
|
505
281
|
|
|
506
|
-
|
|
282
|
+
console.print(PRICE_INFO.format(owned, price_usd, price_usd_owned))
|
|
507
283
|
custom_item_usd_total += price_usd_owned
|
|
508
284
|
|
|
509
285
|
if not self.config.getboolean("App Settings", "use_proxy", fallback=False):
|
|
510
286
|
time.sleep(1)
|
|
511
287
|
except (RetryError, ValueError):
|
|
512
|
-
|
|
513
|
-
"[bold red][!]
|
|
288
|
+
console.print(
|
|
289
|
+
"[bold red][!] Too many requests. (Consider using proxies to prevent rate limiting)\n"
|
|
514
290
|
)
|
|
515
291
|
except Exception as error:
|
|
516
|
-
|
|
517
|
-
f"[bold red][!] An unexpected error occurred while scraping custom item prices: {error}\n"
|
|
518
|
-
)
|
|
292
|
+
console.print(f"[bold red][!] An unexpected error occurred: {error}\n")
|
|
519
293
|
|
|
520
294
|
return custom_item_usd_total
|
|
521
295
|
|
|
522
|
-
def identify_background_task(self):
|
|
523
|
-
"""
|
|
524
|
-
Search the OS for a daily background task that runs the scraper.
|
|
525
|
-
|
|
526
|
-
:return: True if a background task is found, False otherwise.
|
|
527
|
-
"""
|
|
528
|
-
if OS == OSType.WINDOWS:
|
|
529
|
-
cmd = ["schtasks", "/query", "/tn", WIN_BACKGROUND_TASK_NAME]
|
|
530
|
-
return_code = call(cmd, stdout=DEVNULL, stderr=DEVNULL)
|
|
531
|
-
found = return_code == 0
|
|
532
|
-
return found
|
|
533
|
-
else:
|
|
534
|
-
# TODO: implement finder for cron jobs
|
|
535
|
-
return False
|
|
536
|
-
|
|
537
|
-
def _toggle_task_batch_file(self, enabled: bool):
|
|
538
|
-
"""
|
|
539
|
-
Create or delete a batch file that runs the scraper.
|
|
540
|
-
|
|
541
|
-
:param enabled: If True, the batch file will be created; if False, the batch
|
|
542
|
-
file will be deleted.
|
|
543
|
-
"""
|
|
544
|
-
if enabled:
|
|
545
|
-
with open(BATCH_FILE, "w", encoding="utf-8") as batch_file:
|
|
546
|
-
if RUNNING_IN_EXE:
|
|
547
|
-
# The python executable is set to the executable itself
|
|
548
|
-
# for executables created with PyInstaller
|
|
549
|
-
batch_file.write(f"{PYTHON_EXECUTABLE} --only-scrape\n")
|
|
550
|
-
else:
|
|
551
|
-
batch_file.write(f"cd {PROJECT_DIR}\n")
|
|
552
|
-
batch_file.write(f"{PYTHON_EXECUTABLE} -m cs2tracker --only-scrape\n")
|
|
553
|
-
else:
|
|
554
|
-
if os.path.exists(BATCH_FILE):
|
|
555
|
-
os.remove(BATCH_FILE)
|
|
556
|
-
|
|
557
|
-
def _toggle_background_task_windows(self, enabled: bool):
|
|
558
|
-
"""
|
|
559
|
-
Create or delete a daily background task that runs the scraper on Windows.
|
|
560
|
-
|
|
561
|
-
:param enabled: If True, the task will be created; if False, the task will be
|
|
562
|
-
deleted.
|
|
563
|
-
"""
|
|
564
|
-
self._toggle_task_batch_file(enabled)
|
|
565
|
-
if enabled:
|
|
566
|
-
cmd = [
|
|
567
|
-
"schtasks",
|
|
568
|
-
"/create",
|
|
569
|
-
"/tn",
|
|
570
|
-
WIN_BACKGROUND_TASK_NAME,
|
|
571
|
-
"/tr",
|
|
572
|
-
WIN_BACKGROUND_TASK_CMD,
|
|
573
|
-
"/sc",
|
|
574
|
-
WIN_BACKGROUND_TASK_SCHEDULE,
|
|
575
|
-
"/st",
|
|
576
|
-
WIN_BACKGROUND_TASK_TIME,
|
|
577
|
-
]
|
|
578
|
-
return_code = call(cmd, stdout=DEVNULL, stderr=DEVNULL)
|
|
579
|
-
if return_code == 0:
|
|
580
|
-
self.console.print("[bold green][+] Background task enabled.")
|
|
581
|
-
else:
|
|
582
|
-
self.console.print("[bold red][!] Failed to enable background task.")
|
|
583
|
-
else:
|
|
584
|
-
cmd = ["schtasks", "/delete", "/tn", WIN_BACKGROUND_TASK_NAME, "/f"]
|
|
585
|
-
return_code = call(cmd, stdout=DEVNULL, stderr=DEVNULL)
|
|
586
|
-
if return_code == 0:
|
|
587
|
-
self.console.print("[bold green][-] Background task disabled.")
|
|
588
|
-
else:
|
|
589
|
-
self.console.print("[bold red][!] Failed to disable background task.")
|
|
590
|
-
|
|
591
|
-
def toggle_background_task(self, enabled: bool):
|
|
592
|
-
"""
|
|
593
|
-
Create or delete a daily background task that runs the scraper.
|
|
594
|
-
|
|
595
|
-
:param enabled: If True, the task will be created; if False, the task will be
|
|
596
|
-
deleted.
|
|
597
|
-
"""
|
|
598
|
-
if OS == OSType.WINDOWS:
|
|
599
|
-
self._toggle_background_task_windows(enabled)
|
|
600
|
-
else:
|
|
601
|
-
# TODO: implement toggle for cron jobs
|
|
602
|
-
pass
|
|
603
|
-
|
|
604
296
|
def toggle_use_proxy(self, enabled: bool):
|
|
605
297
|
"""
|
|
606
298
|
Toggle the use of proxies for requests. This will update the configuration file.
|
|
607
299
|
|
|
608
300
|
:param enabled: If True, proxies will be used; if False, they will not be used.
|
|
609
301
|
"""
|
|
610
|
-
self.config.
|
|
611
|
-
with open(CONFIG_FILE, "w", encoding="utf-8") as config_file:
|
|
612
|
-
self.config.write(config_file)
|
|
613
|
-
|
|
614
|
-
self.console.print(
|
|
615
|
-
f"[bold green]{'[+] Enabled' if enabled else '[-] Disabled'} proxy usage for requests."
|
|
616
|
-
)
|
|
302
|
+
self.config.toggle_use_proxy(enabled)
|
|
617
303
|
|
|
618
304
|
def toggle_discord_webhook(self, enabled: bool):
|
|
619
305
|
"""
|
|
@@ -622,16 +308,10 @@ class Scraper:
|
|
|
622
308
|
:param enabled: If True, the webhook will be used; if False, it will not be
|
|
623
309
|
used.
|
|
624
310
|
"""
|
|
625
|
-
self.config.
|
|
626
|
-
with open(CONFIG_FILE, "w", encoding="utf-8") as config_file:
|
|
627
|
-
self.config.write(config_file)
|
|
628
|
-
|
|
629
|
-
self.console.print(
|
|
630
|
-
f"[bold green]{'[+] Enabled' if enabled else '[-] Disabled'} Discord webhook notifications."
|
|
631
|
-
)
|
|
311
|
+
self.config.toggle_discord_webhook(enabled)
|
|
632
312
|
|
|
633
313
|
|
|
634
314
|
if __name__ == "__main__":
|
|
635
315
|
scraper = Scraper()
|
|
636
|
-
|
|
316
|
+
console.print(f"[bold yellow]{BANNER}\n{AUTHOR_STRING}\n")
|
|
637
317
|
scraper.scrape_prices()
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from configparser import ConfigParser
|
|
3
|
+
|
|
4
|
+
from cs2tracker.constants import CAPSULE_INFO, CONFIG_FILE
|
|
5
|
+
from cs2tracker.padded_console import PaddedConsole
|
|
6
|
+
|
|
7
|
+
console = PaddedConsole()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
STEAM_MARKET_LISTING_REGEX = r"^https://steamcommunity.com/market/listings/\d+/.+$"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ValidatedConfig(ConfigParser):
|
|
14
|
+
def __init__(self):
|
|
15
|
+
"""Initialize the ValidatedConfig class."""
|
|
16
|
+
super().__init__(delimiters=("~"), interpolation=None)
|
|
17
|
+
self.optionxform = str # type: ignore
|
|
18
|
+
super().read(CONFIG_FILE)
|
|
19
|
+
|
|
20
|
+
self.valid = False
|
|
21
|
+
self.last_error = None
|
|
22
|
+
self._validate_config()
|
|
23
|
+
|
|
24
|
+
def _validate_config_sections(self):
|
|
25
|
+
"""Validate that the configuration file has all required sections."""
|
|
26
|
+
if not self.has_section("User Settings"):
|
|
27
|
+
raise ValueError("Missing 'User Settings' section in the configuration file.")
|
|
28
|
+
if not self.has_section("App Settings"):
|
|
29
|
+
raise ValueError("Missing 'App Settings' section in the configuration file.")
|
|
30
|
+
if not self.has_section("Custom Items"):
|
|
31
|
+
raise ValueError("Missing 'Custom Items' section in the configuration file.")
|
|
32
|
+
if not self.has_section("Cases"):
|
|
33
|
+
raise ValueError("Missing 'Cases' section in the configuration file.")
|
|
34
|
+
for capsule_section in CAPSULE_INFO:
|
|
35
|
+
if not self.has_section(capsule_section):
|
|
36
|
+
raise ValueError(f"Missing '{capsule_section}' section in the configuration file.")
|
|
37
|
+
|
|
38
|
+
def _validate_config_values(self):
|
|
39
|
+
"""Validate that the configuration file has valid values for all sections."""
|
|
40
|
+
try:
|
|
41
|
+
for custom_item_href, custom_item_owned in self.items("Custom Items"):
|
|
42
|
+
if not re.match(STEAM_MARKET_LISTING_REGEX, custom_item_href):
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"Invalid Steam market listing URL in 'Custom Items' section: {custom_item_href}"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
if int(custom_item_owned) < 0:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"Invalid value in 'Custom Items' section: {custom_item_href} = {custom_item_owned}"
|
|
50
|
+
)
|
|
51
|
+
for case_name, case_owned in self.items("Cases"):
|
|
52
|
+
if int(case_owned) < 0:
|
|
53
|
+
raise ValueError(
|
|
54
|
+
f"Invalid value in 'Cases' section: {case_name} = {case_owned}"
|
|
55
|
+
)
|
|
56
|
+
for capsule_section in CAPSULE_INFO:
|
|
57
|
+
for capsule_name, capsule_owned in self.items(capsule_section):
|
|
58
|
+
if int(capsule_owned) < 0:
|
|
59
|
+
raise ValueError(
|
|
60
|
+
f"Invalid value in '{capsule_section}' section: {capsule_name} = {capsule_owned}"
|
|
61
|
+
)
|
|
62
|
+
except ValueError as error:
|
|
63
|
+
if "Invalid " in str(error):
|
|
64
|
+
raise
|
|
65
|
+
raise ValueError("Invalid value type. All values must be integers.") from error
|
|
66
|
+
|
|
67
|
+
def _validate_config(self):
|
|
68
|
+
"""
|
|
69
|
+
Validate the configuration file to ensure all required sections exist with the
|
|
70
|
+
right values.
|
|
71
|
+
|
|
72
|
+
:raises ValueError: If any required section is missing or if any value is
|
|
73
|
+
invalid.
|
|
74
|
+
"""
|
|
75
|
+
try:
|
|
76
|
+
self._validate_config_sections()
|
|
77
|
+
self._validate_config_values()
|
|
78
|
+
self.valid = True
|
|
79
|
+
except ValueError as error:
|
|
80
|
+
console.print(f"[bold red][!] Config error: {error}")
|
|
81
|
+
self.valid = False
|
|
82
|
+
self.last_error = error
|
|
83
|
+
|
|
84
|
+
def write_to_file(self):
|
|
85
|
+
"""Write the current configuration to the configuration file."""
|
|
86
|
+
self._validate_config()
|
|
87
|
+
|
|
88
|
+
if self.valid:
|
|
89
|
+
with open(CONFIG_FILE, "w", encoding="utf-8") as config_file:
|
|
90
|
+
self.write(config_file)
|
|
91
|
+
|
|
92
|
+
def toggle_use_proxy(self, enabled: bool):
|
|
93
|
+
"""
|
|
94
|
+
Toggle the use of proxies for requests. This will update the configuration file.
|
|
95
|
+
|
|
96
|
+
:param enabled: If True, proxies will be used; if False, they will not be used.
|
|
97
|
+
"""
|
|
98
|
+
self.set("App Settings", "use_proxy", str(enabled))
|
|
99
|
+
self.write_to_file()
|
|
100
|
+
|
|
101
|
+
console.print(
|
|
102
|
+
f"[bold green]{'[+] Enabled' if enabled else '[-] Disabled'} proxy usage for requests."
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def toggle_discord_webhook(self, enabled: bool):
|
|
106
|
+
"""
|
|
107
|
+
Toggle the use of a Discord webhook to notify users of price calculations.
|
|
108
|
+
|
|
109
|
+
:param enabled: If True, the webhook will be used; if False, it will not be
|
|
110
|
+
used.
|
|
111
|
+
"""
|
|
112
|
+
self.set("App Settings", "discord_notifications", str(enabled))
|
|
113
|
+
self.write_to_file()
|
|
114
|
+
|
|
115
|
+
console.print(
|
|
116
|
+
f"[bold green]{'[+] Enabled' if enabled else '[-] Disabled'} Discord webhook notifications."
|
|
117
|
+
)
|