eaf-base-api 2.2.5__tar.gz → 2.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/PKG-INFO +3 -2
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/base_api/base.py +247 -81
- eaf_base_api-2.2.7/base_api/modules/config.py +36 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/eaf_base_api.egg-info/PKG-INFO +3 -2
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/eaf_base_api.egg-info/SOURCES.txt +1 -1
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/setup.py +1 -1
- eaf_base_api-2.2.5/base_api/modules/consts.py +0 -16
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/LICENSE +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/README.md +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/base_api/__init__.py +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/base_api/modules/__init__.py +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/base_api/modules/progress_bars.py +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/eaf_base_api.egg-info/dependency_links.txt +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/eaf_base_api.egg-info/requires.txt +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/eaf_base_api.egg-info/top_level.txt +0 -0
- {eaf_base_api-2.2.5 → eaf_base_api-2.2.7}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: eaf_base_api
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.7
|
|
4
4
|
Summary: A base API for EchterAlsFake's Porn APIs
|
|
5
5
|
Home-page: https://github.com/EchterAlsFake/eaf_base_api
|
|
6
6
|
Author: Johannes Habel
|
|
@@ -20,6 +20,7 @@ Dynamic: description
|
|
|
20
20
|
Dynamic: description-content-type
|
|
21
21
|
Dynamic: home-page
|
|
22
22
|
Dynamic: license
|
|
23
|
+
Dynamic: license-file
|
|
23
24
|
Dynamic: requires-dist
|
|
24
25
|
Dynamic: summary
|
|
25
26
|
|
|
@@ -1,12 +1,15 @@
|
|
|
1
|
+
import re
|
|
1
2
|
import os
|
|
3
|
+
import sys
|
|
2
4
|
import time
|
|
3
5
|
import m3u8
|
|
6
|
+
import json
|
|
4
7
|
import httpx
|
|
5
8
|
import random
|
|
6
|
-
import string
|
|
7
9
|
import logging
|
|
8
10
|
import traceback
|
|
9
11
|
import threading
|
|
12
|
+
import http.client
|
|
10
13
|
|
|
11
14
|
from typing import Union
|
|
12
15
|
from functools import lru_cache
|
|
@@ -15,29 +18,137 @@ from ffmpeg_progress_yield import FfmpegProgress
|
|
|
15
18
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
16
19
|
|
|
17
20
|
try:
|
|
18
|
-
from modules import
|
|
21
|
+
from modules import config
|
|
19
22
|
from modules.progress_bars import Callback
|
|
20
23
|
|
|
21
24
|
except (ModuleNotFoundError, ImportError):
|
|
22
|
-
from .modules import
|
|
25
|
+
from .modules import config
|
|
23
26
|
from .modules.progress_bars import Callback
|
|
24
27
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
+
loggers = {}
|
|
29
|
+
|
|
30
|
+
# Default formatter
|
|
31
|
+
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def is_android():
|
|
35
|
+
"""Detects if the script is running on an Android device."""
|
|
36
|
+
return "ANDROID_ROOT" in os.environ and "ANDROID_DATA" in os.environ
|
|
37
|
+
|
|
38
|
+
def get_log_file_path(filename="app.log"):
|
|
39
|
+
"""Returns a valid log file path that works on Android and other OS."""
|
|
40
|
+
if is_android():
|
|
41
|
+
return os.path.join(os.environ["HOME"], filename) # Internal app storage
|
|
42
|
+
return filename # Default for Linux, Windows, Mac
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# Define ANSI color codes for different log levels
|
|
46
|
+
LOG_COLORS = {
|
|
47
|
+
'DEBUG': "\033[37m", # White
|
|
48
|
+
'INFO': "\033[34m", # Blue
|
|
49
|
+
'WARNING': "\033[33m", # Yellow
|
|
50
|
+
'ERROR': "\033[31m", # Red
|
|
51
|
+
'CRITICAL': "\033[41m" # Red background
|
|
52
|
+
}
|
|
53
|
+
RESET_COLOR = "\033[0m"
|
|
54
|
+
|
|
55
|
+
class ColoredFormatter(logging.Formatter):
|
|
56
|
+
def format(self, record):
|
|
57
|
+
log_message = super().format(record)
|
|
58
|
+
color = LOG_COLORS.get(record.levelname, RESET_COLOR)
|
|
59
|
+
return f"{color}{log_message}{RESET_COLOR}"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def send_log_message(ip, port, message):
|
|
63
|
+
"""Sends a log message to a remote server via HTTP."""
|
|
64
|
+
try:
|
|
65
|
+
conn = http.client.HTTPConnection(ip, port, timeout=2)
|
|
66
|
+
headers = {'Content-Type': 'application/json'}
|
|
67
|
+
data = json.dumps({"message": message})
|
|
68
|
+
conn.request("POST", "/log", body=data, headers=headers)
|
|
69
|
+
conn.getresponse() # Read response to ensure completion
|
|
70
|
+
conn.close()
|
|
71
|
+
except Exception as e:
|
|
72
|
+
print(f"Failed to send log to {ip}:{port} - {e}", file=sys.stderr)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class HTTPLogHandler(logging.Handler):
|
|
76
|
+
"""Custom log handler that sends logs to a remote HTTP server."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, ip, port):
|
|
79
|
+
super().__init__()
|
|
80
|
+
self.ip = ip
|
|
81
|
+
self.port = port
|
|
82
|
+
|
|
83
|
+
def emit(self, record):
|
|
84
|
+
log_entry = self.format(record)
|
|
85
|
+
send_log_message(self.ip, self.port, log_entry)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def setup_logger(name, log_file=None, level=logging.CRITICAL, http_ip=None, http_port=None):
|
|
89
|
+
"""Creates or updates a logger for a specific module."""
|
|
90
|
+
if name in loggers:
|
|
91
|
+
logger = loggers[name]
|
|
92
|
+
logger.setLevel(level)
|
|
93
|
+
|
|
94
|
+
file_handler_exists = any(isinstance(h, logging.FileHandler) for h in logger.handlers)
|
|
95
|
+
http_handler_exists = any(isinstance(h, HTTPLogHandler) for h in logger.handlers)
|
|
96
|
+
|
|
97
|
+
if log_file and not file_handler_exists:
|
|
98
|
+
log_file = get_log_file_path(log_file)
|
|
99
|
+
fh = logging.FileHandler(log_file, mode='a')
|
|
100
|
+
fh.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
|
|
101
|
+
logger.addHandler(fh)
|
|
102
|
+
|
|
103
|
+
if http_ip and http_port and not http_handler_exists:
|
|
104
|
+
http_handler = HTTPLogHandler(http_ip, http_port)
|
|
105
|
+
http_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
|
|
106
|
+
logger.addHandler(http_handler)
|
|
107
|
+
|
|
108
|
+
return logger
|
|
109
|
+
|
|
110
|
+
logger = logging.getLogger(name)
|
|
111
|
+
logger.setLevel(level)
|
|
112
|
+
|
|
113
|
+
if not any(isinstance(h, logging.StreamHandler) for h in logger.handlers):
|
|
114
|
+
ch = logging.StreamHandler()
|
|
115
|
+
ch.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
|
|
116
|
+
logger.addHandler(ch)
|
|
117
|
+
|
|
118
|
+
if log_file:
|
|
119
|
+
log_file = get_log_file_path(log_file)
|
|
120
|
+
if not hasattr(sys, '_first_run'):
|
|
121
|
+
sys._first_run = True
|
|
122
|
+
file_mode = 'w'
|
|
123
|
+
else:
|
|
124
|
+
file_mode = 'a'
|
|
125
|
+
fh = logging.FileHandler(log_file, mode=file_mode)
|
|
126
|
+
fh.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
|
|
127
|
+
logger.addHandler(fh)
|
|
28
128
|
|
|
29
|
-
|
|
30
|
-
|
|
129
|
+
if http_ip and http_port:
|
|
130
|
+
http_handler = HTTPLogHandler(http_ip, http_port)
|
|
131
|
+
http_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
|
|
132
|
+
logger.addHandler(http_handler)
|
|
31
133
|
|
|
134
|
+
loggers[name] = logger
|
|
135
|
+
return logger
|
|
32
136
|
|
|
33
137
|
class Cache:
|
|
34
138
|
"""
|
|
35
139
|
Caches content from network requests
|
|
36
140
|
"""
|
|
37
141
|
|
|
38
|
-
def __init__(self):
|
|
142
|
+
def __init__(self, config):
|
|
39
143
|
self.cache_dictionary = {}
|
|
40
144
|
self.lock = threading.Lock()
|
|
145
|
+
self.logger = setup_logger("BASE API - [Cache]", level=logging.CRITICAL)
|
|
146
|
+
|
|
147
|
+
def enable_logging(self, log_file=None, level=logging.DEBUG):
|
|
148
|
+
"""
|
|
149
|
+
Enables logging dynamically for this module.
|
|
150
|
+
"""
|
|
151
|
+
self.logger = setup_logger(name="BASE API - [Cache]", log_file=log_file, level=level)
|
|
41
152
|
|
|
42
153
|
def handle_cache(self, url):
|
|
43
154
|
if url is None:
|
|
@@ -49,137 +160,191 @@ class Cache:
|
|
|
49
160
|
|
|
50
161
|
def save_cache(self, url, content):
|
|
51
162
|
with self.lock:
|
|
52
|
-
if len(self.cache_dictionary.keys()) >=
|
|
163
|
+
if len(self.cache_dictionary.keys()) >= config.config.MAX_CACHE_ITEMS:
|
|
53
164
|
first_key = next(iter(self.cache_dictionary))
|
|
54
165
|
# Delete the first item
|
|
55
166
|
del self.cache_dictionary[first_key]
|
|
56
|
-
logger.info(f"Deleting: {first_key} from cache, due to caching limits...")
|
|
167
|
+
self.logger.info(f"Deleting: {first_key} from cache, due to caching limits...")
|
|
57
168
|
|
|
58
169
|
self.cache_dictionary[url] = content
|
|
59
170
|
|
|
60
|
-
cache = Cache()
|
|
61
|
-
|
|
62
|
-
|
|
63
171
|
class BaseCore:
|
|
64
172
|
"""
|
|
65
173
|
The base class which has all necessary functions for other API packages
|
|
66
174
|
"""
|
|
67
|
-
def __init__(self):
|
|
175
|
+
def __init__(self, config=config.config):
|
|
68
176
|
self.last_request_time = time.time()
|
|
69
177
|
self.total_requests = 0 # Tracks how many requests have been made
|
|
70
178
|
self.session = None
|
|
179
|
+
self.kill_switch = False
|
|
180
|
+
self.config = config
|
|
181
|
+
self.cache = Cache(self.config)
|
|
71
182
|
self.initialize_session()
|
|
183
|
+
self.logger = setup_logger("BASE API - [BaseCore]", log_file=False, level=logging.ERROR)
|
|
72
184
|
|
|
185
|
+
def enable_logging(self, log_file=None, level=logging.DEBUG):
|
|
186
|
+
"""
|
|
187
|
+
Enables logging dynamically for this module.
|
|
188
|
+
"""
|
|
189
|
+
self.logger = setup_logger(name="BASE API - [BaseCore]", log_file=log_file, level=level)
|
|
190
|
+
self.cache.logger = setup_logger(name="BASE API - [Cache]", log_file=log_file, level=level)
|
|
73
191
|
|
|
74
192
|
def update_user_agent(self):
|
|
75
193
|
"""Updates the User-Agent"""
|
|
76
|
-
self.
|
|
194
|
+
self.config.rotate_user_agent()
|
|
195
|
+
self.session.headers.update({"User-Agent": self.config.headers["User-Agent"]})
|
|
196
|
+
|
|
197
|
+
def enable_kill_switch(self):
|
|
198
|
+
"""This is a function that will check and verify if your proxy is working before every request.
|
|
199
|
+
As soon as there's a mismatch in the actual response IP and the proxy IP, the program will exit."""
|
|
200
|
+
self.kill_switch = True
|
|
201
|
+
|
|
202
|
+
def check_kill_switch(self):
|
|
203
|
+
proxy_ip = self.config.PROXY
|
|
204
|
+
pattern = re.compile(
|
|
205
|
+
r'^(?P<scheme>http|socks5)://'
|
|
206
|
+
r'(?:(?:\w+:\w+)@)?' # optional user:pass@
|
|
207
|
+
r'(?P<host>[a-zA-Z0-9.-]+)'
|
|
208
|
+
r':(?P<port>\d{1,5})$'
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
match = pattern.match(proxy_ip
|
|
212
|
+
)
|
|
213
|
+
if match:
|
|
214
|
+
self.logger.info(f"Proxy has valid scheme: {match.group('scheme')} ")
|
|
215
|
+
|
|
216
|
+
else:
|
|
217
|
+
self.logger.critical("Proxy is INVALID! Exiting.")
|
|
218
|
+
|
|
219
|
+
self.logger.info("Checking if proxy is working...")
|
|
220
|
+
self.logger.info("Doing request to httpbin.org to get your real IP to compare it with another request with activated proxy")
|
|
77
221
|
|
|
78
|
-
|
|
79
|
-
|
|
222
|
+
your_ip = httpx.Client().get("https://httpbin.org/ip").json()["origin"]
|
|
223
|
+
self.logger.info(f"Your IP is: {your_ip}")
|
|
80
224
|
|
|
81
|
-
|
|
82
|
-
|
|
225
|
+
self.logger.info("Doing request with Proxy on...")
|
|
226
|
+
proxy_ip_ = self.fetch("https://httpbin.org/ip", bypass_kill_switch=True, get_response=True).json()["origin"]
|
|
227
|
+
self.logger.info(f"Proxy IP is: {proxy_ip_}")
|
|
83
228
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
229
|
+
if your_ip == proxy_ip_:
|
|
230
|
+
self.logger.critical("IP is the same on both requests... Proxy is not working, exiting!")
|
|
231
|
+
raise "CRITICAL PROXY ERROR, CHECK LOGS!"
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def initialize_session(self, verify=True): # Disable SSL verification only if you really need it....
|
|
235
|
+
self.session = httpx.Client(
|
|
236
|
+
proxy=self.config.PROXY,
|
|
237
|
+
headers=self.config.HEADERS,
|
|
238
|
+
timeout=self.config.TIMEOUT,
|
|
239
|
+
follow_redirects=True,
|
|
240
|
+
verify=verify
|
|
241
|
+
)
|
|
89
242
|
|
|
90
243
|
def enforce_delay(self):
|
|
91
244
|
"""Enforces the specified delay in consts.REQUEST_DELAY"""
|
|
92
|
-
delay =
|
|
245
|
+
delay = self.config.REQUEST_DELAY
|
|
93
246
|
if delay > 0:
|
|
94
247
|
time_since_last_request = time.time() - self.last_request_time
|
|
95
|
-
logger.debug(f"Time since last request: {time_since_last_request:.2f} seconds.")
|
|
248
|
+
self.logger.debug(f"Time since last request: {time_since_last_request:.2f} seconds.")
|
|
96
249
|
if time_since_last_request < delay:
|
|
97
250
|
sleep_time = delay - time_since_last_request
|
|
98
|
-
logger.debug(f"Enforcing delay of {sleep_time:.2f} seconds.")
|
|
251
|
+
self.logger.debug(f"Enforcing delay of {sleep_time:.2f} seconds.")
|
|
99
252
|
time.sleep(sleep_time)
|
|
100
253
|
|
|
101
254
|
def fetch(
|
|
102
255
|
self,
|
|
103
256
|
url: str,
|
|
104
257
|
get_bytes: bool = False,
|
|
105
|
-
timeout: int =
|
|
258
|
+
timeout: int = None,
|
|
106
259
|
get_response: bool = False,
|
|
107
260
|
save_cache: bool = True,
|
|
108
261
|
cookies: dict = None,
|
|
109
262
|
allow_redirects: bool = True,
|
|
263
|
+
bypass_kill_switch: bool = False # prevents infinite loop
|
|
110
264
|
) -> Union[bytes, str, httpx.Response, None]:
|
|
111
265
|
"""
|
|
112
266
|
Fetches content in UTF-8 Text, Bytes, or as a stream using multiple request attempts,
|
|
113
267
|
support for proxies and custom timeout.
|
|
114
268
|
"""
|
|
115
269
|
# Check cache first
|
|
116
|
-
|
|
117
|
-
if
|
|
118
|
-
|
|
270
|
+
|
|
271
|
+
if timeout is None:
|
|
272
|
+
timeout = self.config.timeout # pls don't ask thanks
|
|
273
|
+
|
|
274
|
+
content = self.cache.handle_cache(url)
|
|
275
|
+
if content:
|
|
276
|
+
self.logger.info(f"Fetched content for: {url} from cache!")
|
|
119
277
|
return content
|
|
120
278
|
|
|
121
|
-
for attempt in range(1,
|
|
279
|
+
for attempt in range(1, self.config.MAX_RETRIES + 1):
|
|
122
280
|
if attempt != 1:
|
|
123
281
|
time.sleep(1.5) # Sleeping for 1.5 seconds to minimize site overload when doing a lot of requests
|
|
124
282
|
|
|
125
283
|
try:
|
|
126
284
|
# Update user agent periodically
|
|
127
|
-
if self.total_requests %
|
|
285
|
+
if self.total_requests % 30 == 0: # Changing all 30 attempts
|
|
128
286
|
self.update_user_agent()
|
|
129
287
|
|
|
130
288
|
self.enforce_delay()
|
|
131
289
|
|
|
290
|
+
if self.kill_switch and not bypass_kill_switch:
|
|
291
|
+
self.check_kill_switch()
|
|
292
|
+
|
|
132
293
|
# Perform the request with stream handling
|
|
133
|
-
|
|
134
|
-
follow_redirects=allow_redirects)
|
|
135
|
-
|
|
294
|
+
response = self.session.get(url, timeout=timeout, cookies=cookies,
|
|
295
|
+
follow_redirects=allow_redirects)
|
|
296
|
+
self.total_requests += 1
|
|
136
297
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
298
|
+
# Log and handle non-200 status codes
|
|
299
|
+
if response.status_code != 200:
|
|
300
|
+
self.logger.warning(
|
|
301
|
+
f"Attempt {attempt}: Unexpected status code {response.status_code} for URL: {url}")
|
|
141
302
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
303
|
+
if response.status_code == 404:
|
|
304
|
+
self.logger.error("Resource not found (404). This may indicate the content is unavailable.")
|
|
305
|
+
return None # Return None for unavailable resources
|
|
145
306
|
|
|
146
|
-
|
|
307
|
+
elif response.status_code == 403 and attempt >= 2:
|
|
308
|
+
self.logger.error(f"The website rejected access after {attempt} tries. Aborting!")
|
|
309
|
+
return None # Return None for forbidden resources
|
|
147
310
|
|
|
148
|
-
|
|
311
|
+
continue # Retry for other non-200 status codes
|
|
149
312
|
|
|
150
|
-
|
|
151
|
-
if get_response:
|
|
152
|
-
return response
|
|
313
|
+
self.logger.debug(f"Attempt {attempt}: Successfully fetched URL: {url}")
|
|
153
314
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
315
|
+
# Return response if requested
|
|
316
|
+
if get_response:
|
|
317
|
+
return response
|
|
157
318
|
|
|
158
|
-
|
|
159
|
-
|
|
319
|
+
# Process and return content
|
|
320
|
+
# Collecting all chunks before processing because some sites cause issues with real-time decoding
|
|
321
|
+
raw_content = response.content
|
|
160
322
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
323
|
+
if get_bytes:
|
|
324
|
+
content = raw_content
|
|
325
|
+
|
|
326
|
+
else:
|
|
327
|
+
try:
|
|
328
|
+
content = raw_content.decode("utf-8")
|
|
164
329
|
|
|
165
|
-
|
|
166
|
-
|
|
330
|
+
except UnicodeDecodeError:
|
|
331
|
+
content = raw_content.decode("latin1") # Fallback, hope that works somehow idk
|
|
167
332
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
333
|
+
if save_cache and not get_bytes:
|
|
334
|
+
self.logger.debug(f"Saving content of {url} to local cache.")
|
|
335
|
+
self.cache.save_cache(url, content)
|
|
171
336
|
|
|
172
|
-
|
|
337
|
+
return content
|
|
173
338
|
|
|
174
339
|
except httpx.RequestError as e:
|
|
175
|
-
logger.error(f"Attempt {attempt}: Request error for URL {url}: {e}")
|
|
340
|
+
self.logger.error(f"Attempt {attempt}: Request error for URL {url}: {e}")
|
|
176
341
|
|
|
177
342
|
except Exception:
|
|
178
|
-
logger.error(f"Attempt {attempt}: Unexpected error for URL {url}: {traceback.format_exc()}")
|
|
343
|
+
self.logger.error(f"Attempt {attempt}: Unexpected error for URL {url}: {traceback.format_exc()}")
|
|
179
344
|
|
|
180
|
-
logger.info(f"Retrying ({attempt}/{
|
|
345
|
+
self.logger.info(f"Retrying ({attempt}/{self.config.MAX_RETRIES}) for URL: {url}")
|
|
181
346
|
|
|
182
|
-
logger.error(f"Failed to fetch URL {url} after {
|
|
347
|
+
self.logger.error(f"Failed to fetch URL {url} after {self.config.MAX_RETRIES} attempts.")
|
|
183
348
|
return None # Return None if all attempts fail
|
|
184
349
|
|
|
185
350
|
@classmethod
|
|
@@ -188,9 +353,11 @@ class BaseCore:
|
|
|
188
353
|
:param title:
|
|
189
354
|
:return: str: strips out non UTF-8 chars of the title
|
|
190
355
|
"""
|
|
191
|
-
illegal_chars = '
|
|
192
|
-
|
|
193
|
-
|
|
356
|
+
illegal_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*']
|
|
357
|
+
for char in illegal_chars:
|
|
358
|
+
title = title.replace(char, "")
|
|
359
|
+
|
|
360
|
+
return title
|
|
194
361
|
|
|
195
362
|
@lru_cache(maxsize=250)
|
|
196
363
|
def get_m3u8_by_quality(self, m3u8_url: str, quality: str) -> str:
|
|
@@ -198,7 +365,7 @@ class BaseCore:
|
|
|
198
365
|
m3u8 playlist"""
|
|
199
366
|
playlist_content = self.fetch(url=m3u8_url)
|
|
200
367
|
playlist = m3u8.loads(playlist_content)
|
|
201
|
-
logger.debug(f"Resolved m3u8 playlist: {m3u8_url}")
|
|
368
|
+
self.logger.debug(f"Resolved m3u8 playlist: {m3u8_url}")
|
|
202
369
|
|
|
203
370
|
if not playlist.is_variant:
|
|
204
371
|
raise ValueError("Provided URL is not a master playlist.")
|
|
@@ -235,7 +402,7 @@ class BaseCore:
|
|
|
235
402
|
def get_segments(self, m3u8_url_master: str, quality: str) -> list:
|
|
236
403
|
"""Gets all video segments from a given quality and the primary m3u8 URL"""
|
|
237
404
|
m3u8_url = self.get_m3u8_by_quality(m3u8_url=m3u8_url_master, quality=quality)
|
|
238
|
-
logger.debug(f"Trying to fetch segment from m3u8 ->: {m3u8_url}")
|
|
405
|
+
self.logger.debug(f"Trying to fetch segment from m3u8 ->: {m3u8_url}")
|
|
239
406
|
content = self.fetch(url=m3u8_url)
|
|
240
407
|
segments_ = m3u8.loads(content).segments
|
|
241
408
|
segments = []
|
|
@@ -243,7 +410,7 @@ class BaseCore:
|
|
|
243
410
|
for segment in segments_:
|
|
244
411
|
segments.append(urljoin(m3u8_url, segment.uri)) # Get the full URL path to segment
|
|
245
412
|
|
|
246
|
-
logger.debug(f"Fetched {len(segments)} segments from m3u8 URL")
|
|
413
|
+
self.logger.debug(f"Fetched {len(segments)} segments from m3u8 URL")
|
|
247
414
|
return segments
|
|
248
415
|
|
|
249
416
|
def download(self, video, quality: str, downloader: str, path: str, callback=None) -> None:
|
|
@@ -269,14 +436,13 @@ class BaseCore:
|
|
|
269
436
|
elif downloader == "FFMPEG":
|
|
270
437
|
self.FFMPEG(video=video, quality=quality, path=path, callback=callback)
|
|
271
438
|
|
|
272
|
-
|
|
273
|
-
def download_segment(cls, url: str, timeout: int) -> tuple[str, bytes, bool]:
|
|
439
|
+
def download_segment(self, url: str, timeout: int) -> tuple[str, bytes, bool]:
|
|
274
440
|
"""
|
|
275
441
|
Attempt to download a single segment, retrying on failure.
|
|
276
442
|
Returns a tuple of the URL, content (empty if failed after retries), and a success flag.
|
|
277
443
|
"""
|
|
278
444
|
|
|
279
|
-
content =
|
|
445
|
+
content = self.fetch(url, timeout=timeout, get_bytes=True)
|
|
280
446
|
return url, content, True # Success
|
|
281
447
|
|
|
282
448
|
def threaded(self, max_workers: int = 20, timeout: int = 10):
|
|
@@ -307,7 +473,7 @@ class BaseCore:
|
|
|
307
473
|
successful_downloads += 1
|
|
308
474
|
callback(completed, length) # Update progress callback
|
|
309
475
|
except Exception as e:
|
|
310
|
-
logger.error(f"Error processing segment {hls_part}: {e}")
|
|
476
|
+
self.logger.error(f"Error processing segment {hls_part}: {e}")
|
|
311
477
|
|
|
312
478
|
# Write only successful segments to the output file
|
|
313
479
|
with open(path, 'wb') as file:
|
|
@@ -323,7 +489,7 @@ class BaseCore:
|
|
|
323
489
|
if success:
|
|
324
490
|
file.write(data)
|
|
325
491
|
except Exception as e:
|
|
326
|
-
logger.error(f"Exception writing segment {segment_url}: {e}")
|
|
492
|
+
self.logger.error(f"Exception writing segment {segment_url}: {e}")
|
|
327
493
|
|
|
328
494
|
return wrapper
|
|
329
495
|
|
|
@@ -354,7 +520,7 @@ class BaseCore:
|
|
|
354
520
|
|
|
355
521
|
# Build the command for FFMPEG as a list directly
|
|
356
522
|
command = [
|
|
357
|
-
|
|
523
|
+
self.config.FFMPEG_PATH,
|
|
358
524
|
"-i", new_url, # Input URL
|
|
359
525
|
"-bsf:a", "aac_adtstoasc",
|
|
360
526
|
"-y", # Overwrite output files without asking
|
|
@@ -408,10 +574,10 @@ class BaseCore:
|
|
|
408
574
|
return True
|
|
409
575
|
|
|
410
576
|
except httpx.RequestError as e:
|
|
411
|
-
logger.error(f"Request failed for URL {url}: {e}")
|
|
577
|
+
self.logger.error(f"Request failed for URL {url}: {e}")
|
|
412
578
|
return False
|
|
413
579
|
except Exception as e:
|
|
414
|
-
logger.error(f"Unexpected error during download: {e}")
|
|
580
|
+
self.logger.error(f"Unexpected error during download: {e}")
|
|
415
581
|
return False
|
|
416
582
|
|
|
417
583
|
@classmethod
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import random
|
|
2
|
+
|
|
3
|
+
class RuntimeConfig:
|
|
4
|
+
def __init__(self):
|
|
5
|
+
self.max_cache_items = 200
|
|
6
|
+
self.max_retries = 4
|
|
7
|
+
self.request_delay = 0
|
|
8
|
+
self.timeout = 20
|
|
9
|
+
self.ffmpeg_path = "ffmpeg"
|
|
10
|
+
self.proxy = None
|
|
11
|
+
self.verify_ssl = True
|
|
12
|
+
self.user_agents = [
|
|
13
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
14
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.1",
|
|
15
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3",
|
|
16
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3",
|
|
17
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0."
|
|
18
|
+
]
|
|
19
|
+
self.cookies = None
|
|
20
|
+
self.headers = {
|
|
21
|
+
"User-Agent": random.choice(self.user_agents)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
def rotate_user_agent(self):
|
|
25
|
+
self.headers["User-Agent"] = random.choice(self.user_agents)
|
|
26
|
+
|
|
27
|
+
def as_httpx_kwargs(self):
|
|
28
|
+
return {
|
|
29
|
+
"proxies": self.proxy,
|
|
30
|
+
"verify": self.verify_ssl,
|
|
31
|
+
"headers": self.headers,
|
|
32
|
+
"timeout": self.timeout
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
# Singleton instance needed for my Porn Fetch project
|
|
36
|
+
config = RuntimeConfig()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: eaf_base_api
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.7
|
|
4
4
|
Summary: A base API for EchterAlsFake's Porn APIs
|
|
5
5
|
Home-page: https://github.com/EchterAlsFake/eaf_base_api
|
|
6
6
|
Author: Johannes Habel
|
|
@@ -20,6 +20,7 @@ Dynamic: description
|
|
|
20
20
|
Dynamic: description-content-type
|
|
21
21
|
Dynamic: home-page
|
|
22
22
|
Dynamic: license
|
|
23
|
+
Dynamic: license-file
|
|
23
24
|
Dynamic: requires-dist
|
|
24
25
|
Dynamic: summary
|
|
25
26
|
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import random
|
|
2
|
-
|
|
3
|
-
MAX_CACHE_ITEMS = 200 # Lower this value if you are on a low-ram system (<=4 GB)
|
|
4
|
-
MAX_RETRIES = 4
|
|
5
|
-
REQUEST_DELAY = 0
|
|
6
|
-
TIMEOUT = 20 # Maximal request timeout
|
|
7
|
-
FFMPEG_PATH = "ffmpeg"
|
|
8
|
-
PROXY = None
|
|
9
|
-
USER_AGENTS = ["Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
10
|
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.1",
|
|
11
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3",
|
|
12
|
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3",
|
|
13
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0."]
|
|
14
|
-
|
|
15
|
-
HEADERS = {"User-Agent": random.choice(USER_AGENTS)}
|
|
16
|
-
COOKIES = None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|