ryry-cli 6.22__tar.gz → 6.23__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ryry_cli-6.22/ryry_cli.egg-info → ryry_cli-6.23}/PKG-INFO +1 -1
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/constant.py +2 -2
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/ryry_service.py +67 -17
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/store.py +90 -10
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/upload.py +79 -87
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/utils.py +132 -5
- {ryry_cli-6.22 → ryry_cli-6.23/ryry_cli.egg-info}/PKG-INFO +1 -1
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry_cli.egg-info/SOURCES.txt +3 -1
- {ryry_cli-6.22 → ryry_cli-6.23}/setup.py +1 -1
- ryry_cli-6.23/tests/test_network_profile.py +137 -0
- ryry_cli-6.23/tests/test_upload_proxy.py +175 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/LICENSE +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/MANIFEST.in +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/README.md +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/__init__.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/assets/fonts/OFL.txt +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/assets/fonts/ryry-report-cjk-bold.otf +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/daemon_base.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/daemon_manager.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/main.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/proxy_manager.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/ryry_server_socket.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/ryry_webapi.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/ryry_widget.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/script_template/__init__.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/script_template/daemon.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/script_template/main.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/script_template/run.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/server_func.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/shared_memory.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/task.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry/taskUtils.py +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry_cli.egg-info/dependency_links.txt +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry_cli.egg-info/entry_points.txt +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry_cli.egg-info/requires.txt +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/ryry_cli.egg-info/top_level.txt +0 -0
- {ryry_cli-6.22 → ryry_cli-6.23}/setup.cfg +0 -0
|
@@ -160,21 +160,72 @@ class ryryDaemonThread(Thread):
|
|
|
160
160
|
def markStop(self):
|
|
161
161
|
print(f" DaemonThread waiting stop")
|
|
162
162
|
|
|
163
|
-
class ryryStateThread(Thread):
|
|
164
|
-
def __init__(self):
|
|
165
|
-
super().__init__()
|
|
166
|
-
self.name = f"ryryStateThread"
|
|
167
|
-
self.daemon = True
|
|
168
|
-
self.tik_time = 30.0
|
|
169
|
-
self.
|
|
163
|
+
class ryryStateThread(Thread):
|
|
164
|
+
def __init__(self):
|
|
165
|
+
super().__init__()
|
|
166
|
+
self.name = f"ryryStateThread"
|
|
167
|
+
self.daemon = True
|
|
168
|
+
self.tik_time = 30.0
|
|
169
|
+
self.next_network_refresh_at = 0
|
|
170
|
+
self.start()
|
|
171
|
+
|
|
172
|
+
def refreshNetworkProfileIfDue(self):
|
|
173
|
+
try:
|
|
174
|
+
now = int(time.time())
|
|
175
|
+
if now < self.next_network_refresh_at:
|
|
176
|
+
return
|
|
177
|
+
|
|
178
|
+
profile = store.getNetworkProfile()
|
|
179
|
+
if not store.isNetworkProfileRefreshDue(profile, now):
|
|
180
|
+
self.next_network_refresh_at = int(profile.get("next_refresh_at", now + self.tik_time))
|
|
181
|
+
return
|
|
182
|
+
|
|
183
|
+
self.next_network_refresh_at = now + store.NETWORK_PROFILE_RETRY_SECONDS
|
|
184
|
+
detected = utils.detectNetworkProfile()
|
|
185
|
+
updated_profile = dict(profile)
|
|
186
|
+
updated_profile.update({
|
|
187
|
+
"version": 2,
|
|
188
|
+
"last_attempt_at": now,
|
|
189
|
+
"baidu_ms": detected.get("baidu_ms"),
|
|
190
|
+
"google_ms": detected.get("google_ms"),
|
|
191
|
+
"r2_ms": detected.get("r2_ms"),
|
|
192
|
+
"aliyun_ms": detected.get("aliyun_ms"),
|
|
193
|
+
"country": detected.get("country"),
|
|
194
|
+
"method": detected.get("method", "inconclusive"),
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
is_oversea = detected.get("is_oversea")
|
|
198
|
+
if isinstance(is_oversea, bool):
|
|
199
|
+
updated_profile.update({
|
|
200
|
+
"is_oversea": is_oversea,
|
|
201
|
+
"checked_at": now,
|
|
202
|
+
"last_success_at": now,
|
|
203
|
+
"next_refresh_at": now + store.NETWORK_PROFILE_REFRESH_SECONDS,
|
|
204
|
+
"failure_count": 0,
|
|
205
|
+
})
|
|
206
|
+
updated_profile.pop("last_error", None)
|
|
207
|
+
self.next_network_refresh_at = updated_profile["next_refresh_at"]
|
|
208
|
+
else:
|
|
209
|
+
updated_profile.update({
|
|
210
|
+
"next_refresh_at": now + store.NETWORK_PROFILE_RETRY_SECONDS,
|
|
211
|
+
"failure_count": int(updated_profile.get("failure_count", 0)) + 1,
|
|
212
|
+
"last_error": "network detection inconclusive",
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
if not store.saveNetworkProfile(updated_profile):
|
|
216
|
+
self.next_network_refresh_at = now + store.NETWORK_PROFILE_RETRY_SECONDS
|
|
217
|
+
except Exception:
|
|
218
|
+
self.next_network_refresh_at = int(time.time()) + store.NETWORK_PROFILE_RETRY_SECONDS
|
|
170
219
|
|
|
171
|
-
def run(self):
|
|
172
|
-
print(f" {self.name} start")
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
220
|
+
def run(self):
|
|
221
|
+
print(f" {self.name} start")
|
|
222
|
+
self.refreshNetworkProfileIfDue()
|
|
223
|
+
taskUtils.onlineNotify()
|
|
224
|
+
while (os.path.exists(stop_thread_file) == False):
|
|
225
|
+
time.sleep(self.tik_time)
|
|
226
|
+
try:
|
|
227
|
+
self.refreshNetworkProfileIfDue()
|
|
228
|
+
task_config = TaskConnector._getTaskConfig()
|
|
178
229
|
if task_config["last_task_pts"] > 0:
|
|
179
230
|
cnt = (calendar.timegm(time.gmtime()) - task_config["last_task_pts"]) #second
|
|
180
231
|
if cnt >= (60*60) and cnt/(60*60)%1 <= self.tik_time/3600:
|
|
@@ -204,6 +255,5 @@ class ryryStateThread(Thread):
|
|
|
204
255
|
time.sleep(60)
|
|
205
256
|
print(f" StateChecker stop")
|
|
206
257
|
notify_other_stoped() #because other thread is waiting some signal to close
|
|
207
|
-
def markStop(self):
|
|
208
|
-
print(f" StateChecker waiting stop")
|
|
209
|
-
|
|
258
|
+
def markStop(self):
|
|
259
|
+
print(f" StateChecker waiting stop")
|
|
@@ -1,5 +1,13 @@
|
|
|
1
|
-
import os, platform, json, requests, locale
|
|
2
|
-
from ryry import constant
|
|
1
|
+
import os, platform, json, requests, locale, time
|
|
2
|
+
from ryry import constant
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
import portalocker
|
|
6
|
+
except ImportError:
|
|
7
|
+
portalocker = None
|
|
8
|
+
|
|
9
|
+
NETWORK_PROFILE_REFRESH_SECONDS = 6 * 60 * 60
|
|
10
|
+
NETWORK_PROFILE_RETRY_SECONDS = 15 * 60
|
|
3
11
|
|
|
4
12
|
def singleton(cls):
|
|
5
13
|
_instance = {}
|
|
@@ -11,7 +19,7 @@ def singleton(cls):
|
|
|
11
19
|
return inner
|
|
12
20
|
|
|
13
21
|
@singleton
|
|
14
|
-
class Store(object):
|
|
22
|
+
class Store(object):
|
|
15
23
|
|
|
16
24
|
def __init__(self):
|
|
17
25
|
self.path = os.path.join(constant.base_path, f"data.json")
|
|
@@ -26,9 +34,78 @@ class Store(object):
|
|
|
26
34
|
|
|
27
35
|
return data
|
|
28
36
|
|
|
29
|
-
def write(self, data):
|
|
30
|
-
with open(self.path, 'w', encoding='utf-8') as f:
|
|
31
|
-
json.dump(data, f)
|
|
37
|
+
def write(self, data):
|
|
38
|
+
with open(self.path, 'w', encoding='utf-8') as f:
|
|
39
|
+
json.dump(data, f)
|
|
40
|
+
|
|
41
|
+
def networkProfilePath():
|
|
42
|
+
configured_path = os.environ.get("RYRY_NETWORK_PROFILE_PATH", "").strip()
|
|
43
|
+
if configured_path:
|
|
44
|
+
return os.path.abspath(os.path.expanduser(configured_path))
|
|
45
|
+
return os.path.join(constant.base_path, "network_profile.json")
|
|
46
|
+
|
|
47
|
+
def getNetworkProfile():
|
|
48
|
+
try:
|
|
49
|
+
with open(networkProfilePath(), 'r', encoding='utf-8') as f:
|
|
50
|
+
profile = json.load(f)
|
|
51
|
+
return profile if isinstance(profile, dict) else {}
|
|
52
|
+
except Exception:
|
|
53
|
+
return {}
|
|
54
|
+
|
|
55
|
+
def getNetworkIsOversea():
|
|
56
|
+
try:
|
|
57
|
+
value = getNetworkProfile().get("is_oversea")
|
|
58
|
+
return value if isinstance(value, bool) else None
|
|
59
|
+
except Exception:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
def isNetworkProfileRefreshDue(profile=None, now=None):
|
|
63
|
+
try:
|
|
64
|
+
profile = profile if isinstance(profile, dict) else getNetworkProfile()
|
|
65
|
+
next_refresh_at = float(profile.get("next_refresh_at", 0))
|
|
66
|
+
return next_refresh_at <= (now if now is not None else time.time())
|
|
67
|
+
except Exception:
|
|
68
|
+
return True
|
|
69
|
+
|
|
70
|
+
def saveNetworkProfile(profile):
|
|
71
|
+
if not isinstance(profile, dict):
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
path = networkProfilePath()
|
|
75
|
+
directory = os.path.dirname(path) or "."
|
|
76
|
+
lock_path = f"{path}.lock"
|
|
77
|
+
temp_path = f"{path}.{os.getpid()}.{time.time_ns()}.tmp"
|
|
78
|
+
lock_file = None
|
|
79
|
+
try:
|
|
80
|
+
os.makedirs(directory, exist_ok=True)
|
|
81
|
+
lock_file = open(lock_path, 'a+', encoding='utf-8')
|
|
82
|
+
if portalocker:
|
|
83
|
+
portalocker.lock(lock_file, portalocker.LOCK_EX)
|
|
84
|
+
|
|
85
|
+
with open(temp_path, 'w', encoding='utf-8') as f:
|
|
86
|
+
json.dump(profile, f, ensure_ascii=False)
|
|
87
|
+
f.flush()
|
|
88
|
+
os.fsync(f.fileno())
|
|
89
|
+
os.replace(temp_path, path)
|
|
90
|
+
return True
|
|
91
|
+
except Exception:
|
|
92
|
+
return False
|
|
93
|
+
finally:
|
|
94
|
+
if lock_file:
|
|
95
|
+
if portalocker:
|
|
96
|
+
try:
|
|
97
|
+
portalocker.unlock(lock_file)
|
|
98
|
+
except Exception:
|
|
99
|
+
pass
|
|
100
|
+
try:
|
|
101
|
+
lock_file.close()
|
|
102
|
+
except Exception:
|
|
103
|
+
pass
|
|
104
|
+
if os.path.exists(temp_path):
|
|
105
|
+
try:
|
|
106
|
+
os.remove(temp_path)
|
|
107
|
+
except Exception:
|
|
108
|
+
pass
|
|
32
109
|
|
|
33
110
|
#============================== widget ================================
|
|
34
111
|
def isCreateWidget():
|
|
@@ -197,7 +274,7 @@ def _format_percent(value):
|
|
|
197
274
|
except:
|
|
198
275
|
return "0%"
|
|
199
276
|
|
|
200
|
-
def _extendWithRuntimeInfo():
|
|
277
|
+
def _extendWithRuntimeInfo():
|
|
201
278
|
try:
|
|
202
279
|
data = json.loads(GLOBAL_EXT_JSON)
|
|
203
280
|
from ryry import utils
|
|
@@ -224,9 +301,12 @@ def _extendWithRuntimeInfo():
|
|
|
224
301
|
disk["percent"] = _format_percent(disk_runtime.get("percent"))
|
|
225
302
|
|
|
226
303
|
data["gpu"] = runtime.get("gpu", data.get("gpu", {}))
|
|
227
|
-
data["task_slots"] = runtime.get("task_slots", {})
|
|
228
|
-
data["timestamp"] = runtime.get("timestamp")
|
|
229
|
-
|
|
304
|
+
data["task_slots"] = runtime.get("task_slots", {})
|
|
305
|
+
data["timestamp"] = runtime.get("timestamp")
|
|
306
|
+
network_profile = getNetworkProfile()
|
|
307
|
+
if network_profile:
|
|
308
|
+
data["network_profile"] = network_profile
|
|
309
|
+
data.pop("metrics", None)
|
|
230
310
|
return json.dumps(data)
|
|
231
311
|
except:
|
|
232
312
|
return GLOBAL_EXT_JSON
|
|
@@ -1,5 +1,12 @@
|
|
|
1
|
-
from urllib.parse import *
|
|
2
|
-
import time
|
|
1
|
+
from urllib.parse import *
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
def getCachedNetworkIsOversea():
|
|
5
|
+
try:
|
|
6
|
+
from ryry import store
|
|
7
|
+
return store.getNetworkIsOversea()
|
|
8
|
+
except Exception:
|
|
9
|
+
return None
|
|
3
10
|
|
|
4
11
|
def transcode(srcFile):
|
|
5
12
|
try:
|
|
@@ -91,17 +98,21 @@ def uploadByDomain(src, targetDomain,
|
|
|
91
98
|
|
|
92
99
|
parsed_domain, upload_path = parseDomainAndPath(targetDomain)
|
|
93
100
|
|
|
94
|
-
s3_config = getS3Config(parsed_domain)
|
|
95
|
-
if s3_config:
|
|
96
|
-
fallback = s3_config.get("fallback")
|
|
97
|
-
if isS3ConfigReady(s3_config):
|
|
98
|
-
new_file_name = ''.join(str(uuid.uuid4()).split('-'))
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
s3_config = getS3Config(parsed_domain)
|
|
102
|
+
if s3_config:
|
|
103
|
+
fallback = s3_config.get("fallback")
|
|
104
|
+
if isS3ConfigReady(s3_config):
|
|
105
|
+
new_file_name = ''.join(str(uuid.uuid4()).split('-'))
|
|
106
|
+
use_proxy = shouldUseS3Proxy(s3_config)
|
|
107
|
+
try:
|
|
108
|
+
if use_proxy:
|
|
109
|
+
return uploadToS3Proxy(src, f"{new_file_name}.{ext}", s3_config, upload_path)
|
|
110
|
+
return uploadToS3(src, f"{new_file_name}.{ext}", s3_config, upload_path)
|
|
111
|
+
except Exception as e:
|
|
112
|
+
if not fallback:
|
|
113
|
+
raise
|
|
114
|
+
upload_type = "S3 proxy" if use_proxy else "S3"
|
|
115
|
+
print(f"{upload_type} upload failed, fallback to api: {e}")
|
|
105
116
|
return uploadByApiFallback(src, ext, fallback,
|
|
106
117
|
keepItAlways=keepItAlways,
|
|
107
118
|
needTranscode=needTranscode,
|
|
@@ -171,7 +182,7 @@ def uploadByApiFallback(src, ext, fallback,
|
|
|
171
182
|
needTranscode=needTranscode,
|
|
172
183
|
additionalUrl=additionalUrl)
|
|
173
184
|
|
|
174
|
-
def uploadToOss(file, new_file_name, oss_config, upload_path=""):
|
|
185
|
+
def uploadToOss(file, new_file_name, oss_config, upload_path="", return_domain=None):
|
|
175
186
|
import hashlib
|
|
176
187
|
import hmac
|
|
177
188
|
import base64
|
|
@@ -186,11 +197,7 @@ def uploadToOss(file, new_file_name, oss_config, upload_path=""):
|
|
|
186
197
|
timestamp = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
|
|
187
198
|
content_type = "application/octet-stream"
|
|
188
199
|
|
|
189
|
-
|
|
190
|
-
upload_path = upload_path.strip('/')
|
|
191
|
-
object_key = f"{upload_path}/{new_file_name}"
|
|
192
|
-
else:
|
|
193
|
-
object_key = f"temp/{new_file_name}"
|
|
200
|
+
object_key = _upload_object_key(new_file_name, upload_path)
|
|
194
201
|
|
|
195
202
|
resource = f"/{bucket}/{object_key}"
|
|
196
203
|
string_to_sign = f"PUT\n\n{content_type}\n{timestamp}\n{resource}"
|
|
@@ -208,13 +215,14 @@ def uploadToOss(file, new_file_name, oss_config, upload_path=""):
|
|
|
208
215
|
with open(file, "rb") as f:
|
|
209
216
|
file_data = f.read()
|
|
210
217
|
|
|
211
|
-
response = requests.put(url, data=file_data, headers=headers)
|
|
212
|
-
if response.status_code == 200:
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
+
response = requests.put(url, data=file_data, headers=headers)
|
|
219
|
+
if response.status_code == 200:
|
|
220
|
+
domain = return_domain or oss_config["domain"]
|
|
221
|
+
return f"{domain.rstrip('/')}/{object_key}"
|
|
222
|
+
else:
|
|
223
|
+
raise Exception(f"OSS upload failed: {response.status_code}")
|
|
224
|
+
|
|
225
|
+
def _safe_join_path(*parts):
|
|
218
226
|
clean_parts = []
|
|
219
227
|
for part in parts:
|
|
220
228
|
if not part:
|
|
@@ -222,14 +230,17 @@ def _safe_join_path(*parts):
|
|
|
222
230
|
clean = str(part).strip("/")
|
|
223
231
|
if clean:
|
|
224
232
|
clean_parts.append(clean)
|
|
225
|
-
return "/".join(clean_parts)
|
|
226
|
-
|
|
227
|
-
def
|
|
228
|
-
if upload_path:
|
|
229
|
-
return _safe_join_path(upload_path, new_file_name)
|
|
230
|
-
return _safe_join_path("temp", new_file_name)
|
|
231
|
-
|
|
232
|
-
def
|
|
233
|
+
return "/".join(clean_parts)
|
|
234
|
+
|
|
235
|
+
def _upload_object_key(new_file_name, upload_path=""):
|
|
236
|
+
if upload_path:
|
|
237
|
+
return _safe_join_path(upload_path, new_file_name)
|
|
238
|
+
return _safe_join_path("temp", new_file_name)
|
|
239
|
+
|
|
240
|
+
def _s3_object_key(new_file_name, s3_config, upload_path=""):
|
|
241
|
+
return _upload_object_key(new_file_name, upload_path)
|
|
242
|
+
|
|
243
|
+
def uploadToS3(file, new_file_name, s3_config, upload_path=""):
|
|
233
244
|
bucket = s3_config["bucket"]
|
|
234
245
|
content_type = s3_config.get("content_type", "application/octet-stream")
|
|
235
246
|
object_key = _s3_object_key(new_file_name, s3_config, upload_path)
|
|
@@ -241,7 +252,19 @@ def uploadToS3(file, new_file_name, s3_config, upload_path=""):
|
|
|
241
252
|
Body=f,
|
|
242
253
|
ContentType=content_type,
|
|
243
254
|
)
|
|
244
|
-
return f"{s3_config['domain'].rstrip('/')}/{object_key}"
|
|
255
|
+
return f"{s3_config['domain'].rstrip('/')}/{object_key}"
|
|
256
|
+
|
|
257
|
+
def shouldUseS3Proxy(s3_config):
|
|
258
|
+
return bool(s3_config.get("proxy")) and getCachedNetworkIsOversea() is False
|
|
259
|
+
|
|
260
|
+
def uploadToS3Proxy(file, new_file_name, s3_config, upload_path=""):
|
|
261
|
+
return uploadToOss(
|
|
262
|
+
file,
|
|
263
|
+
new_file_name,
|
|
264
|
+
s3_config["proxy"],
|
|
265
|
+
upload_path,
|
|
266
|
+
return_domain=s3_config["domain"],
|
|
267
|
+
)
|
|
245
268
|
|
|
246
269
|
def deepFtpUpload(file, new_file_name, ftp, writepath, readpath):
|
|
247
270
|
try:
|
|
@@ -385,15 +408,18 @@ oss_template_config = {
|
|
|
385
408
|
"domain": "https://oss.zjtemplate.com"
|
|
386
409
|
}
|
|
387
410
|
|
|
388
|
-
REP = utils._decode_key(utils.RK23)
|
|
389
|
-
def r2Config(bucket, domain, fallback="dalipen_api"):
|
|
390
|
-
|
|
391
|
-
"bucket": bucket,
|
|
392
|
-
"endpoint": REP,
|
|
393
|
-
"region": "auto",
|
|
394
|
-
"domain": f"https://{domain}",
|
|
395
|
-
"fallback": fallback,
|
|
396
|
-
}
|
|
411
|
+
REP = utils._decode_key(utils.RK23)
|
|
412
|
+
def r2Config(bucket, domain, fallback="dalipen_api", proxy=None):
|
|
413
|
+
config = {
|
|
414
|
+
"bucket": bucket,
|
|
415
|
+
"endpoint": REP,
|
|
416
|
+
"region": "auto",
|
|
417
|
+
"domain": f"https://{domain}",
|
|
418
|
+
"fallback": fallback,
|
|
419
|
+
}
|
|
420
|
+
if proxy:
|
|
421
|
+
config["proxy"] = proxy
|
|
422
|
+
return config
|
|
397
423
|
|
|
398
424
|
def match_wildcard_domain(target_domain, pattern):
|
|
399
425
|
if pattern.startswith("*."):
|
|
@@ -420,15 +446,15 @@ DOMAIN_CONFIG = {
|
|
|
420
446
|
# "ftp://192.168.50.12/mnt/NAS/mcn/cache": {"type": "ftp", "config": ftp_192_168_50_12},
|
|
421
447
|
# "ftp://183.6.90.205/mnt/NAS/mcn/cache": {"type": "ftp", "config": ftp_219_136_123_179},
|
|
422
448
|
# "ftp://219.136.123.179/mnt/NAS/mcn/cache": {"type": "ftp", "config": ftp_219_136_123_179},
|
|
423
|
-
"app-upload.dalipen.com": {"type": "s3", "config": r2Config("p-app-upload", "app-upload.dalipen.com")},
|
|
449
|
+
"app-upload.dalipen.com": {"type": "s3", "config": r2Config("p-app-upload", "app-upload.dalipen.com", proxy=oss_upload_config)},
|
|
424
450
|
"datanet.dalipen.com": {"type": "s3", "config": r2Config("p-crawler", "datanet.dalipen.com", "dalipen_datanet_api")},
|
|
425
451
|
"model.dalipen.com": {"type": "s3", "config": r2Config("p-model", "model.dalipen.com")},
|
|
426
452
|
"r2.dalipen.com": {"type": "s3", "config": r2Config("p-template", "r2.dalipen.com")},
|
|
427
|
-
"upload.dalipen.com": {"type": "s3", "config": r2Config("p-upload", "upload.dalipen.com"), "oversea": True},
|
|
453
|
+
"upload.dalipen.com": {"type": "s3", "config": r2Config("p-upload", "upload.dalipen.com", proxy=oss_upload_config), "oversea": True},
|
|
428
454
|
"widget.dalipen.com": {"type": "s3", "config": r2Config("p-widget", "widget.dalipen.com")},
|
|
429
|
-
"app-upload.mekoapp.com": {"type": "s3", "config": r2Config("p-app-upload", "app-upload.mekoapp.com", "meko_api")},
|
|
430
|
-
"m.mekoapp.com": {"type": "s3", "config": r2Config("p-app-upload", "m.mekoapp.com", "meko_api")},
|
|
431
|
-
"upload.mekoapp.com": {"type": "s3", "config": r2Config("p-upload", "upload.mekoapp.com", "meko_api")},
|
|
455
|
+
"app-upload.mekoapp.com": {"type": "s3", "config": r2Config("p-app-upload", "app-upload.mekoapp.com", "meko_api", proxy=oss_upload_config)},
|
|
456
|
+
"m.mekoapp.com": {"type": "s3", "config": r2Config("p-app-upload", "m.mekoapp.com", "meko_api", proxy=oss_upload_config)},
|
|
457
|
+
"upload.mekoapp.com": {"type": "s3", "config": r2Config("p-upload", "upload.mekoapp.com", "meko_api", proxy=oss_upload_config)},
|
|
432
458
|
"res.zjtemplate.com": {"type": "oss", "config": oss_res_config},
|
|
433
459
|
"upload.zjtemplate.com": {"type": "oss", "config": oss_upload_config, "oversea": False},
|
|
434
460
|
"oss.zjtemplate.com": {"type": "oss", "config": oss_template_config},
|
|
@@ -505,42 +531,8 @@ def getSubdomain(targetDomain):
|
|
|
505
531
|
return host_item
|
|
506
532
|
return None
|
|
507
533
|
|
|
508
|
-
def getFirstSupportSubdomain():
|
|
509
|
-
def
|
|
510
|
-
import platform
|
|
511
|
-
import re
|
|
512
|
-
import subprocess
|
|
513
|
-
try:
|
|
514
|
-
if platform.system() == "Windows":
|
|
515
|
-
cmd = ["ping", "-n", "1", "-w", str(int(timeout * 1000)), host]
|
|
516
|
-
else:
|
|
517
|
-
cmd = ["ping", "-c", "1", host]
|
|
518
|
-
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
519
|
-
timeout=timeout + 0.5, text=True)
|
|
520
|
-
if result.returncode != 0:
|
|
521
|
-
return None
|
|
522
|
-
match = re.search(r"(?:time|时间)[=<]\s*([0-9.]+)\s*ms", result.stdout, re.IGNORECASE)
|
|
523
|
-
if match:
|
|
524
|
-
return float(match.group(1))
|
|
525
|
-
except:
|
|
526
|
-
pass
|
|
527
|
-
return None
|
|
528
|
-
|
|
529
|
-
def is_oversea_network():
|
|
530
|
-
baidu_ms = ping_ms("baidu.com", timeout=1)
|
|
531
|
-
if baidu_ms is not None and baidu_ms <= 80:
|
|
532
|
-
return False
|
|
533
|
-
|
|
534
|
-
google_ms = ping_ms("google.com", timeout=1)
|
|
535
|
-
if google_ms is None and baidu_ms is not None:
|
|
536
|
-
return False
|
|
537
|
-
if google_ms is not None and baidu_ms is None:
|
|
538
|
-
return True
|
|
539
|
-
if google_ms is not None and baidu_ms is not None:
|
|
540
|
-
return google_ms < baidu_ms
|
|
541
|
-
return None
|
|
542
|
-
|
|
543
|
-
def get_network_hash(oversea):
|
|
534
|
+
def getFirstSupportSubdomain():
|
|
535
|
+
def get_network_hash(oversea):
|
|
544
536
|
import socket
|
|
545
537
|
from hashlib import md5
|
|
546
538
|
try:
|
|
@@ -561,7 +553,7 @@ def getFirstSupportSubdomain():
|
|
|
561
553
|
return {"type": "ftp", "config": ftp_config, "domain": ftp_config["readpath"]}
|
|
562
554
|
return None
|
|
563
555
|
|
|
564
|
-
oversea =
|
|
556
|
+
oversea = getCachedNetworkIsOversea()
|
|
565
557
|
network_hash = get_network_hash(oversea)
|
|
566
558
|
|
|
567
559
|
def _getFirstSupportSubdomain():
|
|
@@ -836,4 +828,4 @@ def downloadDir(url, saveDir, useCount=-1, autoDelete=False):
|
|
|
836
828
|
print(f"url {url} not support")
|
|
837
829
|
except Exception as e:
|
|
838
830
|
print(f"downloadDir fail: {e}")
|
|
839
|
-
return None
|
|
831
|
+
return None
|
|
@@ -15,9 +15,15 @@ from datetime import datetime, timedelta
|
|
|
15
15
|
import http
|
|
16
16
|
import json
|
|
17
17
|
from pathlib import Path
|
|
18
|
-
import socket
|
|
19
|
-
from
|
|
20
|
-
from
|
|
18
|
+
import socket
|
|
19
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
20
|
+
from PIL import Image
|
|
21
|
+
from ryry import constant
|
|
22
|
+
|
|
23
|
+
NETWORK_PROBE_TIMEOUT_SECONDS = 10
|
|
24
|
+
NETWORK_WHEREAMI_URL = "https://api.dalipen.com/whereami"
|
|
25
|
+
NETWORK_ALIYUN_GUANGZHOU_URL = "https://p-upload-gz.oss-cn-guangzhou.aliyuncs.com/"
|
|
26
|
+
NETWORK_LATENCY_DECISIVE_RATIO = 1.5
|
|
21
27
|
|
|
22
28
|
def get_mac_from_nettools():
|
|
23
29
|
try:
|
|
@@ -106,8 +112,129 @@ def get_cpu_serial():
|
|
|
106
112
|
cpu_serial = None
|
|
107
113
|
return cpu_serial
|
|
108
114
|
|
|
109
|
-
def get_hostname():
|
|
110
|
-
return socket.gethostname()
|
|
115
|
+
def get_hostname():
|
|
116
|
+
return socket.gethostname()
|
|
117
|
+
|
|
118
|
+
def _network_ping_ms(host, timeout=1):
|
|
119
|
+
import re
|
|
120
|
+
try:
|
|
121
|
+
if platform.system() == "Windows":
|
|
122
|
+
cmd = ["ping", "-n", "3", "-w", str(int(timeout * 1000)), host]
|
|
123
|
+
else:
|
|
124
|
+
cmd = ["ping", "-c", "3", host]
|
|
125
|
+
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
126
|
+
timeout=timeout + 0.5, text=True)
|
|
127
|
+
if result.returncode != 0:
|
|
128
|
+
return None
|
|
129
|
+
matches = re.findall(r"(?:time|时间)[=<]\s*([0-9.]+)\s*ms", result.stdout, re.IGNORECASE)
|
|
130
|
+
if matches:
|
|
131
|
+
samples = [float(value) for value in matches]
|
|
132
|
+
return round(sum(samples) / len(samples), 3)
|
|
133
|
+
except Exception:
|
|
134
|
+
pass
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
def _network_http_ms(url, timeout=NETWORK_PROBE_TIMEOUT_SECONDS):
|
|
138
|
+
if not url:
|
|
139
|
+
return None
|
|
140
|
+
try:
|
|
141
|
+
started_at = time.perf_counter()
|
|
142
|
+
response = requests.head(url, allow_redirects=False, timeout=timeout)
|
|
143
|
+
response.close()
|
|
144
|
+
return round((time.perf_counter() - started_at) * 1000, 2)
|
|
145
|
+
except Exception:
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
def _network_r2_url():
|
|
149
|
+
try:
|
|
150
|
+
return _decode_key(RK23)
|
|
151
|
+
except Exception:
|
|
152
|
+
return None
|
|
153
|
+
|
|
154
|
+
def _network_whereami_country(timeout=NETWORK_PROBE_TIMEOUT_SECONDS):
|
|
155
|
+
try:
|
|
156
|
+
response = requests.get(NETWORK_WHEREAMI_URL, timeout=timeout)
|
|
157
|
+
response.raise_for_status()
|
|
158
|
+
payload = response.json()
|
|
159
|
+
if not isinstance(payload, dict) or payload.get("code") != 0:
|
|
160
|
+
return None
|
|
161
|
+
country = payload.get("data")
|
|
162
|
+
if isinstance(country, dict):
|
|
163
|
+
country = country.get("country") or country.get("country_code")
|
|
164
|
+
if isinstance(country, str) and len(country.strip()) == 2 and country.strip().isalpha():
|
|
165
|
+
return country.strip().upper()
|
|
166
|
+
except Exception:
|
|
167
|
+
pass
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
def _network_latency_preference(domestic_ms, oversea_ms):
|
|
171
|
+
if domestic_ms is None and oversea_ms is None:
|
|
172
|
+
return None, 0
|
|
173
|
+
if domestic_ms is None:
|
|
174
|
+
return True, float("inf")
|
|
175
|
+
if oversea_ms is None:
|
|
176
|
+
return False, float("inf")
|
|
177
|
+
if domestic_ms == oversea_ms:
|
|
178
|
+
return None, 1
|
|
179
|
+
|
|
180
|
+
is_oversea = oversea_ms < domestic_ms
|
|
181
|
+
return is_oversea, max(domestic_ms, oversea_ms) / min(domestic_ms, oversea_ms)
|
|
182
|
+
|
|
183
|
+
def _network_latency_decision(baidu_ms, google_ms, r2_ms, aliyun_ms):
|
|
184
|
+
location_result, _ = _network_latency_preference(baidu_ms, google_ms)
|
|
185
|
+
storage_result, storage_ratio = _network_latency_preference(aliyun_ms, r2_ms)
|
|
186
|
+
|
|
187
|
+
if location_result is None:
|
|
188
|
+
return storage_result, "storage_latency" if storage_result is not None else "inconclusive"
|
|
189
|
+
if storage_result is None:
|
|
190
|
+
return location_result, "baidu_google_latency"
|
|
191
|
+
if location_result == storage_result:
|
|
192
|
+
return location_result, "combined_latency"
|
|
193
|
+
if storage_ratio >= NETWORK_LATENCY_DECISIVE_RATIO:
|
|
194
|
+
return storage_result, "storage_latency"
|
|
195
|
+
return location_result, "combined_latency"
|
|
196
|
+
|
|
197
|
+
def detectNetworkProfile(timeout=NETWORK_PROBE_TIMEOUT_SECONDS):
|
|
198
|
+
probes = {
|
|
199
|
+
"baidu_ms": (_network_ping_ms, "baidu.com"),
|
|
200
|
+
"google_ms": (_network_ping_ms, "google.com"),
|
|
201
|
+
"r2_ms": (_network_http_ms, _network_r2_url()),
|
|
202
|
+
"aliyun_ms": (_network_http_ms, NETWORK_ALIYUN_GUANGZHOU_URL),
|
|
203
|
+
}
|
|
204
|
+
results = {}
|
|
205
|
+
with ThreadPoolExecutor(max_workers=len(probes)) as executor:
|
|
206
|
+
futures = {
|
|
207
|
+
name: executor.submit(probe, target, timeout=timeout)
|
|
208
|
+
for name, (probe, target) in probes.items()
|
|
209
|
+
}
|
|
210
|
+
for name, future in futures.items():
|
|
211
|
+
try:
|
|
212
|
+
results[name] = future.result()
|
|
213
|
+
except Exception:
|
|
214
|
+
results[name] = None
|
|
215
|
+
|
|
216
|
+
is_oversea, method = _network_latency_decision(
|
|
217
|
+
results.get("baidu_ms"),
|
|
218
|
+
results.get("google_ms"),
|
|
219
|
+
results.get("r2_ms"),
|
|
220
|
+
results.get("aliyun_ms"),
|
|
221
|
+
)
|
|
222
|
+
country = None
|
|
223
|
+
if is_oversea is None:
|
|
224
|
+
country = _network_whereami_country(timeout=timeout)
|
|
225
|
+
if country:
|
|
226
|
+
is_oversea = country != "CN"
|
|
227
|
+
method = "whereami"
|
|
228
|
+
|
|
229
|
+
return {
|
|
230
|
+
"is_oversea": is_oversea,
|
|
231
|
+
"baidu_ms": results.get("baidu_ms"),
|
|
232
|
+
"google_ms": results.get("google_ms"),
|
|
233
|
+
"r2_ms": results.get("r2_ms"),
|
|
234
|
+
"aliyun_ms": results.get("aliyun_ms"),
|
|
235
|
+
"country": country,
|
|
236
|
+
"method": method,
|
|
237
|
+
}
|
|
111
238
|
|
|
112
239
|
def generate_unique_id():
|
|
113
240
|
mac = get_mac_address()
|
|
@@ -30,4 +30,6 @@ ryry_cli.egg-info/SOURCES.txt
|
|
|
30
30
|
ryry_cli.egg-info/dependency_links.txt
|
|
31
31
|
ryry_cli.egg-info/entry_points.txt
|
|
32
32
|
ryry_cli.egg-info/requires.txt
|
|
33
|
-
ryry_cli.egg-info/top_level.txt
|
|
33
|
+
ryry_cli.egg-info/top_level.txt
|
|
34
|
+
tests/test_network_profile.py
|
|
35
|
+
tests/test_upload_proxy.py
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import unittest
|
|
4
|
+
from tempfile import TemporaryDirectory
|
|
5
|
+
from unittest.mock import Mock, patch
|
|
6
|
+
|
|
7
|
+
from ryry import store, upload, utils
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class NetworkProfileStoreTest(unittest.TestCase):
|
|
11
|
+
def setUp(self):
|
|
12
|
+
self.temp_dir = TemporaryDirectory()
|
|
13
|
+
self.profile_path = os.path.join(self.temp_dir.name, "network_profile.json")
|
|
14
|
+
self.path_patch = patch.dict(
|
|
15
|
+
os.environ,
|
|
16
|
+
{"RYRY_NETWORK_PROFILE_PATH": self.profile_path},
|
|
17
|
+
)
|
|
18
|
+
self.path_patch.start()
|
|
19
|
+
|
|
20
|
+
def tearDown(self):
|
|
21
|
+
self.path_patch.stop()
|
|
22
|
+
self.temp_dir.cleanup()
|
|
23
|
+
|
|
24
|
+
def test_profile_round_trip(self):
|
|
25
|
+
profile = {
|
|
26
|
+
"is_oversea": False,
|
|
27
|
+
"checked_at": 100,
|
|
28
|
+
"next_refresh_at": 200,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
self.assertTrue(store.saveNetworkProfile(profile))
|
|
32
|
+
self.assertEqual(store.getNetworkProfile(), profile)
|
|
33
|
+
self.assertFalse(store.getNetworkIsOversea())
|
|
34
|
+
|
|
35
|
+
def test_corrupt_profile_falls_back_to_unknown(self):
|
|
36
|
+
with open(self.profile_path, 'w', encoding='utf-8') as f:
|
|
37
|
+
f.write("{invalid json")
|
|
38
|
+
|
|
39
|
+
self.assertEqual(store.getNetworkProfile(), {})
|
|
40
|
+
self.assertIsNone(store.getNetworkIsOversea())
|
|
41
|
+
|
|
42
|
+
def test_invalid_profile_does_not_enable_proxy(self):
|
|
43
|
+
with open(self.profile_path, 'w', encoding='utf-8') as f:
|
|
44
|
+
json.dump({"is_oversea": "false"}, f)
|
|
45
|
+
|
|
46
|
+
s3_config = upload.getS3Config("upload.dalipen.com")
|
|
47
|
+
self.assertFalse(upload.shouldUseS3Proxy(s3_config))
|
|
48
|
+
|
|
49
|
+
def test_profile_read_exception_does_not_enable_proxy(self):
|
|
50
|
+
s3_config = upload.getS3Config("upload.dalipen.com")
|
|
51
|
+
with patch("ryry.store.getNetworkProfile", side_effect=OSError("read failed")):
|
|
52
|
+
self.assertFalse(upload.shouldUseS3Proxy(s3_config))
|
|
53
|
+
|
|
54
|
+
@patch("ryry.utils._network_whereami_country")
|
|
55
|
+
@patch("ryry.utils._network_http_ms")
|
|
56
|
+
@patch("ryry.utils._network_ping_ms")
|
|
57
|
+
def test_combined_domestic_latency_uses_proxy(self, ping, http, whereami):
|
|
58
|
+
ping.side_effect = lambda host, timeout: {
|
|
59
|
+
"baidu.com": 20,
|
|
60
|
+
"google.com": 180,
|
|
61
|
+
}[host]
|
|
62
|
+
http.side_effect = lambda url, timeout: (
|
|
63
|
+
40 if url == utils.NETWORK_ALIYUN_GUANGZHOU_URL else 260
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
profile = utils.detectNetworkProfile()
|
|
67
|
+
|
|
68
|
+
self.assertFalse(profile["is_oversea"])
|
|
69
|
+
self.assertEqual(profile["method"], "combined_latency")
|
|
70
|
+
self.assertEqual(profile["r2_ms"], 260)
|
|
71
|
+
self.assertEqual(profile["aliyun_ms"], 40)
|
|
72
|
+
whereami.assert_not_called()
|
|
73
|
+
self.assertEqual(
|
|
74
|
+
{call.args for call in ping.call_args_list},
|
|
75
|
+
{("baidu.com",), ("google.com",)},
|
|
76
|
+
)
|
|
77
|
+
self.assertTrue(all(call.kwargs["timeout"] == 10 for call in ping.call_args_list))
|
|
78
|
+
|
|
79
|
+
@patch("ryry.utils._network_whereami_country")
|
|
80
|
+
@patch("ryry.utils._network_http_ms")
|
|
81
|
+
@patch("ryry.utils._network_ping_ms")
|
|
82
|
+
def test_decisive_storage_latency_wins_conflicting_location(self, ping, http, whereami):
|
|
83
|
+
ping.side_effect = lambda host, timeout: {
|
|
84
|
+
"baidu.com": 20,
|
|
85
|
+
"google.com": 180,
|
|
86
|
+
}[host]
|
|
87
|
+
http.side_effect = lambda url, timeout: (
|
|
88
|
+
300 if url == utils.NETWORK_ALIYUN_GUANGZHOU_URL else 40
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
profile = utils.detectNetworkProfile()
|
|
92
|
+
|
|
93
|
+
self.assertTrue(profile["is_oversea"])
|
|
94
|
+
self.assertEqual(profile["method"], "storage_latency")
|
|
95
|
+
whereami.assert_not_called()
|
|
96
|
+
|
|
97
|
+
@patch("ryry.utils._network_whereami_country", return_value="CN")
|
|
98
|
+
@patch("ryry.utils._network_http_ms", return_value=None)
|
|
99
|
+
@patch("ryry.utils._network_ping_ms", return_value=None)
|
|
100
|
+
def test_whereami_cn_is_used_when_latency_is_inconclusive(self, ping, http, whereami):
|
|
101
|
+
profile = utils.detectNetworkProfile()
|
|
102
|
+
|
|
103
|
+
self.assertFalse(profile["is_oversea"])
|
|
104
|
+
self.assertEqual(profile["country"], "CN")
|
|
105
|
+
self.assertEqual(profile["method"], "whereami")
|
|
106
|
+
whereami.assert_called_once_with(timeout=10)
|
|
107
|
+
|
|
108
|
+
@patch("ryry.utils._network_whereami_country", return_value="US")
|
|
109
|
+
@patch("ryry.utils._network_http_ms", return_value=None)
|
|
110
|
+
@patch("ryry.utils._network_ping_ms", return_value=None)
|
|
111
|
+
def test_whereami_non_cn_stays_on_r2(self, ping, http, whereami):
|
|
112
|
+
profile = utils.detectNetworkProfile()
|
|
113
|
+
|
|
114
|
+
self.assertTrue(profile["is_oversea"])
|
|
115
|
+
self.assertEqual(profile["country"], "US")
|
|
116
|
+
|
|
117
|
+
@patch("ryry.utils._network_whereami_country", return_value=None)
|
|
118
|
+
@patch("ryry.utils._network_http_ms", return_value=None)
|
|
119
|
+
@patch("ryry.utils._network_ping_ms", return_value=None)
|
|
120
|
+
def test_all_detection_failures_remain_unknown(self, ping, http, whereami):
|
|
121
|
+
profile = utils.detectNetworkProfile()
|
|
122
|
+
|
|
123
|
+
self.assertIsNone(profile["is_oversea"])
|
|
124
|
+
self.assertEqual(profile["method"], "inconclusive")
|
|
125
|
+
|
|
126
|
+
@patch("ryry.utils.requests.get")
|
|
127
|
+
def test_whereami_response_is_parsed(self, get):
|
|
128
|
+
response = Mock()
|
|
129
|
+
response.json.return_value = {"code": 0, "data": "cn"}
|
|
130
|
+
get.return_value = response
|
|
131
|
+
|
|
132
|
+
self.assertEqual(utils._network_whereami_country(), "CN")
|
|
133
|
+
get.assert_called_once_with(utils.NETWORK_WHEREAMI_URL, timeout=10)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
if __name__ == "__main__":
|
|
137
|
+
unittest.main()
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from tempfile import NamedTemporaryFile
|
|
3
|
+
from unittest.mock import Mock, patch
|
|
4
|
+
|
|
5
|
+
from ryry import upload
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class UploadProxyTest(unittest.TestCase):
|
|
9
|
+
def setUp(self):
|
|
10
|
+
self.s3_config = upload.getS3Config("app-upload.mekoapp.com")
|
|
11
|
+
self.proxy_config = self.s3_config["proxy"]
|
|
12
|
+
|
|
13
|
+
def test_worker_domains_enable_upload_proxy(self):
|
|
14
|
+
domains = [
|
|
15
|
+
"upload.dalipen.com",
|
|
16
|
+
"app-upload.dalipen.com",
|
|
17
|
+
"upload.mekoapp.com",
|
|
18
|
+
"app-upload.mekoapp.com",
|
|
19
|
+
"m.mekoapp.com",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
for domain in domains:
|
|
23
|
+
with self.subTest(domain=domain):
|
|
24
|
+
self.assertIs(upload.getS3Config(domain)["proxy"], upload.oss_upload_config)
|
|
25
|
+
|
|
26
|
+
def test_other_s3_domains_do_not_enable_upload_proxy(self):
|
|
27
|
+
domains = [
|
|
28
|
+
"datanet.dalipen.com",
|
|
29
|
+
"model.dalipen.com",
|
|
30
|
+
"r2.dalipen.com",
|
|
31
|
+
"widget.dalipen.com",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
for domain in domains:
|
|
35
|
+
with self.subTest(domain=domain):
|
|
36
|
+
self.assertNotIn("proxy", upload.getS3Config(domain))
|
|
37
|
+
|
|
38
|
+
@patch("uuid.uuid4", return_value="fixed-id")
|
|
39
|
+
@patch("ryry.upload.uploadToS3")
|
|
40
|
+
@patch("ryry.upload.uploadToS3Proxy")
|
|
41
|
+
@patch("ryry.upload.getCachedNetworkIsOversea", return_value=False)
|
|
42
|
+
def test_domestic_s3_upload_uses_proxy(self, _network, proxy_upload, s3_upload, _uuid):
|
|
43
|
+
proxy_upload.return_value = "https://app-upload.mekoapp.com/upload/2026/07/fixed-id.png"
|
|
44
|
+
|
|
45
|
+
result = upload.uploadByDomain(
|
|
46
|
+
"/tmp/source.png",
|
|
47
|
+
"https://app-upload.mekoapp.com/upload/2026/07",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
self.assertEqual(
|
|
51
|
+
result,
|
|
52
|
+
"https://app-upload.mekoapp.com/upload/2026/07/fixed-id.png",
|
|
53
|
+
)
|
|
54
|
+
proxy_upload.assert_called_once_with(
|
|
55
|
+
"/tmp/source.png",
|
|
56
|
+
"fixedid.png",
|
|
57
|
+
self.s3_config,
|
|
58
|
+
"upload/2026/07",
|
|
59
|
+
)
|
|
60
|
+
s3_upload.assert_not_called()
|
|
61
|
+
|
|
62
|
+
@patch("uuid.uuid4", return_value="fixed-id")
|
|
63
|
+
@patch("ryry.upload.uploadToS3")
|
|
64
|
+
@patch("ryry.upload.uploadToS3Proxy")
|
|
65
|
+
@patch("ryry.upload.getCachedNetworkIsOversea", return_value=True)
|
|
66
|
+
def test_oversea_s3_upload_stays_on_r2(self, _network, proxy_upload, s3_upload, _uuid):
|
|
67
|
+
s3_upload.return_value = "https://app-upload.mekoapp.com/temp/fixed-id.png"
|
|
68
|
+
|
|
69
|
+
result = upload.uploadByDomain("/tmp/source.png", "app-upload.mekoapp.com")
|
|
70
|
+
|
|
71
|
+
self.assertEqual(result, "https://app-upload.mekoapp.com/temp/fixed-id.png")
|
|
72
|
+
s3_upload.assert_called_once_with(
|
|
73
|
+
"/tmp/source.png",
|
|
74
|
+
"fixedid.png",
|
|
75
|
+
self.s3_config,
|
|
76
|
+
"",
|
|
77
|
+
)
|
|
78
|
+
proxy_upload.assert_not_called()
|
|
79
|
+
|
|
80
|
+
@patch("uuid.uuid4", return_value="fixed-id")
|
|
81
|
+
@patch("ryry.upload.uploadToS3")
|
|
82
|
+
@patch("ryry.upload.uploadToS3Proxy")
|
|
83
|
+
@patch("ryry.upload.getCachedNetworkIsOversea", return_value=None)
|
|
84
|
+
def test_unknown_network_stays_on_r2(self, _network, proxy_upload, s3_upload, _uuid):
|
|
85
|
+
s3_upload.return_value = "https://app-upload.mekoapp.com/temp/fixed-id.png"
|
|
86
|
+
|
|
87
|
+
result = upload.uploadByDomain("/tmp/source.png", "app-upload.mekoapp.com")
|
|
88
|
+
|
|
89
|
+
self.assertEqual(result, "https://app-upload.mekoapp.com/temp/fixed-id.png")
|
|
90
|
+
s3_upload.assert_called_once()
|
|
91
|
+
proxy_upload.assert_not_called()
|
|
92
|
+
|
|
93
|
+
@patch("uuid.uuid4", return_value="fixed-id")
|
|
94
|
+
@patch("ryry.upload.uploadToS3")
|
|
95
|
+
@patch("ryry.upload.getCachedNetworkIsOversea")
|
|
96
|
+
def test_s3_without_proxy_does_not_check_network(self, network, s3_upload, _uuid):
|
|
97
|
+
config = dict(self.s3_config)
|
|
98
|
+
config.pop("proxy")
|
|
99
|
+
s3_upload.return_value = "https://model.dalipen.com/temp/fixed-id.png"
|
|
100
|
+
|
|
101
|
+
with patch("ryry.upload.getS3Config", return_value=config):
|
|
102
|
+
result = upload.uploadByDomain("/tmp/source.png", "model.dalipen.com")
|
|
103
|
+
|
|
104
|
+
self.assertEqual(result, "https://model.dalipen.com/temp/fixed-id.png")
|
|
105
|
+
network.assert_not_called()
|
|
106
|
+
|
|
107
|
+
@patch("uuid.uuid4", return_value="fixed-id")
|
|
108
|
+
@patch("ryry.upload.uploadByApiFallback", return_value="https://fallback.example/file.png")
|
|
109
|
+
@patch("ryry.upload.uploadToS3Proxy", side_effect=RuntimeError("proxy unavailable"))
|
|
110
|
+
@patch("ryry.upload.getCachedNetworkIsOversea", return_value=False)
|
|
111
|
+
def test_proxy_failure_uses_existing_api_fallback(
|
|
112
|
+
self,
|
|
113
|
+
_network,
|
|
114
|
+
_proxy_upload,
|
|
115
|
+
api_fallback,
|
|
116
|
+
_uuid,
|
|
117
|
+
):
|
|
118
|
+
result = upload.uploadByDomain("/tmp/source.png", "app-upload.mekoapp.com")
|
|
119
|
+
|
|
120
|
+
self.assertEqual(result, "https://fallback.example/file.png")
|
|
121
|
+
api_fallback.assert_called_once_with(
|
|
122
|
+
"/tmp/source.png",
|
|
123
|
+
"png",
|
|
124
|
+
"meko_api",
|
|
125
|
+
keepItAlways=False,
|
|
126
|
+
needTranscode=False,
|
|
127
|
+
additionalUrl=False,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
@patch("ryry.upload.uploadToOss")
|
|
131
|
+
def test_proxy_upload_returns_target_s3_domain(self, oss_upload):
|
|
132
|
+
oss_upload.return_value = "https://app-upload.mekoapp.com/temp/file.png"
|
|
133
|
+
|
|
134
|
+
result = upload.uploadToS3Proxy(
|
|
135
|
+
"/tmp/source.png",
|
|
136
|
+
"file.png",
|
|
137
|
+
self.s3_config,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
self.assertEqual(result, "https://app-upload.mekoapp.com/temp/file.png")
|
|
141
|
+
oss_upload.assert_called_once_with(
|
|
142
|
+
"/tmp/source.png",
|
|
143
|
+
"file.png",
|
|
144
|
+
self.proxy_config,
|
|
145
|
+
"",
|
|
146
|
+
return_domain="https://app-upload.mekoapp.com",
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
@patch("requests.put")
|
|
150
|
+
def test_proxy_oss_put_uses_proxy_bucket_and_target_url(self, put_request):
|
|
151
|
+
put_request.return_value = Mock(status_code=200)
|
|
152
|
+
|
|
153
|
+
with NamedTemporaryFile() as source:
|
|
154
|
+
source.write(b"image")
|
|
155
|
+
source.flush()
|
|
156
|
+
result = upload.uploadToOss(
|
|
157
|
+
source.name,
|
|
158
|
+
"file.png",
|
|
159
|
+
self.proxy_config,
|
|
160
|
+
"upload/2026/07",
|
|
161
|
+
return_domain="https://app-upload.mekoapp.com",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
self.assertEqual(
|
|
165
|
+
result,
|
|
166
|
+
"https://app-upload.mekoapp.com/upload/2026/07/file.png",
|
|
167
|
+
)
|
|
168
|
+
self.assertEqual(
|
|
169
|
+
put_request.call_args.args[0],
|
|
170
|
+
"https://p-upload-gz.oss-cn-guangzhou.aliyuncs.com/upload/2026/07/file.png",
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == "__main__":
|
|
175
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|