ryry-cli 6.15__tar.gz → 6.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ryry_cli-6.15/ryry_cli.egg-info → ryry_cli-6.17}/PKG-INFO +1 -1
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/constant.py +2 -2
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/main.py +8 -20
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/ryry_server_socket.py +91 -15
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/shared_memory.py +107 -31
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/store.py +1 -1
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/upload.py +5 -18
- {ryry_cli-6.15 → ryry_cli-6.17/ryry_cli.egg-info}/PKG-INFO +1 -1
- {ryry_cli-6.15 → ryry_cli-6.17}/setup.py +1 -1
- {ryry_cli-6.15 → ryry_cli-6.17}/LICENSE +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/README.md +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/__init__.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/daemon_base.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/daemon_manager.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/proxy_manager.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/ryry_service.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/ryry_webapi.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/ryry_widget.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/script_template/__init__.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/script_template/daemon.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/script_template/main.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/script_template/run.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/server_func.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/task.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/taskUtils.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry/utils.py +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry_cli.egg-info/SOURCES.txt +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry_cli.egg-info/dependency_links.txt +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry_cli.egg-info/entry_points.txt +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry_cli.egg-info/requires.txt +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/ryry_cli.egg-info/top_level.txt +0 -0
- {ryry_cli-6.15 → ryry_cli-6.17}/setup.cfg +0 -0
|
@@ -173,26 +173,14 @@ def status():
|
|
|
173
173
|
deviceid = utils.generate_unique_id()
|
|
174
174
|
import socket
|
|
175
175
|
machine_name = socket.gethostname()
|
|
176
|
-
service = ryry_service.ryryService()
|
|
177
|
-
thread_num = store.get_multithread()
|
|
178
|
-
def get_shared_memory_max_counter():
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
elif system == 'Darwin':
|
|
185
|
-
cache_dir = os.path.expanduser('~/Library/Caches')
|
|
186
|
-
data_dir = os.path.join(cache_dir, 'widget_shared_memory')
|
|
187
|
-
else:
|
|
188
|
-
data_dir = '/tmp/widget_shared_memory'
|
|
189
|
-
data_file = os.path.join(data_dir, 'widget_power.json')
|
|
190
|
-
try:
|
|
191
|
-
with open(data_file, 'r', encoding='utf-8') as f:
|
|
192
|
-
data = json.load(f)
|
|
193
|
-
return data.get('max_counter', None)
|
|
194
|
-
except Exception:
|
|
195
|
-
return None
|
|
176
|
+
service = ryry_service.ryryService()
|
|
177
|
+
thread_num = store.get_multithread()
|
|
178
|
+
def get_shared_memory_max_counter():
|
|
179
|
+
try:
|
|
180
|
+
from ryry.shared_memory import shared_memory_service
|
|
181
|
+
return shared_memory_service.get_max_counter()
|
|
182
|
+
except Exception:
|
|
183
|
+
return None
|
|
196
184
|
shared_max_counter = get_shared_memory_max_counter()
|
|
197
185
|
shared_max_counter_str = ""
|
|
198
186
|
if shared_max_counter and shared_max_counter != thread_num:
|
|
@@ -17,8 +17,20 @@ class RyryTaskExecutor(Thread):
|
|
|
17
17
|
ttt = 0
|
|
18
18
|
|
|
19
19
|
def appendTask(self, data, callback):
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
if not self.workers_started:
|
|
21
|
+
raise Exception("task workers are not ready")
|
|
22
|
+
counter_incremented = False
|
|
23
|
+
try:
|
|
24
|
+
shared_memory_service.increment_cur_counter()
|
|
25
|
+
counter_incremented = True
|
|
26
|
+
self.task_queue.put([data, callback])
|
|
27
|
+
except Exception:
|
|
28
|
+
if counter_incremented:
|
|
29
|
+
try:
|
|
30
|
+
shared_memory_service.decrement_cur_counter()
|
|
31
|
+
except:
|
|
32
|
+
pass
|
|
33
|
+
raise
|
|
22
34
|
try:
|
|
23
35
|
taskUUID = data["taskUUID"]
|
|
24
36
|
taskUtils.taskPrint(taskUUID, f"{current_thread().name} === addQueue task : {taskUUID}")
|
|
@@ -27,6 +39,15 @@ class RyryTaskExecutor(Thread):
|
|
|
27
39
|
|
|
28
40
|
def idlePower(self):
|
|
29
41
|
return shared_memory_service.get_max_counter() - shared_memory_service.get_cur_counter()
|
|
42
|
+
|
|
43
|
+
def canAcceptTask(self):
|
|
44
|
+
if not self.workers_started:
|
|
45
|
+
return False
|
|
46
|
+
try:
|
|
47
|
+
return shared_memory_service.ensure_available()
|
|
48
|
+
except Exception as e:
|
|
49
|
+
taskUtils.taskPrint(None, f"{current_thread().name} === shared memory unavailable : {e}")
|
|
50
|
+
return False
|
|
30
51
|
|
|
31
52
|
def widgetHasPower(self, cur_widget_id):
|
|
32
53
|
widget_power = shared_memory_service.get_widget_power(cur_widget_id)
|
|
@@ -58,6 +79,7 @@ class RyryTaskExecutor(Thread):
|
|
|
58
79
|
self.max_counter = store.get_multithread()
|
|
59
80
|
self.is_running = True
|
|
60
81
|
self.THEADING_LIST = []
|
|
82
|
+
self.workers_started = False
|
|
61
83
|
self.start()
|
|
62
84
|
|
|
63
85
|
def taskRunning(self):
|
|
@@ -67,6 +89,14 @@ class RyryTaskExecutor(Thread):
|
|
|
67
89
|
data, callback = self.task_queue.get()
|
|
68
90
|
if data is None:
|
|
69
91
|
break
|
|
92
|
+
taskUUID = data.get("taskUUID", "")
|
|
93
|
+
widget_id = None
|
|
94
|
+
task_record_added = False
|
|
95
|
+
widget_power_added = False
|
|
96
|
+
callback_needed = False
|
|
97
|
+
is_ok = False
|
|
98
|
+
msg = ""
|
|
99
|
+
result = "{}"
|
|
70
100
|
try:
|
|
71
101
|
data_config = json.loads(data["config"])
|
|
72
102
|
widget_id = data_config["widget_id"]
|
|
@@ -76,34 +106,55 @@ class RyryTaskExecutor(Thread):
|
|
|
76
106
|
timeout = data_config.get("timeout", 600)
|
|
77
107
|
if "timeout" in data:
|
|
78
108
|
timeout = data["timeout"]
|
|
79
|
-
|
|
80
|
-
taskUUID = data["taskUUID"]
|
|
81
109
|
domain = ""
|
|
82
110
|
params_tmp = json.loads(data["data"])
|
|
83
111
|
if "domain" in params_tmp:
|
|
84
112
|
domain = params_tmp["domain"]
|
|
85
113
|
taskUtils.taskPrint(taskUUID, f"{current_thread().name} === receive task : {taskUUID}")
|
|
86
114
|
_appendTask(taskUUID, domain)
|
|
115
|
+
task_record_added = True
|
|
87
116
|
self.addWidgetPower(widget_id, max_task_number)
|
|
117
|
+
widget_power_added = True
|
|
88
118
|
is_ok, msg, result = task.runTask(data, timeout)
|
|
119
|
+
callback_needed = True
|
|
89
120
|
if is_ok == False:
|
|
90
121
|
taskUtils.notifyTaskFail(taskUUID, msg)
|
|
91
|
-
shared_memory_service.decrement_cur_counter()
|
|
92
|
-
callback(taskUUID, is_ok, msg, result)
|
|
93
|
-
_removeTask(taskUUID)
|
|
94
|
-
self.delWidgetPower(widget_id)
|
|
95
122
|
except Exception as e:
|
|
96
|
-
|
|
97
|
-
|
|
123
|
+
msg = f"{current_thread().name} === task exception : {e}"
|
|
124
|
+
result = "{}"
|
|
125
|
+
callback_needed = bool(taskUUID)
|
|
126
|
+
taskUtils.taskPrint(taskUUID, msg)
|
|
127
|
+
if taskUUID:
|
|
128
|
+
taskUtils.notifyScriptError(taskUUID)
|
|
98
129
|
finally:
|
|
99
|
-
|
|
130
|
+
try:
|
|
131
|
+
shared_memory_service.decrement_cur_counter()
|
|
132
|
+
except Exception as e:
|
|
133
|
+
taskUtils.taskPrint(taskUUID, f"{current_thread().name} === decrement counter exception : {e}")
|
|
134
|
+
if callback_needed and taskUUID:
|
|
135
|
+
try:
|
|
136
|
+
callback(taskUUID, is_ok, msg, result)
|
|
137
|
+
except Exception as e:
|
|
138
|
+
taskUtils.taskPrint(taskUUID, f"{current_thread().name} === task callback exception : {e}")
|
|
139
|
+
if task_record_added:
|
|
140
|
+
try:
|
|
141
|
+
_removeTask(taskUUID)
|
|
142
|
+
except Exception as e:
|
|
143
|
+
taskUtils.taskPrint(taskUUID, f"{current_thread().name} === remove task exception : {e}")
|
|
144
|
+
if widget_power_added:
|
|
145
|
+
try:
|
|
146
|
+
self.delWidgetPower(widget_id)
|
|
147
|
+
except Exception as e:
|
|
148
|
+
taskUtils.taskPrint(taskUUID, f"{current_thread().name} === widget power cleanup exception : {e}")
|
|
149
|
+
if taskUUID:
|
|
150
|
+
taskUtils.taskPrint(taskUUID, None)
|
|
100
151
|
self.task_queue.task_done()
|
|
101
152
|
except Exception as ex:
|
|
102
153
|
taskUtils.taskPrint(None, f"{current_thread().name} === exception : {ex}")
|
|
103
154
|
pass
|
|
104
155
|
print(f" {current_thread().name} taskRunning stop")
|
|
105
156
|
|
|
106
|
-
def
|
|
157
|
+
def startTaskWorkers(self):
|
|
107
158
|
shared_memory_service.start_service()
|
|
108
159
|
shared_memory_service.set_max_counter(self.max_counter)
|
|
109
160
|
shared_memory_service.set_cur_counter(0)
|
|
@@ -111,7 +162,18 @@ class RyryTaskExecutor(Thread):
|
|
|
111
162
|
self.THEADING_LIST.append(Thread(name=f"exector-{idx}",target=self.taskRunning))
|
|
112
163
|
for t in self.THEADING_LIST:
|
|
113
164
|
t.start()
|
|
165
|
+
self.workers_started = True
|
|
166
|
+
|
|
167
|
+
def run(self):
|
|
114
168
|
while self.is_running:
|
|
169
|
+
if not self.workers_started:
|
|
170
|
+
try:
|
|
171
|
+
self.startTaskWorkers()
|
|
172
|
+
print(f" {self.name} workers started")
|
|
173
|
+
except Exception as e:
|
|
174
|
+
taskUtils.taskPrint(None, f"{self.name} === start workers exception : {e}")
|
|
175
|
+
time.sleep(5)
|
|
176
|
+
continue
|
|
115
177
|
time.sleep(5)
|
|
116
178
|
for t in self.THEADING_LIST:
|
|
117
179
|
t.join()
|
|
@@ -123,7 +185,10 @@ class RyryTaskExecutor(Thread):
|
|
|
123
185
|
for _ in range(self.max_counter*2):
|
|
124
186
|
self.task_queue.put([None, None])
|
|
125
187
|
print(f" {self.name} stop")
|
|
126
|
-
|
|
188
|
+
try:
|
|
189
|
+
shared_memory_service.stop_service()
|
|
190
|
+
except Exception as e:
|
|
191
|
+
taskUtils.taskPrint(None, f"{self.name} === stop shared memory exception : {e}")
|
|
127
192
|
|
|
128
193
|
lock = Lock()
|
|
129
194
|
task_config_file = os.path.join(constant.base_path, f"task_config.txt")
|
|
@@ -238,6 +303,9 @@ class RyryShortConnectThread(Thread):
|
|
|
238
303
|
if self.executor.idlePower() <= 0:
|
|
239
304
|
time.sleep(1)
|
|
240
305
|
continue
|
|
306
|
+
if not self.executor.canAcceptTask():
|
|
307
|
+
time.sleep(wait_time)
|
|
308
|
+
continue
|
|
241
309
|
|
|
242
310
|
widget_list = {}
|
|
243
311
|
map = store.widgetMap()
|
|
@@ -250,7 +318,15 @@ class RyryShortConnectThread(Thread):
|
|
|
250
318
|
widget_list[it] = map[it].get("version", "0.0")
|
|
251
319
|
datas = ryry_webapi.GetTask(widget_list)
|
|
252
320
|
for it in datas:
|
|
253
|
-
|
|
321
|
+
try:
|
|
322
|
+
self.executor.appendTask(it, self.taskCallback)
|
|
323
|
+
except Exception as e:
|
|
324
|
+
taskUUID = it.get("taskUUID", "")
|
|
325
|
+
msg = f"{self.name} === append task exception : {e}"
|
|
326
|
+
taskUtils.taskPrint(taskUUID, msg)
|
|
327
|
+
if taskUUID:
|
|
328
|
+
taskUtils.notifyScriptError(taskUUID)
|
|
329
|
+
self.taskCallback(taskUUID, False, msg, "{}")
|
|
254
330
|
|
|
255
331
|
# if task empty -> full, set wait_time = max -> min
|
|
256
332
|
if len(datas) > 0:
|
|
@@ -281,4 +357,4 @@ class RyryShortConnectThread(Thread):
|
|
|
281
357
|
|
|
282
358
|
def markStop(self):
|
|
283
359
|
print(f" {self.name} waiting stop")
|
|
284
|
-
self.is_running = False
|
|
360
|
+
self.is_running = False
|
|
@@ -2,7 +2,6 @@ import os
|
|
|
2
2
|
import json
|
|
3
3
|
import time
|
|
4
4
|
import threading
|
|
5
|
-
import tempfile
|
|
6
5
|
import platform
|
|
7
6
|
import subprocess
|
|
8
7
|
from threading import Lock
|
|
@@ -28,17 +27,20 @@ class SharedMemoryService:
|
|
|
28
27
|
def __init__(self):
|
|
29
28
|
self.service_name = "widget_power"
|
|
30
29
|
# 使用系统推荐目录,确保多进程共享
|
|
31
|
-
self.
|
|
32
|
-
self.data_file = os.path.join(self.data_dir, f"{self.service_name}.json")
|
|
33
|
-
self.lock_file = os.path.join(self.data_dir, f"{self.service_name}.lock")
|
|
34
|
-
self.process_count_file = os.path.join(self.data_dir, f"{self.service_name}_process_count.json")
|
|
35
|
-
self.process_count_lock_file = self.process_count_file + '.lock'
|
|
30
|
+
self._set_data_dir(self._get_system_shared_dir())
|
|
36
31
|
self.current_service_name = os.path.basename(sys.argv[0])
|
|
37
|
-
#
|
|
32
|
+
# 确保共享目录存在;锁文件保持固定路径,供其他程序共同读写
|
|
38
33
|
os.makedirs(self.data_dir, exist_ok=True)
|
|
39
34
|
|
|
40
35
|
# 增加启动标志位,避免重复初始化
|
|
41
36
|
self._started = False
|
|
37
|
+
|
|
38
|
+
def _set_data_dir(self, data_dir: str):
|
|
39
|
+
self.data_dir = data_dir
|
|
40
|
+
self.data_file = os.path.join(self.data_dir, f"{self.service_name}.json")
|
|
41
|
+
self.lock_file = os.path.join(self.data_dir, f"{self.service_name}.lock")
|
|
42
|
+
self.process_count_file = os.path.join(self.data_dir, f"{self.service_name}_process_count.json")
|
|
43
|
+
self.process_count_lock_file = self.process_count_file + '.lock'
|
|
42
44
|
|
|
43
45
|
def _get_system_shared_dir(self) -> str:
|
|
44
46
|
"""获取系统推荐的共享目录"""
|
|
@@ -57,6 +59,68 @@ class SharedMemoryService:
|
|
|
57
59
|
else: # Linux 和其他 Unix-like 系统
|
|
58
60
|
# Linux: 使用 /tmp 或 /var/tmp
|
|
59
61
|
return '/tmp/widget_shared_memory'
|
|
62
|
+
|
|
63
|
+
def _chmod_if_possible(self, path: str, mode: int):
|
|
64
|
+
if platform.system() == 'Windows':
|
|
65
|
+
return
|
|
66
|
+
try:
|
|
67
|
+
os.chmod(path, mode)
|
|
68
|
+
except (PermissionError, FileNotFoundError):
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
def _shared_dir_mode(self) -> int:
|
|
72
|
+
if platform.system() == 'Linux':
|
|
73
|
+
return 0o1777
|
|
74
|
+
return 0o700
|
|
75
|
+
|
|
76
|
+
def _shared_file_mode(self) -> int:
|
|
77
|
+
if platform.system() == 'Linux':
|
|
78
|
+
return 0o666
|
|
79
|
+
return 0o600
|
|
80
|
+
|
|
81
|
+
def _prepare_existing_shared_file(self, path: str):
|
|
82
|
+
if os.path.isdir(path):
|
|
83
|
+
raise PermissionError(f"shared memory file path is a directory: {path}")
|
|
84
|
+
if os.path.exists(path):
|
|
85
|
+
self._chmod_if_possible(path, self._shared_file_mode())
|
|
86
|
+
|
|
87
|
+
def _touch_shared_file(self, path: str):
|
|
88
|
+
self._prepare_existing_shared_file(path)
|
|
89
|
+
with open(path, 'a+', encoding='utf-8'):
|
|
90
|
+
pass
|
|
91
|
+
self._chmod_if_possible(path, self._shared_file_mode())
|
|
92
|
+
|
|
93
|
+
def _open_lock_file(self, path: str):
|
|
94
|
+
self._ensure_shared_storage()
|
|
95
|
+
return open(path, 'a+', encoding='utf-8')
|
|
96
|
+
|
|
97
|
+
def _ensure_shared_storage(self):
|
|
98
|
+
"""确保固定共享目录和锁文件可写"""
|
|
99
|
+
os.makedirs(self.data_dir, mode=self._shared_dir_mode(), exist_ok=True)
|
|
100
|
+
self._chmod_if_possible(self.data_dir, self._shared_dir_mode())
|
|
101
|
+
self._touch_shared_file(self.lock_file)
|
|
102
|
+
self._touch_shared_file(self.process_count_lock_file)
|
|
103
|
+
self._prepare_existing_shared_file(self.data_file)
|
|
104
|
+
self._prepare_existing_shared_file(self.process_count_file)
|
|
105
|
+
|
|
106
|
+
def _default_data(self) -> Dict[str, Any]:
|
|
107
|
+
return {
|
|
108
|
+
"max_counter": 999,
|
|
109
|
+
"cur_counter": 0,
|
|
110
|
+
"widget_power": {},
|
|
111
|
+
"last_update": time.time()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
def _validate_counter_data(self, data: Any) -> bool:
|
|
115
|
+
if not isinstance(data, dict):
|
|
116
|
+
return False
|
|
117
|
+
if not isinstance(data.get("max_counter", 999), int):
|
|
118
|
+
return False
|
|
119
|
+
if not isinstance(data.get("cur_counter", 0), int):
|
|
120
|
+
return False
|
|
121
|
+
if not isinstance(data.get("widget_power", {}), dict):
|
|
122
|
+
return False
|
|
123
|
+
return True
|
|
60
124
|
|
|
61
125
|
def _acquire_shared_lock(self, file_obj):
|
|
62
126
|
"""获取共享锁(跨平台)"""
|
|
@@ -102,9 +166,10 @@ class SharedMemoryService:
|
|
|
102
166
|
def operation(data):
|
|
103
167
|
# 验证数据完整性
|
|
104
168
|
if not self._validate_process_data(data):
|
|
105
|
-
data
|
|
169
|
+
data.clear()
|
|
170
|
+
data["processes"] = {}
|
|
106
171
|
|
|
107
|
-
#
|
|
172
|
+
# 清理共享文件中的死亡进程,避免旧状态长期占用任务槽
|
|
108
173
|
self._cleanup_dead_processes_internal(data, current_time, skip_pid=current_pid)
|
|
109
174
|
|
|
110
175
|
# 检查当前进程是否已经注册
|
|
@@ -121,7 +186,7 @@ class SharedMemoryService:
|
|
|
121
186
|
return len(data["processes"])
|
|
122
187
|
|
|
123
188
|
# 原子性地注册进程
|
|
124
|
-
with
|
|
189
|
+
with self._open_lock_file(self.process_count_lock_file) as lock_f:
|
|
125
190
|
self._acquire_exclusive_lock(lock_f)
|
|
126
191
|
try:
|
|
127
192
|
if os.path.exists(self.process_count_file):
|
|
@@ -132,6 +197,7 @@ class SharedMemoryService:
|
|
|
132
197
|
data = {"processes": {}}
|
|
133
198
|
else:
|
|
134
199
|
data = {"processes": {}}
|
|
200
|
+
data = self._normalize_process_data(data)
|
|
135
201
|
|
|
136
202
|
process_count = operation(data)
|
|
137
203
|
|
|
@@ -170,11 +236,7 @@ class SharedMemoryService:
|
|
|
170
236
|
if skip_pid is not None and pid == skip_pid:
|
|
171
237
|
continue # 跳过当前进程
|
|
172
238
|
|
|
173
|
-
#
|
|
174
|
-
if info.get("service_name") != self.current_service_name:
|
|
175
|
-
continue
|
|
176
|
-
|
|
177
|
-
# 检测1: 检查进程是否真的存在
|
|
239
|
+
# 检查进程是否真的存在;其他服务的死进程也要清理,避免共享计数长期卡住
|
|
178
240
|
if not self._is_process_alive(pid):
|
|
179
241
|
dead_processes.append(pid_str)
|
|
180
242
|
continue
|
|
@@ -197,9 +259,11 @@ class SharedMemoryService:
|
|
|
197
259
|
def operation(data):
|
|
198
260
|
# 验证数据完整性
|
|
199
261
|
if not self._validate_process_data(data):
|
|
262
|
+
data.clear()
|
|
263
|
+
data["processes"] = {}
|
|
200
264
|
return 0
|
|
201
265
|
|
|
202
|
-
#
|
|
266
|
+
# 清理共享文件中的死亡进程,避免旧状态长期占用任务槽
|
|
203
267
|
self._cleanup_dead_processes_internal(data, current_time, skip_pid=current_pid)
|
|
204
268
|
|
|
205
269
|
# 注销当前进程(只注销本服务的)
|
|
@@ -209,7 +273,7 @@ class SharedMemoryService:
|
|
|
209
273
|
return len(data["processes"])
|
|
210
274
|
|
|
211
275
|
# 原子性地注销进程
|
|
212
|
-
with
|
|
276
|
+
with self._open_lock_file(self.process_count_lock_file) as lock_f:
|
|
213
277
|
self._acquire_exclusive_lock(lock_f)
|
|
214
278
|
try:
|
|
215
279
|
if os.path.exists(self.process_count_file):
|
|
@@ -217,7 +281,8 @@ class SharedMemoryService:
|
|
|
217
281
|
with open(self.process_count_file, 'r', encoding='utf-8') as f:
|
|
218
282
|
data = json.load(f)
|
|
219
283
|
except (json.JSONDecodeError, UnicodeDecodeError, FileNotFoundError):
|
|
220
|
-
|
|
284
|
+
data = {"processes": {}}
|
|
285
|
+
data = self._normalize_process_data(data)
|
|
221
286
|
|
|
222
287
|
remaining_processes = operation(data)
|
|
223
288
|
|
|
@@ -241,11 +306,10 @@ class SharedMemoryService:
|
|
|
241
306
|
return 0
|
|
242
307
|
|
|
243
308
|
# 验证数据完整性
|
|
244
|
-
|
|
245
|
-
return 0
|
|
309
|
+
data = self._normalize_process_data(data)
|
|
246
310
|
|
|
247
311
|
# 清理死亡进程
|
|
248
|
-
with
|
|
312
|
+
with self._open_lock_file(self.process_count_lock_file) as lock_f:
|
|
249
313
|
self._acquire_exclusive_lock(lock_f)
|
|
250
314
|
try:
|
|
251
315
|
self._cleanup_dead_processes_internal(data, time.time())
|
|
@@ -264,6 +328,8 @@ class SharedMemoryService:
|
|
|
264
328
|
|
|
265
329
|
def operation(data):
|
|
266
330
|
if "processes" not in data:
|
|
331
|
+
data.clear()
|
|
332
|
+
data["processes"] = {}
|
|
267
333
|
return 0
|
|
268
334
|
|
|
269
335
|
dead_processes = []
|
|
@@ -290,7 +356,7 @@ class SharedMemoryService:
|
|
|
290
356
|
return len(data["processes"])
|
|
291
357
|
|
|
292
358
|
# 原子性地清理死亡进程
|
|
293
|
-
with
|
|
359
|
+
with self._open_lock_file(self.process_count_lock_file) as lock_f:
|
|
294
360
|
self._acquire_exclusive_lock(lock_f)
|
|
295
361
|
try:
|
|
296
362
|
if os.path.exists(self.process_count_file):
|
|
@@ -299,6 +365,7 @@ class SharedMemoryService:
|
|
|
299
365
|
data = json.load(f)
|
|
300
366
|
except (json.JSONDecodeError, UnicodeDecodeError, FileNotFoundError):
|
|
301
367
|
data = {"processes": {}}
|
|
368
|
+
data = self._normalize_process_data(data)
|
|
302
369
|
|
|
303
370
|
remaining_processes = operation(data)
|
|
304
371
|
|
|
@@ -358,6 +425,11 @@ class SharedMemoryService:
|
|
|
358
425
|
|
|
359
426
|
except Exception:
|
|
360
427
|
return False
|
|
428
|
+
|
|
429
|
+
def _normalize_process_data(self, data: Any) -> Dict[str, Any]:
|
|
430
|
+
if self._validate_process_data(data):
|
|
431
|
+
return data
|
|
432
|
+
return {"processes": {}}
|
|
361
433
|
|
|
362
434
|
def _update_heartbeat(self):
|
|
363
435
|
"""更新当前进程的心跳(简化版,仅保持数据结构一致性)"""
|
|
@@ -370,7 +442,7 @@ class SharedMemoryService:
|
|
|
370
442
|
return len(data.get("processes", {}))
|
|
371
443
|
|
|
372
444
|
# 原子性地更新心跳
|
|
373
|
-
with
|
|
445
|
+
with self._open_lock_file(self.process_count_lock_file) as lock_f:
|
|
374
446
|
self._acquire_exclusive_lock(lock_f)
|
|
375
447
|
try:
|
|
376
448
|
if os.path.exists(self.process_count_file):
|
|
@@ -379,6 +451,7 @@ class SharedMemoryService:
|
|
|
379
451
|
data = json.load(f)
|
|
380
452
|
except (json.JSONDecodeError, UnicodeDecodeError, FileNotFoundError):
|
|
381
453
|
data = {"processes": {}}
|
|
454
|
+
data = self._normalize_process_data(data)
|
|
382
455
|
|
|
383
456
|
operation(data)
|
|
384
457
|
|
|
@@ -424,15 +497,12 @@ class SharedMemoryService:
|
|
|
424
497
|
data = json.load(f)
|
|
425
498
|
finally:
|
|
426
499
|
self._release_lock(f)
|
|
500
|
+
if not self._validate_counter_data(data):
|
|
501
|
+
return self._default_data()
|
|
427
502
|
return data
|
|
428
|
-
except (FileNotFoundError, json.JSONDecodeError):
|
|
503
|
+
except (FileNotFoundError, json.JSONDecodeError, UnicodeDecodeError):
|
|
429
504
|
# 如果文件不存在或损坏,返回默认数据
|
|
430
|
-
return
|
|
431
|
-
"max_counter": 999,
|
|
432
|
-
"cur_counter": 0,
|
|
433
|
-
"widget_power": {},
|
|
434
|
-
"last_update": time.time()
|
|
435
|
-
}
|
|
505
|
+
return self._default_data()
|
|
436
506
|
|
|
437
507
|
def _write_data(self, data: Dict[str, Any]):
|
|
438
508
|
"""写入数据文件"""
|
|
@@ -459,7 +529,7 @@ class SharedMemoryService:
|
|
|
459
529
|
def _atomic_operation(self, operation):
|
|
460
530
|
"""执行原子操作(多进程安全)"""
|
|
461
531
|
# 使用文件锁确保多进程间的原子性
|
|
462
|
-
with
|
|
532
|
+
with self._open_lock_file(self.lock_file) as lock_f:
|
|
463
533
|
# 获取独占锁,阻塞直到获得锁
|
|
464
534
|
self._acquire_exclusive_lock(lock_f)
|
|
465
535
|
try:
|
|
@@ -469,6 +539,12 @@ class SharedMemoryService:
|
|
|
469
539
|
return result
|
|
470
540
|
finally:
|
|
471
541
|
self._release_lock(lock_f)
|
|
542
|
+
|
|
543
|
+
def ensure_available(self) -> bool:
|
|
544
|
+
"""确认共享计数文件可读写,避免拉到任务后本地无法入队"""
|
|
545
|
+
def operation(data):
|
|
546
|
+
return True
|
|
547
|
+
return self._atomic_operation(operation)
|
|
472
548
|
|
|
473
549
|
def start_service(self):
|
|
474
550
|
"""启动共享内存服务(轻量级,无需额外进程)"""
|
|
@@ -177,7 +177,7 @@ def extend():
|
|
|
177
177
|
_genExtend()
|
|
178
178
|
elif read_data["extend"].get("device_id", "") != read_data.get("deviceInfo", {}).get("device_id", ""):
|
|
179
179
|
_genExtend()
|
|
180
|
-
elif read_data["extend"].get("app", "") == "ryry-cli 6.
|
|
180
|
+
elif read_data["extend"].get("app", "") == "ryry-cli 6.17":
|
|
181
181
|
_genExtend()
|
|
182
182
|
elif any(k not in read_data["extend"] for k in ["device_id", "host_name", "cpu", "memory", "disk", "gpu"]):
|
|
183
183
|
_genExtend()
|
|
@@ -44,7 +44,7 @@ def additionalUrl(srcFile, ossUrl):
|
|
|
44
44
|
except:
|
|
45
45
|
return ossUrl
|
|
46
46
|
|
|
47
|
-
def upload(src, taskUUID=None, needTranscode=False, keepItAlways=False, additionalUrl=False
|
|
47
|
+
def upload(src, taskUUID=None, needTranscode=False, keepItAlways=False, additionalUrl=False):
|
|
48
48
|
import os
|
|
49
49
|
from pathlib import Path
|
|
50
50
|
from ryry import taskUtils as ryry_taskUtils
|
|
@@ -74,8 +74,7 @@ def upload(src, taskUUID=None, needTranscode=False, keepItAlways=False, addition
|
|
|
74
74
|
|
|
75
75
|
# 根据targetDomain判断使用不同的上传逻辑
|
|
76
76
|
ossurl = uploadByDomain(newSrc, targetDomain, taskUUID,
|
|
77
|
-
keepItAlways=keepItAlways,
|
|
78
|
-
uploadPath=uploadPath,
|
|
77
|
+
keepItAlways=keepItAlways,
|
|
79
78
|
needTranscode=needTranscode,
|
|
80
79
|
additionalUrl=additionalUrl)
|
|
81
80
|
|
|
@@ -85,7 +84,9 @@ def upload(src, taskUUID=None, needTranscode=False, keepItAlways=False, addition
|
|
|
85
84
|
os.remove(newSrc)
|
|
86
85
|
return ossurl
|
|
87
86
|
|
|
88
|
-
def uploadByDomain(src, targetDomain,
|
|
87
|
+
def uploadByDomain(src, targetDomain,
|
|
88
|
+
taskUUID, keepItAlways=False,
|
|
89
|
+
needTranscode=False, additionalUrl=False):
|
|
89
90
|
import os
|
|
90
91
|
from pathlib import Path
|
|
91
92
|
import uuid
|
|
@@ -95,8 +96,6 @@ def uploadByDomain(src, targetDomain, taskUUID, keepItAlways=False, uploadPath=N
|
|
|
95
96
|
ext = os.path.splitext(file_name)[-1][1:]
|
|
96
97
|
|
|
97
98
|
parsed_domain, upload_path = parseDomainAndPath(targetDomain)
|
|
98
|
-
if uploadPath:
|
|
99
|
-
upload_path = uploadPath
|
|
100
99
|
if isMecordConfig(parsed_domain):
|
|
101
100
|
from mecord import upload as mecord_upload
|
|
102
101
|
return mecord_upload.upload(src, taskUUID,
|
|
@@ -114,18 +113,6 @@ def uploadByDomain(src, targetDomain, taskUUID, keepItAlways=False, uploadPath=N
|
|
|
114
113
|
ftp_config = getFtpConfig(parsed_domain)
|
|
115
114
|
if ftp_config:
|
|
116
115
|
return ftpUpload(src, file_name, ftp_config, upload_path)
|
|
117
|
-
|
|
118
|
-
if upload_path:
|
|
119
|
-
if "mnt/NAS/mcn" in upload_path or "aigc_output/" in upload_path:
|
|
120
|
-
ftp_config = getFtpConfig("219.136.123.179")
|
|
121
|
-
return ftpUpload(src, file_name, ftp_config, upload_path)
|
|
122
|
-
elif (targetDomain != None and"aigc_output/" in targetDomain):
|
|
123
|
-
ftp_config = getFtpConfig("219.136.123.179")
|
|
124
|
-
return ftpUpload(src, file_name, ftp_config, upload_path)
|
|
125
|
-
elif (targetDomain == None or len(targetDomain) <= 0) and (uploadPath and len(uploadPath) > 0):
|
|
126
|
-
#找不到domain,但是有uploadPath的情况下,默认走ftp
|
|
127
|
-
ftp_config = getFtpConfig("219.136.123.179")
|
|
128
|
-
return ftpUpload(src, file_name, ftp_config, upload_path)
|
|
129
116
|
|
|
130
117
|
from ryry import ryry_webapi
|
|
131
118
|
return ryry_webapi.upload(src, ext,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|