psr-factory 5.0.0b16__py3-none-win_amd64.whl → 5.0.0b67__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psr/execqueue/client.py +35 -14
- psr/execqueue/config.py +8 -0
- psr/execqueue/db.py +77 -11
- psr/execqueue/server.py +363 -57
- psr/execqueue/watcher.py +25 -11
- psr/factory/__init__.py +1 -1
- psr/factory/api.py +814 -523
- psr/factory/factory.dll +0 -0
- psr/factory/factory.pmd +120 -51
- psr/factory/factory.pmk +3028 -2199
- psr/factory/factorylib.py +32 -2
- psr/factory/samples/sddp_case01.py +3 -2
- psr/factory/samples/sddp_case21.py +3 -3
- psr/{cloud/version.py → outputs/__init__.py} +2 -2
- psr/outputs/outputs.py +179 -0
- psr/outputs/resample.py +289 -0
- psr/psrfcommon/psrfcommon.py +4 -1
- psr/runner/runner.py +74 -23
- psr_factory-5.0.0b67.dist-info/METADATA +47 -0
- psr_factory-5.0.0b67.dist-info/RECORD +32 -0
- psr/cloud/__init__.py +0 -7
- psr/cloud/aws.py +0 -256
- psr/cloud/cloud.py +0 -1444
- psr/cloud/data.py +0 -127
- psr/cloud/desktop.py +0 -82
- psr/cloud/log.py +0 -40
- psr/cloud/status.py +0 -81
- psr/cloud/tempfile.py +0 -117
- psr/cloud/xml.py +0 -57
- psr/factory/libcurl-x64.dll +0 -0
- psr_factory-5.0.0b16.dist-info/METADATA +0 -123
- psr_factory-5.0.0b16.dist-info/RECORD +0 -40
- {psr_factory-5.0.0b16.dist-info → psr_factory-5.0.0b67.dist-info}/WHEEL +0 -0
- {psr_factory-5.0.0b16.dist-info → psr_factory-5.0.0b67.dist-info}/licenses/LICENSE.txt +0 -0
- {psr_factory-5.0.0b16.dist-info → psr_factory-5.0.0b67.dist-info}/top_level.txt +0 -0
psr/execqueue/client.py
CHANGED
|
@@ -28,12 +28,23 @@ def upload_case_file(zip_path, server_url):
|
|
|
28
28
|
print("Upload failed:", response.text)
|
|
29
29
|
return None
|
|
30
30
|
|
|
31
|
+
def run_module(case_id: str, module_name: str, server_url: str) -> Optional[str]:
|
|
32
|
+
"""Add a module to the execution queue. Returns the execution id."""
|
|
33
|
+
data = {"case_id": case_id, "module_name": module_name}
|
|
34
|
+
response = requests.post(f"{server_url}/run_module",data=data)
|
|
35
|
+
|
|
36
|
+
if response.status_code == 200:
|
|
37
|
+
print("Added to execution queue successfully!")
|
|
38
|
+
print("Execution ID:", response.json().get('execution_id'))
|
|
39
|
+
return response.json().get('execution_id')
|
|
40
|
+
else:
|
|
41
|
+
print("Module enqueue failed:", response.status_code, response.text)
|
|
42
|
+
return None
|
|
31
43
|
|
|
32
44
|
def run_case(case_id: str, server_url: str, cloud_execution: bool = False) -> Optional[str]:
|
|
33
45
|
"""Add a case to the execution queue. For server-local run,
|
|
34
46
|
returns the execution id. For cloud run, returns the cloud upload id."""
|
|
35
|
-
|
|
36
|
-
response = requests.post(f"{server_url}/run",data=data)
|
|
47
|
+
response = requests.post(f"{server_url}/run" , params={'cloud_execution': cloud_execution})
|
|
37
48
|
|
|
38
49
|
if response.status_code == 200:
|
|
39
50
|
if not cloud_execution:
|
|
@@ -45,7 +56,20 @@ def run_case(case_id: str, server_url: str, cloud_execution: bool = False) -> Op
|
|
|
45
56
|
print("Cloud upload ID:", response.json().get('cloud_upload_id'))
|
|
46
57
|
return response.json().get('cloud_upload_id')
|
|
47
58
|
else:
|
|
48
|
-
print("
|
|
59
|
+
print("Run case failed:", response.status_code, response.text)
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def get_module_log(case_id: str, server_url: str, module_name: Optional[str] = None) -> Optional[str]:
|
|
64
|
+
"""Fetch the content of a module's fixed log file. If module_name is None, returns last module run log."""
|
|
65
|
+
params = {}
|
|
66
|
+
if module_name:
|
|
67
|
+
params['module'] = module_name
|
|
68
|
+
response = requests.get(f"{server_url}/module_log/{case_id}", params=params)
|
|
69
|
+
if response.status_code == 200:
|
|
70
|
+
return response.text
|
|
71
|
+
else:
|
|
72
|
+
print("Fetch module log failed:", response.text)
|
|
49
73
|
return None
|
|
50
74
|
|
|
51
75
|
|
|
@@ -53,8 +77,7 @@ def upload_and_run_file(zip_path: str, server_url: str, cloud_execution: bool =
|
|
|
53
77
|
"""Upload a zip file to the server."""
|
|
54
78
|
with open(zip_path, 'rb') as f:
|
|
55
79
|
files = {'file': (os.path.basename(zip_path), f)}
|
|
56
|
-
|
|
57
|
-
response = requests.post(f"{server_url}/upload_and_run", files=files, data=data)
|
|
80
|
+
response = requests.post(f"{server_url}/upload_and_run", files=files , params={'cloud_execution': cloud_execution} )
|
|
58
81
|
|
|
59
82
|
if response.status_code == 200:
|
|
60
83
|
print("Upload successful! Waiting for execution.")
|
|
@@ -70,17 +93,16 @@ def upload_and_run_file(zip_path: str, server_url: str, cloud_execution: bool =
|
|
|
70
93
|
return None
|
|
71
94
|
|
|
72
95
|
|
|
73
|
-
def get_execution_status(execution_id: str, server_url: str, cloud_execution: bool = False
|
|
96
|
+
def get_execution_status(execution_id: str, server_url: str, cloud_execution: bool = False) -> Optional[tuple[int, str]]:
|
|
74
97
|
"""Get the status of an execution."""
|
|
75
|
-
|
|
76
|
-
response = requests.get(f"{server_url}/status/{execution_id}",
|
|
98
|
+
print("Getting status for execution ID:", execution_id)
|
|
99
|
+
response = requests.get(f"{server_url}/status/{execution_id}", params={'cloud_execution': cloud_execution})
|
|
77
100
|
result = response.status_code == 200
|
|
78
|
-
return
|
|
79
|
-
|
|
101
|
+
return response.json().get('status_id'), response.json().get('status_msg') if result else None
|
|
80
102
|
|
|
81
103
|
def get_results(execution_id, server_url, cloud_execution=False) -> Optional[List[str]]:
|
|
82
104
|
"""Download the results of an execution."""
|
|
83
|
-
response = requests.get(f"{server_url}/results/{execution_id}",
|
|
105
|
+
response = requests.get(f"{server_url}/results/{execution_id}", params={'cloud_execution': cloud_execution})
|
|
84
106
|
|
|
85
107
|
if response.status_code == 200:
|
|
86
108
|
print("Results downloaded successfully!")
|
|
@@ -88,14 +110,13 @@ def get_results(execution_id, server_url, cloud_execution=False) -> Optional[Lis
|
|
|
88
110
|
print("Files:", files)
|
|
89
111
|
return files
|
|
90
112
|
else:
|
|
91
|
-
print("
|
|
113
|
+
print("Results download failed:", response.text)
|
|
92
114
|
return None
|
|
93
115
|
|
|
94
116
|
|
|
95
117
|
def download_execution_file(execution_id: str, server_url: str, file: str, download_path: str, cloud_execution: bool = False):
|
|
96
118
|
"""Download the results of an execution."""
|
|
97
|
-
|
|
98
|
-
response = requests.get(f"{server_url}/results/{execution_id}/{file}", data=data)
|
|
119
|
+
response = requests.get(f"{server_url}/results/{execution_id}/{file}", params={'cloud_execution': cloud_execution})
|
|
99
120
|
|
|
100
121
|
# TODO: add validation for download_path existence.
|
|
101
122
|
if response.status_code == 200:
|
psr/execqueue/config.py
CHANGED
|
@@ -4,6 +4,7 @@ import tomllib
|
|
|
4
4
|
__version__ = "0.3.0"
|
|
5
5
|
_app_name = "PSR Factory ExecQueue"
|
|
6
6
|
DEFAULT_PORT = 5000
|
|
7
|
+
DEFAULT_HOST = "127.0.0.1"
|
|
7
8
|
FLASK_DEBUG = False
|
|
8
9
|
_SETTINGS_FILE_PATH = "server_settings.toml"
|
|
9
10
|
|
|
@@ -42,3 +43,10 @@ CLOUD_RESULTS_FOLDER = os.path.join(STORAGE_PATH, 'cloud_results')
|
|
|
42
43
|
# Where temporary extracted case files will be stored
|
|
43
44
|
TEMPORARY_UPLOAD_FOLDER = os.path.join(STORAGE_PATH, 'tmp')
|
|
44
45
|
|
|
46
|
+
# Optional: modules configuration
|
|
47
|
+
# Expected format in server_settings.toml:
|
|
48
|
+
# [modules.<name>]
|
|
49
|
+
# command = "python some_script.py --case \"{case_path}\""
|
|
50
|
+
# log_file = "<optional fixed log file name>"
|
|
51
|
+
MODULES = settings.get("modules", {})
|
|
52
|
+
|
psr/execqueue/db.py
CHANGED
|
@@ -24,6 +24,7 @@ class CloudStatus(Enum):
|
|
|
24
24
|
FINISHED = 3
|
|
25
25
|
ERROR = 4
|
|
26
26
|
RESULTS_AVAILABLE = 5
|
|
27
|
+
LOGS_AVAILABLE_ERROR = 6
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
DB_NAME = "app.db"
|
|
@@ -61,7 +62,10 @@ class LocalExecution(Base):
|
|
|
61
62
|
case_id = Column(String(26), ForeignKey('cases.case_id'))
|
|
62
63
|
start_time = Column(TIMESTAMP, default=datetime.datetime.utcnow)
|
|
63
64
|
finish_time = Column(TIMESTAMP)
|
|
64
|
-
status = Column(Integer, default=
|
|
65
|
+
status = Column(Integer, default=LOCAL_EXECUTION_RUNNING)
|
|
66
|
+
# Module-related fields
|
|
67
|
+
is_module = Column(Integer, default=0) # 0 = no, 1 = yes
|
|
68
|
+
module = Column(String(128)) # optional module name
|
|
65
69
|
|
|
66
70
|
case = relationship("Case", back_populates="local_executions")
|
|
67
71
|
|
|
@@ -102,6 +106,23 @@ def initialize():
|
|
|
102
106
|
if _create_db:
|
|
103
107
|
Base.metadata.create_all(engine)
|
|
104
108
|
_first_time_setup(session)
|
|
109
|
+
else:
|
|
110
|
+
# Basic migration: add columns if missing
|
|
111
|
+
# Note: SQLite supports limited ALTERs; use simple try/except for idempotency
|
|
112
|
+
from sqlalchemy import inspect
|
|
113
|
+
inspector = inspect(engine)
|
|
114
|
+
cols = {c['name'] for c in inspector.get_columns('local_executions')}
|
|
115
|
+
with engine.connect() as conn:
|
|
116
|
+
if 'is_module' not in cols:
|
|
117
|
+
try:
|
|
118
|
+
conn.execute("ALTER TABLE local_executions ADD COLUMN is_module INTEGER DEFAULT 0")
|
|
119
|
+
except Exception:
|
|
120
|
+
pass
|
|
121
|
+
if 'module' not in cols:
|
|
122
|
+
try:
|
|
123
|
+
conn.execute("ALTER TABLE local_executions ADD COLUMN module VARCHAR(128)")
|
|
124
|
+
except Exception:
|
|
125
|
+
pass
|
|
105
126
|
|
|
106
127
|
return session, engine
|
|
107
128
|
|
|
@@ -126,12 +147,15 @@ def register_case(session, case_id, checksum):
|
|
|
126
147
|
return case
|
|
127
148
|
|
|
128
149
|
|
|
129
|
-
def register_local_execution(session, case_id: str, execution_id: str):
|
|
150
|
+
def register_local_execution(session, case_id: str, execution_id: str, *, is_module: int = 0, module: Optional[str] = None):
|
|
130
151
|
case = session.query(Case).filter(Case.case_id == case_id).first()
|
|
131
|
-
local_execution = LocalExecution(
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
152
|
+
local_execution = LocalExecution(
|
|
153
|
+
execution_id=execution_id,
|
|
154
|
+
case_id=case_id,
|
|
155
|
+
start_time=datetime.datetime.utcnow(),
|
|
156
|
+
is_module=is_module,
|
|
157
|
+
module=module,
|
|
158
|
+
)
|
|
135
159
|
case.local_executions.append(local_execution)
|
|
136
160
|
session.commit()
|
|
137
161
|
return local_execution
|
|
@@ -142,7 +166,7 @@ def get_case_id_from_execution_id(session, execution_id: str) -> Optional[str]:
|
|
|
142
166
|
return local_execution.case_id if local_execution else None
|
|
143
167
|
|
|
144
168
|
|
|
145
|
-
def update_local_execution_status(session, execution_id: str, status: int):
|
|
169
|
+
def update_local_execution_status(session, execution_id: str, status: int) -> bool:
|
|
146
170
|
local_execution = session.query(LocalExecution).filter(LocalExecution.execution_id == execution_id).first()
|
|
147
171
|
if local_execution:
|
|
148
172
|
if status not in [LOCAL_EXECUTION_FINISHED, LOCAL_EXECUTION_ERROR,
|
|
@@ -153,21 +177,60 @@ def update_local_execution_status(session, execution_id: str, status: int):
|
|
|
153
177
|
local_execution.finish_time = datetime.datetime.utcnow()
|
|
154
178
|
session.commit()
|
|
155
179
|
return True
|
|
180
|
+
return True
|
|
156
181
|
|
|
157
|
-
|
|
182
|
+
|
|
183
|
+
def update_cloud_execution_status(session, repository_id: int, status: int) -> bool:
|
|
158
184
|
cloud_execution = session.query(CloudExecution).filter(CloudExecution.repository_id == repository_id).first()
|
|
159
185
|
if cloud_execution:
|
|
160
|
-
if status not in CloudStatus:
|
|
186
|
+
if CloudStatus(status) not in CloudStatus:
|
|
161
187
|
raise ValueError("Wrong status for update.")
|
|
162
188
|
cloud_execution.status = status
|
|
163
189
|
session.commit()
|
|
164
190
|
return True
|
|
165
|
-
|
|
191
|
+
return False
|
|
192
|
+
|
|
193
|
+
|
|
166
194
|
def get_local_execution_status(session, execution_id: str) -> Optional[int]:
|
|
167
195
|
local_execution = session.query(LocalExecution).filter(LocalExecution.execution_id == execution_id).first()
|
|
168
196
|
return local_execution.status if local_execution else None
|
|
169
197
|
|
|
170
198
|
|
|
199
|
+
def any_running_modules_for_case(session, case_id: str) -> bool:
|
|
200
|
+
return session.query(LocalExecution).filter(
|
|
201
|
+
LocalExecution.case_id == case_id,
|
|
202
|
+
LocalExecution.is_module == 1,
|
|
203
|
+
LocalExecution.status == LOCAL_EXECUTION_RUNNING
|
|
204
|
+
).count() > 0
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def any_failed_modules_for_case(session, case_id: str) -> bool:
|
|
208
|
+
return session.query(LocalExecution).filter(
|
|
209
|
+
LocalExecution.case_id == case_id,
|
|
210
|
+
LocalExecution.is_module == 1,
|
|
211
|
+
LocalExecution.status == LOCAL_EXECUTION_ERROR
|
|
212
|
+
).count() > 0
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def last_module_execution_for_case(session, case_id: str, module: Optional[str] = None) -> Optional[LocalExecution]:
|
|
216
|
+
q = session.query(LocalExecution).filter(
|
|
217
|
+
LocalExecution.case_id == case_id,
|
|
218
|
+
LocalExecution.is_module == 1
|
|
219
|
+
)
|
|
220
|
+
if module:
|
|
221
|
+
q = q.filter(LocalExecution.module == module)
|
|
222
|
+
return q.order_by(LocalExecution.start_time.desc()).first()
|
|
223
|
+
|
|
224
|
+
def get_distinct_module_names_for_case(session, case_id: str) -> List[str]:
|
|
225
|
+
rows = session.query(LocalExecution.module).filter(
|
|
226
|
+
LocalExecution.case_id == case_id,
|
|
227
|
+
LocalExecution.is_module == 1,
|
|
228
|
+
LocalExecution.module.isnot(None)
|
|
229
|
+
).distinct().all()
|
|
230
|
+
# rows is a list of tuples [(module,), ...]
|
|
231
|
+
return [r[0] for r in rows if r and r[0]]
|
|
232
|
+
|
|
233
|
+
|
|
171
234
|
def register_cloud_upload(session, case_id: str, cloud_upload_id: str):
|
|
172
235
|
case = session.query(Case).filter(Case.case_id == case_id).first()
|
|
173
236
|
cloud_upload = CloudUpload(cloud_upload_id=cloud_upload_id,
|
|
@@ -217,4 +280,7 @@ def get_cloud_execution_status(session, repository_id: int) -> Optional[int]:
|
|
|
217
280
|
return cloud_execution.status if cloud_execution else None
|
|
218
281
|
|
|
219
282
|
def get_cloud_finished_executions(session) -> List[CloudExecution]:
|
|
220
|
-
return session.query(CloudExecution).filter(CloudExecution.status == CloudStatus.FINISHED.value).all()
|
|
283
|
+
return session.query(CloudExecution).filter(CloudExecution.status == CloudStatus.FINISHED.value).all()
|
|
284
|
+
|
|
285
|
+
def get_cloud_failed_executions(session) -> List[CloudExecution]:
|
|
286
|
+
return session.query(CloudExecution).filter(CloudExecution.status == CloudStatus.ERROR.value).all()
|