cornflow 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cornflow/app.py +4 -2
- cornflow/cli/__init__.py +4 -0
- cornflow/cli/actions.py +4 -0
- cornflow/cli/config.py +4 -0
- cornflow/cli/migrations.py +13 -8
- cornflow/cli/permissions.py +4 -0
- cornflow/cli/roles.py +5 -1
- cornflow/cli/schemas.py +5 -0
- cornflow/cli/service.py +263 -131
- cornflow/cli/tools/api_generator.py +13 -10
- cornflow/cli/tools/endpoint_tools.py +191 -196
- cornflow/cli/tools/models_tools.py +87 -60
- cornflow/cli/tools/schema_generator.py +161 -67
- cornflow/cli/tools/schemas_tools.py +4 -5
- cornflow/cli/users.py +8 -0
- cornflow/cli/views.py +4 -0
- cornflow/commands/access.py +14 -3
- cornflow/commands/auxiliar.py +106 -0
- cornflow/commands/dag.py +3 -2
- cornflow/commands/permissions.py +186 -81
- cornflow/commands/roles.py +15 -14
- cornflow/commands/schemas.py +6 -4
- cornflow/commands/users.py +12 -17
- cornflow/commands/views.py +171 -41
- cornflow/endpoints/dag.py +27 -25
- cornflow/endpoints/data_check.py +128 -165
- cornflow/endpoints/example_data.py +9 -3
- cornflow/endpoints/execution.py +40 -34
- cornflow/endpoints/health.py +7 -7
- cornflow/endpoints/instance.py +39 -12
- cornflow/endpoints/meta_resource.py +4 -5
- cornflow/schemas/execution.py +9 -1
- cornflow/schemas/health.py +1 -0
- cornflow/shared/authentication/auth.py +76 -45
- cornflow/shared/const.py +10 -1
- cornflow/shared/exceptions.py +3 -1
- cornflow/shared/utils_tables.py +36 -8
- cornflow/shared/validators.py +1 -1
- cornflow/tests/const.py +1 -0
- cornflow/tests/custom_test_case.py +4 -4
- cornflow/tests/unit/test_alarms.py +1 -2
- cornflow/tests/unit/test_cases.py +4 -7
- cornflow/tests/unit/test_executions.py +22 -1
- cornflow/tests/unit/test_external_role_creation.py +785 -0
- cornflow/tests/unit/test_health.py +4 -1
- cornflow/tests/unit/test_log_in.py +46 -9
- cornflow/tests/unit/test_tables.py +3 -3
- {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/METADATA +2 -2
- {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/RECORD +52 -50
- {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/WHEEL +1 -1
- {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/entry_points.txt +0 -0
- {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/top_level.txt +0 -0
cornflow/endpoints/data_check.py
CHANGED
@@ -14,25 +14,105 @@ from cornflow.models import InstanceModel, ExecutionModel, CaseModel, DeployedDA
|
|
14
14
|
from cornflow.schemas.execution import ExecutionDetailsEndpointResponse
|
15
15
|
from cornflow.shared.authentication import Auth, authenticate
|
16
16
|
from cornflow.shared.const import (
|
17
|
+
AIRFLOW_ERROR_MSG,
|
18
|
+
AIRFLOW_NOT_REACHABLE_MSG,
|
19
|
+
DAG_PAUSED_MSG,
|
17
20
|
EXEC_STATE_QUEUED,
|
18
21
|
EXEC_STATE_ERROR,
|
19
22
|
EXEC_STATE_ERROR_START,
|
20
23
|
EXEC_STATE_NOT_RUN,
|
21
|
-
EXECUTION_STATE_MESSAGE_DICT,
|
24
|
+
EXECUTION_STATE_MESSAGE_DICT,
|
25
|
+
PLANNER_ROLE,
|
26
|
+
ADMIN_ROLE,
|
22
27
|
)
|
23
28
|
from cornflow.shared.exceptions import (
|
24
29
|
AirflowError,
|
25
30
|
ObjectDoesNotExist,
|
26
31
|
InvalidUsage,
|
27
|
-
InvalidData
|
32
|
+
InvalidData,
|
28
33
|
)
|
29
34
|
from cornflow.shared.validators import json_schema_validate_as_string
|
30
35
|
|
31
36
|
|
37
|
+
def _run_airflow_data_check(
|
38
|
+
af_client: Airflow,
|
39
|
+
execution: ExecutionModel,
|
40
|
+
schema: str,
|
41
|
+
user,
|
42
|
+
context_str: str,
|
43
|
+
**run_dag_kwargs,
|
44
|
+
):
|
45
|
+
"""
|
46
|
+
Helper function to check Airflow status and run a data check DAG.
|
47
|
+
|
48
|
+
:param af_client: Initialized Airflow client.
|
49
|
+
:param execution: The ExecutionModel object to update.
|
50
|
+
:param schema: The name of the schema/DAG to run.
|
51
|
+
:param user: The user object performing the action.
|
52
|
+
:param context_str: A string describing the context (e.g., "execution {id}") for logging.
|
53
|
+
:param run_dag_kwargs: Additional keyword arguments for af_client.run_dag.
|
54
|
+
:return: None. Updates execution object in place and raises AirflowError on failure.
|
55
|
+
"""
|
56
|
+
# Check Airflow liveness
|
57
|
+
if not af_client.is_alive():
|
58
|
+
current_app.logger.error(AIRFLOW_NOT_REACHABLE_MSG)
|
59
|
+
execution.update_state(EXEC_STATE_ERROR_START)
|
60
|
+
raise AirflowError(
|
61
|
+
error=AIRFLOW_NOT_REACHABLE_MSG,
|
62
|
+
payload=dict(
|
63
|
+
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
64
|
+
state=EXEC_STATE_ERROR_START,
|
65
|
+
),
|
66
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
67
|
+
+ AIRFLOW_NOT_REACHABLE_MSG,
|
68
|
+
)
|
69
|
+
|
70
|
+
# Check if DAG is paused
|
71
|
+
schema_info = af_client.get_dag_info(schema)
|
72
|
+
info = schema_info.json()
|
73
|
+
if info.get("is_paused", False):
|
74
|
+
current_app.logger.error(DAG_PAUSED_MSG)
|
75
|
+
execution.update_state(EXEC_STATE_ERROR_START)
|
76
|
+
raise AirflowError(
|
77
|
+
error=DAG_PAUSED_MSG,
|
78
|
+
payload=dict(
|
79
|
+
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
80
|
+
state=EXEC_STATE_ERROR_START,
|
81
|
+
),
|
82
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
83
|
+
+ DAG_PAUSED_MSG,
|
84
|
+
)
|
85
|
+
|
86
|
+
# Run the DAG
|
87
|
+
try:
|
88
|
+
response = af_client.run_dag(
|
89
|
+
execution.id, dag_name=schema, checks_only=True, **run_dag_kwargs
|
90
|
+
)
|
91
|
+
except AirflowError as err:
|
92
|
+
error = f"{AIRFLOW_ERROR_MSG} {err}"
|
93
|
+
current_app.logger.error(error)
|
94
|
+
execution.update_state(EXEC_STATE_ERROR)
|
95
|
+
raise AirflowError(
|
96
|
+
error=error,
|
97
|
+
payload=dict(
|
98
|
+
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
|
99
|
+
state=EXEC_STATE_ERROR,
|
100
|
+
),
|
101
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
102
|
+
+ error,
|
103
|
+
)
|
104
|
+
|
105
|
+
# Update execution on success
|
106
|
+
af_data = response.json()
|
107
|
+
execution.dag_run_id = af_data.get("dag_run_id")
|
108
|
+
execution.update_state(EXEC_STATE_QUEUED)
|
109
|
+
|
110
|
+
|
32
111
|
class DataCheckExecutionEndpoint(BaseMetaResource):
|
33
112
|
"""
|
34
113
|
Endpoint used to execute the instance and solution checks on an execution
|
35
114
|
"""
|
115
|
+
|
36
116
|
ROLES_WITH_ACCESS = [PLANNER_ROLE, ADMIN_ROLE]
|
37
117
|
|
38
118
|
def __init__(self):
|
@@ -56,13 +136,16 @@ class DataCheckExecutionEndpoint(BaseMetaResource):
|
|
56
136
|
:rtype: Tuple(dict, integer)
|
57
137
|
"""
|
58
138
|
config = current_app.config
|
139
|
+
user = self.get_user()
|
140
|
+
context_str = f"execution {idx}"
|
59
141
|
|
60
142
|
execution = ExecutionModel.get_one_object(user=self.get_user(), idx=idx)
|
61
143
|
if execution is None:
|
62
144
|
err = "The execution to check does not exist"
|
63
145
|
raise ObjectDoesNotExist(
|
64
146
|
error=err,
|
65
|
-
log_txt=f"Error while user {
|
147
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
148
|
+
+ err,
|
66
149
|
)
|
67
150
|
|
68
151
|
schema = execution.schema
|
@@ -72,7 +155,8 @@ class DataCheckExecutionEndpoint(BaseMetaResource):
|
|
72
155
|
err = "The execution is still running"
|
73
156
|
raise InvalidUsage(
|
74
157
|
error=err,
|
75
|
-
log_txt=f"Error while user {
|
158
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
159
|
+
+ err,
|
76
160
|
)
|
77
161
|
|
78
162
|
# this allows testing without airflow interaction:
|
@@ -80,58 +164,12 @@ class DataCheckExecutionEndpoint(BaseMetaResource):
|
|
80
164
|
execution.update_state(EXEC_STATE_NOT_RUN)
|
81
165
|
return execution, 201
|
82
166
|
|
83
|
-
#
|
167
|
+
# Initialize Airflow client
|
84
168
|
af_client = Airflow.from_config(config)
|
85
|
-
if not af_client.is_alive():
|
86
|
-
err = "Airflow is not accessible"
|
87
|
-
current_app.logger.error(err)
|
88
|
-
execution.update_state(EXEC_STATE_ERROR_START)
|
89
|
-
raise AirflowError(
|
90
|
-
error=err,
|
91
|
-
payload=dict(
|
92
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
93
|
-
state=EXEC_STATE_ERROR_START,
|
94
|
-
),
|
95
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + err
|
96
|
-
)
|
97
|
-
# ask airflow if dag_name exists
|
98
|
-
schema_info = af_client.get_dag_info(schema)
|
99
|
-
|
100
|
-
info = schema_info.json()
|
101
|
-
if info["is_paused"]:
|
102
|
-
err = "The dag exists but it is paused in airflow"
|
103
|
-
current_app.logger.error(err)
|
104
|
-
execution.update_state(EXEC_STATE_ERROR_START)
|
105
|
-
raise AirflowError(
|
106
|
-
error=err,
|
107
|
-
payload=dict(
|
108
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
109
|
-
state=EXEC_STATE_ERROR_START,
|
110
|
-
),
|
111
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + err
|
112
|
-
)
|
113
169
|
|
114
|
-
|
115
|
-
|
116
|
-
execution.id, dag_name=schema, checks_only=True
|
117
|
-
)
|
118
|
-
except AirflowError as err:
|
119
|
-
error = "Airflow responded with an error: {}".format(err)
|
120
|
-
current_app.logger.error(error)
|
121
|
-
execution.update_state(EXEC_STATE_ERROR)
|
122
|
-
raise AirflowError(
|
123
|
-
error=error,
|
124
|
-
payload=dict(
|
125
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
|
126
|
-
state=EXEC_STATE_ERROR,
|
127
|
-
),
|
128
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + error
|
129
|
-
)
|
170
|
+
# Call the shared Airflow execution logic
|
171
|
+
_run_airflow_data_check(af_client, execution, schema, user, context_str)
|
130
172
|
|
131
|
-
# if we succeed, we register the dag_run_id in the execution table:
|
132
|
-
af_data = response.json()
|
133
|
-
execution.dag_run_id = af_data["dag_run_id"]
|
134
|
-
execution.update_state(EXEC_STATE_QUEUED)
|
135
173
|
current_app.logger.info(
|
136
174
|
"User {} launches checks of execution {}".format(
|
137
175
|
self.get_user_id(), execution.id
|
@@ -144,7 +182,9 @@ class DataCheckInstanceEndpoint(BaseMetaResource):
|
|
144
182
|
"""
|
145
183
|
Endpoint used to execute the instance and solution checks on an execution
|
146
184
|
"""
|
185
|
+
|
147
186
|
ROLES_WITH_ACCESS = [PLANNER_ROLE, ADMIN_ROLE]
|
187
|
+
|
148
188
|
def __init__(self):
|
149
189
|
super().__init__()
|
150
190
|
self.model = ExecutionModel
|
@@ -169,13 +209,16 @@ class DataCheckInstanceEndpoint(BaseMetaResource):
|
|
169
209
|
:rtype: Tuple(dict, integer)
|
170
210
|
"""
|
171
211
|
config = current_app.config
|
212
|
+
user = self.get_user()
|
213
|
+
context_str = f"instance {idx}"
|
172
214
|
|
173
215
|
instance = InstanceModel.get_one_object(user=self.get_user(), idx=idx)
|
174
216
|
if instance is None:
|
175
217
|
err = "The instance to check does not exist"
|
176
218
|
raise ObjectDoesNotExist(
|
177
219
|
error=err,
|
178
|
-
log_txt=f"Error while user {
|
220
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
221
|
+
+ err,
|
179
222
|
)
|
180
223
|
payload = dict(
|
181
224
|
config=dict(checks_only=True),
|
@@ -185,67 +228,19 @@ class DataCheckInstanceEndpoint(BaseMetaResource):
|
|
185
228
|
)
|
186
229
|
schema = instance.schema
|
187
230
|
|
188
|
-
execution,
|
231
|
+
execution, _ = self.post_list(data=payload)
|
189
232
|
|
190
233
|
# this allows testing without airflow interaction:
|
191
234
|
if request.args.get("run", "1") == "0":
|
192
235
|
execution.update_state(EXEC_STATE_NOT_RUN)
|
193
236
|
return execution, 201
|
194
237
|
|
195
|
-
#
|
238
|
+
# Initialize Airflow client
|
196
239
|
af_client = Airflow.from_config(config)
|
197
|
-
if not af_client.is_alive():
|
198
|
-
err = "Airflow is not accessible"
|
199
|
-
current_app.logger.error(err)
|
200
|
-
execution.update_state(EXEC_STATE_ERROR_START)
|
201
|
-
raise AirflowError(
|
202
|
-
error=err,
|
203
|
-
payload=dict(
|
204
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
205
|
-
state=EXEC_STATE_ERROR_START,
|
206
|
-
),
|
207
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + err
|
208
240
|
|
209
|
-
|
210
|
-
|
211
|
-
schema_info = af_client.get_dag_info(schema)
|
212
|
-
|
213
|
-
info = schema_info.json()
|
214
|
-
if info["is_paused"]:
|
215
|
-
err = "The dag exists but it is paused in airflow"
|
216
|
-
current_app.logger.error(err)
|
217
|
-
execution.update_state(EXEC_STATE_ERROR_START)
|
218
|
-
raise AirflowError(
|
219
|
-
error=err,
|
220
|
-
payload=dict(
|
221
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
222
|
-
state=EXEC_STATE_ERROR_START,
|
223
|
-
),
|
224
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + err
|
225
|
-
|
226
|
-
)
|
241
|
+
# Call the shared Airflow execution logic
|
242
|
+
_run_airflow_data_check(af_client, execution, schema, user, context_str)
|
227
243
|
|
228
|
-
try:
|
229
|
-
response = af_client.run_dag(
|
230
|
-
execution.id, dag_name=schema, checks_only=True
|
231
|
-
)
|
232
|
-
except AirflowError as err:
|
233
|
-
error = "Airflow responded with an error: {}".format(err)
|
234
|
-
current_app.logger.error(error)
|
235
|
-
execution.update_state(EXEC_STATE_ERROR)
|
236
|
-
raise AirflowError(
|
237
|
-
error=error,
|
238
|
-
payload=dict(
|
239
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
|
240
|
-
state=EXEC_STATE_ERROR,
|
241
|
-
),
|
242
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + error
|
243
|
-
)
|
244
|
-
|
245
|
-
# if we succeed, we register the dag_run_id in the execution table:
|
246
|
-
af_data = response.json()
|
247
|
-
execution.dag_run_id = af_data["dag_run_id"]
|
248
|
-
execution.update_state(EXEC_STATE_QUEUED)
|
249
244
|
current_app.logger.info(
|
250
245
|
"User {} creates instance check execution {}".format(
|
251
246
|
self.get_user_id(), execution.id
|
@@ -258,7 +253,9 @@ class DataCheckCaseEndpoint(BaseMetaResource):
|
|
258
253
|
"""
|
259
254
|
Endpoint used to execute the instance and solution checks on an execution
|
260
255
|
"""
|
256
|
+
|
261
257
|
ROLES_WITH_ACCESS = [PLANNER_ROLE, ADMIN_ROLE]
|
258
|
+
|
262
259
|
def __init__(self):
|
263
260
|
super().__init__()
|
264
261
|
self.model = ExecutionModel
|
@@ -283,13 +280,16 @@ class DataCheckCaseEndpoint(BaseMetaResource):
|
|
283
280
|
:rtype: Tuple(dict, integer)
|
284
281
|
"""
|
285
282
|
config = current_app.config
|
283
|
+
user = self.get_user()
|
284
|
+
context_str = f"case {idx}"
|
286
285
|
|
287
286
|
case = CaseModel.get_one_object(user=self.get_user(), idx=idx)
|
288
287
|
if case is None:
|
289
288
|
err = "The case to check does not exist"
|
290
289
|
raise ObjectDoesNotExist(
|
291
290
|
error=err,
|
292
|
-
log_txt=f"Error while user {
|
291
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
292
|
+
+ err,
|
293
293
|
)
|
294
294
|
|
295
295
|
schema = case.schema or "solve_model_dag"
|
@@ -309,14 +309,18 @@ class DataCheckCaseEndpoint(BaseMetaResource):
|
|
309
309
|
if schema == "pulp":
|
310
310
|
validation_schema = "solve_model_dag"
|
311
311
|
|
312
|
-
data_jsonschema = DeployedDAG.get_one_schema(
|
313
|
-
|
312
|
+
data_jsonschema = DeployedDAG.get_one_schema(
|
313
|
+
config, validation_schema, INSTANCE_SCHEMA
|
314
|
+
)
|
315
|
+
validation_errors = json_schema_validate_as_string(
|
316
|
+
data_jsonschema, instance_payload["data"]
|
317
|
+
)
|
314
318
|
|
315
319
|
if validation_errors:
|
316
320
|
raise InvalidData(
|
317
321
|
payload=dict(jsonschema_errors=validation_errors),
|
318
|
-
log_txt=f"Error while user {
|
319
|
-
|
322
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
323
|
+
f"Instance data does not match the jsonschema.",
|
320
324
|
)
|
321
325
|
|
322
326
|
instance, _ = self.post_list(data=instance_payload)
|
@@ -334,14 +338,18 @@ class DataCheckCaseEndpoint(BaseMetaResource):
|
|
334
338
|
|
335
339
|
payload["data"] = case.solution
|
336
340
|
|
337
|
-
data_jsonschema = DeployedDAG.get_one_schema(
|
338
|
-
|
341
|
+
data_jsonschema = DeployedDAG.get_one_schema(
|
342
|
+
config, validation_schema, SOLUTION_SCHEMA
|
343
|
+
)
|
344
|
+
validation_errors = json_schema_validate_as_string(
|
345
|
+
data_jsonschema, payload["data"]
|
346
|
+
)
|
339
347
|
|
340
348
|
if validation_errors:
|
341
349
|
raise InvalidData(
|
342
350
|
payload=dict(jsonschema_errors=validation_errors),
|
343
|
-
log_txt=f"Error while user {
|
344
|
-
|
351
|
+
log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
|
352
|
+
f"Solution data does not match the jsonschema.",
|
345
353
|
)
|
346
354
|
|
347
355
|
self.data_model = ExecutionModel
|
@@ -354,59 +362,14 @@ class DataCheckCaseEndpoint(BaseMetaResource):
|
|
354
362
|
execution.update_state(EXEC_STATE_NOT_RUN)
|
355
363
|
return execution, 201
|
356
364
|
|
357
|
-
#
|
365
|
+
# Initialize Airflow client
|
358
366
|
af_client = Airflow.from_config(config)
|
359
|
-
if not af_client.is_alive():
|
360
|
-
err = "Airflow is not accessible"
|
361
|
-
current_app.logger.error(err)
|
362
|
-
execution.update_state(EXEC_STATE_ERROR_START)
|
363
|
-
raise AirflowError(
|
364
|
-
error=err,
|
365
|
-
payload=dict(
|
366
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
367
|
-
state=EXEC_STATE_ERROR_START,
|
368
|
-
),
|
369
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + err
|
370
|
-
)
|
371
|
-
# ask airflow if dag_name exists
|
372
|
-
schema_info = af_client.get_dag_info(schema)
|
373
|
-
|
374
|
-
info = schema_info.json()
|
375
|
-
if info["is_paused"]:
|
376
|
-
err = "The dag exists but it is paused in airflow"
|
377
|
-
current_app.logger.error(err)
|
378
|
-
execution.update_state(EXEC_STATE_ERROR_START)
|
379
|
-
raise AirflowError(
|
380
|
-
error=err,
|
381
|
-
payload=dict(
|
382
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
383
|
-
state=EXEC_STATE_ERROR_START,
|
384
|
-
),
|
385
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + err
|
386
|
-
)
|
387
|
-
|
388
|
-
try:
|
389
|
-
response = af_client.run_dag(
|
390
|
-
execution.id, dag_name=schema, checks_only=True, case_id=idx
|
391
|
-
)
|
392
367
|
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
raise AirflowError(
|
398
|
-
error=error,
|
399
|
-
payload=dict(
|
400
|
-
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
|
401
|
-
state=EXEC_STATE_ERROR,
|
402
|
-
),
|
403
|
-
log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + error
|
404
|
-
)
|
368
|
+
# Call the shared Airflow execution logic, passing case_id
|
369
|
+
_run_airflow_data_check(
|
370
|
+
af_client, execution, schema, user, context_str, case_id=idx
|
371
|
+
)
|
405
372
|
|
406
|
-
# if we succeed, we register the dag_run_id in the execution table:
|
407
|
-
af_data = response.json()
|
408
|
-
execution.dag_run_id = af_data["dag_run_id"]
|
409
|
-
execution.update_state(EXEC_STATE_QUEUED)
|
410
373
|
current_app.logger.info(
|
411
374
|
"User {} creates case check execution {}".format(
|
412
375
|
self.get_user_id(), execution.id
|
@@ -1,6 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Endpoints to get the example data from a DAG
|
3
3
|
"""
|
4
|
+
|
4
5
|
import json
|
5
6
|
|
6
7
|
from cornflow_client.airflow.api import Airflow
|
@@ -11,7 +12,12 @@ from cornflow.endpoints.meta_resource import BaseMetaResource
|
|
11
12
|
from cornflow.models import PermissionsDAG
|
12
13
|
from cornflow.schemas.example_data import ExampleListData, ExampleDetailData
|
13
14
|
from cornflow.shared.authentication import Auth, authenticate
|
14
|
-
from cornflow.shared.const import
|
15
|
+
from cornflow.shared.const import (
|
16
|
+
AIRFLOW_NOT_REACHABLE_MSG,
|
17
|
+
VIEWER_ROLE,
|
18
|
+
PLANNER_ROLE,
|
19
|
+
ADMIN_ROLE,
|
20
|
+
)
|
15
21
|
from cornflow.shared.exceptions import AirflowError, NoPermission, ObjectDoesNotExist
|
16
22
|
|
17
23
|
|
@@ -44,7 +50,7 @@ class ExampleDataListEndpoint(BaseMetaResource):
|
|
44
50
|
current_app.logger.error(
|
45
51
|
"Airflow not accessible when getting data {}".format(dag_name)
|
46
52
|
)
|
47
|
-
raise AirflowError(error="
|
53
|
+
raise AirflowError(error=f"{AIRFLOW_NOT_REACHABLE_MSG}")
|
48
54
|
|
49
55
|
# try airflow and see if dag_name exists
|
50
56
|
af_client.get_dag_info(dag_name)
|
@@ -87,7 +93,7 @@ class ExampleDataDetailEndpoint(BaseMetaResource):
|
|
87
93
|
current_app.logger.error(
|
88
94
|
"Airflow not accessible when getting data {}".format(dag_name)
|
89
95
|
)
|
90
|
-
raise AirflowError(error="
|
96
|
+
raise AirflowError(error=f"{AIRFLOW_NOT_REACHABLE_MSG}")
|
91
97
|
|
92
98
|
# try airflow and see if dag_name exists
|
93
99
|
af_client.get_dag_info(dag_name)
|
cornflow/endpoints/execution.py
CHANGED
@@ -24,11 +24,14 @@ from cornflow.schemas.execution import (
|
|
24
24
|
ExecutionEditRequest,
|
25
25
|
QueryFiltersExecution,
|
26
26
|
ReLaunchExecutionRequest,
|
27
|
-
ExecutionDetailsWithIndicatorsAndLogResponse
|
27
|
+
ExecutionDetailsWithIndicatorsAndLogResponse,
|
28
28
|
)
|
29
29
|
from cornflow.shared.authentication import Auth, authenticate
|
30
30
|
from cornflow.shared.compress import compressed
|
31
31
|
from cornflow.shared.const import (
|
32
|
+
AIRFLOW_ERROR_MSG,
|
33
|
+
AIRFLOW_NOT_REACHABLE_MSG,
|
34
|
+
DAG_PAUSED_MSG,
|
32
35
|
EXEC_STATE_RUNNING,
|
33
36
|
EXEC_STATE_ERROR,
|
34
37
|
EXEC_STATE_ERROR_START,
|
@@ -100,8 +103,8 @@ class ExecutionEndpoint(BaseMetaResource):
|
|
100
103
|
af_client = Airflow.from_config(current_app.config)
|
101
104
|
if not af_client.is_alive():
|
102
105
|
current_app.logger.warning(
|
103
|
-
"Error while the app tried to update the status of all running executions."
|
104
|
-
"
|
106
|
+
f"Error while the app tried to update the status of all running executions."
|
107
|
+
f"{AIRFLOW_NOT_REACHABLE_MSG}"
|
105
108
|
)
|
106
109
|
continue
|
107
110
|
|
@@ -112,7 +115,7 @@ class ExecutionEndpoint(BaseMetaResource):
|
|
112
115
|
except AirflowError as err:
|
113
116
|
current_app.logger.warning(
|
114
117
|
"Error while the app tried to update the status of all running executions."
|
115
|
-
f"
|
118
|
+
f"{AIRFLOW_ERROR_MSG} {err}"
|
116
119
|
)
|
117
120
|
continue
|
118
121
|
|
@@ -137,18 +140,21 @@ class ExecutionEndpoint(BaseMetaResource):
|
|
137
140
|
the reference_id for the newly created execution if successful) and a integer wit the HTTP status code
|
138
141
|
:rtype: Tuple(dict, integer)
|
139
142
|
"""
|
140
|
-
|
141
|
-
# TODO: should the schema field be cross validated with the instance schema field?
|
143
|
+
|
142
144
|
config = current_app.config
|
143
145
|
|
144
146
|
if "schema" not in kwargs:
|
145
147
|
kwargs["schema"] = "solve_model_dag"
|
146
148
|
|
147
|
-
execution,
|
149
|
+
execution, _ = self.post_list(data=kwargs)
|
148
150
|
instance = InstanceModel.get_one_object(
|
149
151
|
user=self.get_user(), idx=execution.instance_id
|
150
152
|
)
|
151
153
|
|
154
|
+
if execution.schema != instance.schema:
|
155
|
+
execution.delete()
|
156
|
+
raise InvalidData(error="Instance and execution schema mismatch")
|
157
|
+
|
152
158
|
current_app.logger.debug(f"The request is: {request.args.get('run')}")
|
153
159
|
# this allows testing without airflow interaction:
|
154
160
|
if request.args.get("run", "1") == "0":
|
@@ -161,17 +167,17 @@ class ExecutionEndpoint(BaseMetaResource):
|
|
161
167
|
# We now try to launch the task in airflow
|
162
168
|
af_client = Airflow.from_config(config)
|
163
169
|
if not af_client.is_alive():
|
164
|
-
|
165
|
-
current_app.logger.error(
|
170
|
+
|
171
|
+
current_app.logger.error(AIRFLOW_NOT_REACHABLE_MSG)
|
166
172
|
execution.update_state(EXEC_STATE_ERROR_START)
|
167
173
|
raise AirflowError(
|
168
|
-
error=
|
174
|
+
error=AIRFLOW_NOT_REACHABLE_MSG,
|
169
175
|
payload=dict(
|
170
176
|
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
171
177
|
state=EXEC_STATE_ERROR_START,
|
172
178
|
),
|
173
179
|
log_txt=f"Error while user {self.get_user()} tries to create an execution "
|
174
|
-
+
|
180
|
+
+ AIRFLOW_NOT_REACHABLE_MSG,
|
175
181
|
)
|
176
182
|
# ask airflow if dag_name exists
|
177
183
|
schema = execution.schema
|
@@ -231,23 +237,23 @@ class ExecutionEndpoint(BaseMetaResource):
|
|
231
237
|
|
232
238
|
info = schema_info.json()
|
233
239
|
if info["is_paused"]:
|
234
|
-
|
235
|
-
current_app.logger.error(
|
240
|
+
|
241
|
+
current_app.logger.error(DAG_PAUSED_MSG)
|
236
242
|
execution.update_state(EXEC_STATE_ERROR_START)
|
237
243
|
raise AirflowError(
|
238
|
-
error=
|
244
|
+
error=DAG_PAUSED_MSG,
|
239
245
|
payload=dict(
|
240
246
|
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
241
247
|
state=EXEC_STATE_ERROR_START,
|
242
248
|
),
|
243
249
|
log_txt=f"Error while user {self.get_user()} tries to create an execution. "
|
244
|
-
+
|
250
|
+
+ DAG_PAUSED_MSG,
|
245
251
|
)
|
246
252
|
|
247
253
|
try:
|
248
254
|
response = af_client.run_dag(execution.id, dag_name=schema)
|
249
255
|
except AirflowError as err:
|
250
|
-
error = "
|
256
|
+
error = f"{AIRFLOW_ERROR_MSG} {err}"
|
251
257
|
current_app.logger.error(error)
|
252
258
|
execution.update_state(EXEC_STATE_ERROR)
|
253
259
|
raise AirflowError(
|
@@ -339,17 +345,17 @@ class ExecutionRelaunchEndpoint(BaseMetaResource):
|
|
339
345
|
# We now try to launch the task in airflow
|
340
346
|
af_client = Airflow.from_config(config)
|
341
347
|
if not af_client.is_alive():
|
342
|
-
|
343
|
-
current_app.logger.error(
|
348
|
+
|
349
|
+
current_app.logger.error(AIRFLOW_NOT_REACHABLE_MSG)
|
344
350
|
execution.update_state(EXEC_STATE_ERROR_START)
|
345
351
|
raise AirflowError(
|
346
|
-
error=
|
352
|
+
error=AIRFLOW_NOT_REACHABLE_MSG,
|
347
353
|
payload=dict(
|
348
354
|
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
349
355
|
state=EXEC_STATE_ERROR_START,
|
350
356
|
),
|
351
357
|
log_txt=f"Error while user {self.get_user()} tries to relaunch execution {idx}. "
|
352
|
-
+
|
358
|
+
+ AIRFLOW_NOT_REACHABLE_MSG,
|
353
359
|
)
|
354
360
|
# ask airflow if dag_name exists
|
355
361
|
schema = execution.schema
|
@@ -357,23 +363,23 @@ class ExecutionRelaunchEndpoint(BaseMetaResource):
|
|
357
363
|
|
358
364
|
info = schema_info.json()
|
359
365
|
if info["is_paused"]:
|
360
|
-
|
361
|
-
current_app.logger.error(
|
366
|
+
|
367
|
+
current_app.logger.error(DAG_PAUSED_MSG)
|
362
368
|
execution.update_state(EXEC_STATE_ERROR_START)
|
363
369
|
raise AirflowError(
|
364
|
-
error=
|
370
|
+
error=DAG_PAUSED_MSG,
|
365
371
|
payload=dict(
|
366
372
|
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
|
367
373
|
state=EXEC_STATE_ERROR_START,
|
368
374
|
),
|
369
375
|
log_txt=f"Error while user {self.get_user()} tries to relaunch execution {idx}. "
|
370
|
-
+
|
376
|
+
+ DAG_PAUSED_MSG,
|
371
377
|
)
|
372
378
|
|
373
379
|
try:
|
374
380
|
response = af_client.run_dag(execution.id, dag_name=schema)
|
375
381
|
except AirflowError as err:
|
376
|
-
error = "
|
382
|
+
error = f"{AIRFLOW_ERROR_MSG} {err}"
|
377
383
|
current_app.logger.error(error)
|
378
384
|
execution.update_state(EXEC_STATE_ERROR)
|
379
385
|
raise AirflowError(
|
@@ -490,13 +496,13 @@ class ExecutionDetailsEndpoint(ExecutionDetailsEndpointBase):
|
|
490
496
|
)
|
491
497
|
af_client = Airflow.from_config(current_app.config)
|
492
498
|
if not af_client.is_alive():
|
493
|
-
|
499
|
+
|
494
500
|
raise AirflowError(
|
495
|
-
error=
|
501
|
+
error=AIRFLOW_NOT_REACHABLE_MSG,
|
496
502
|
log_txt=f"Error while user {self.get_user()} tries to stop execution {idx}. "
|
497
|
-
+
|
503
|
+
+ AIRFLOW_NOT_REACHABLE_MSG,
|
498
504
|
)
|
499
|
-
|
505
|
+
af_client.set_dag_run_to_fail(
|
500
506
|
dag_name=execution.schema, dag_run_id=execution.dag_run_id
|
501
507
|
)
|
502
508
|
execution.update_state(EXEC_STATE_STOPPED)
|
@@ -563,21 +569,21 @@ class ExecutionStatusEndpoint(BaseMetaResource):
|
|
563
569
|
|
564
570
|
af_client = Airflow.from_config(current_app.config)
|
565
571
|
if not af_client.is_alive():
|
566
|
-
|
572
|
+
|
567
573
|
_raise_af_error(
|
568
574
|
execution,
|
569
|
-
|
575
|
+
AIRFLOW_NOT_REACHABLE_MSG,
|
570
576
|
log_txt=f"Error while user {self.get_user()} tries to get the status of execution {idx}. "
|
571
|
-
+
|
577
|
+
+ AIRFLOW_NOT_REACHABLE_MSG,
|
572
578
|
)
|
573
579
|
|
574
580
|
try:
|
575
|
-
|
581
|
+
|
576
582
|
response = af_client.get_dag_run_status(
|
577
583
|
dag_name=execution.schema, dag_run_id=dag_run_id
|
578
584
|
)
|
579
585
|
except AirflowError as err:
|
580
|
-
error = f"
|
586
|
+
error = f"{AIRFLOW_ERROR_MSG} {err}"
|
581
587
|
_raise_af_error(
|
582
588
|
execution,
|
583
589
|
error,
|