cornflow 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. cornflow/app.py +4 -2
  2. cornflow/cli/__init__.py +4 -0
  3. cornflow/cli/actions.py +4 -0
  4. cornflow/cli/config.py +4 -0
  5. cornflow/cli/migrations.py +13 -8
  6. cornflow/cli/permissions.py +4 -0
  7. cornflow/cli/roles.py +5 -1
  8. cornflow/cli/schemas.py +5 -0
  9. cornflow/cli/service.py +263 -131
  10. cornflow/cli/tools/api_generator.py +13 -10
  11. cornflow/cli/tools/endpoint_tools.py +191 -196
  12. cornflow/cli/tools/models_tools.py +87 -60
  13. cornflow/cli/tools/schema_generator.py +161 -67
  14. cornflow/cli/tools/schemas_tools.py +4 -5
  15. cornflow/cli/users.py +8 -0
  16. cornflow/cli/views.py +4 -0
  17. cornflow/commands/access.py +14 -3
  18. cornflow/commands/auxiliar.py +106 -0
  19. cornflow/commands/dag.py +3 -2
  20. cornflow/commands/permissions.py +186 -81
  21. cornflow/commands/roles.py +15 -14
  22. cornflow/commands/schemas.py +6 -4
  23. cornflow/commands/users.py +12 -17
  24. cornflow/commands/views.py +171 -41
  25. cornflow/endpoints/dag.py +27 -25
  26. cornflow/endpoints/data_check.py +128 -165
  27. cornflow/endpoints/example_data.py +9 -3
  28. cornflow/endpoints/execution.py +40 -34
  29. cornflow/endpoints/health.py +7 -7
  30. cornflow/endpoints/instance.py +39 -12
  31. cornflow/endpoints/meta_resource.py +4 -5
  32. cornflow/schemas/execution.py +9 -1
  33. cornflow/schemas/health.py +1 -0
  34. cornflow/shared/authentication/auth.py +76 -45
  35. cornflow/shared/const.py +10 -1
  36. cornflow/shared/exceptions.py +3 -1
  37. cornflow/shared/utils_tables.py +36 -8
  38. cornflow/shared/validators.py +1 -1
  39. cornflow/tests/const.py +1 -0
  40. cornflow/tests/custom_test_case.py +4 -4
  41. cornflow/tests/unit/test_alarms.py +1 -2
  42. cornflow/tests/unit/test_cases.py +4 -7
  43. cornflow/tests/unit/test_executions.py +22 -1
  44. cornflow/tests/unit/test_external_role_creation.py +785 -0
  45. cornflow/tests/unit/test_health.py +4 -1
  46. cornflow/tests/unit/test_log_in.py +46 -9
  47. cornflow/tests/unit/test_tables.py +3 -3
  48. {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/METADATA +2 -2
  49. {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/RECORD +52 -50
  50. {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/WHEEL +1 -1
  51. {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/entry_points.txt +0 -0
  52. {cornflow-1.2.1.dist-info → cornflow-1.2.3.dist-info}/top_level.txt +0 -0
@@ -14,25 +14,105 @@ from cornflow.models import InstanceModel, ExecutionModel, CaseModel, DeployedDA
14
14
  from cornflow.schemas.execution import ExecutionDetailsEndpointResponse
15
15
  from cornflow.shared.authentication import Auth, authenticate
16
16
  from cornflow.shared.const import (
17
+ AIRFLOW_ERROR_MSG,
18
+ AIRFLOW_NOT_REACHABLE_MSG,
19
+ DAG_PAUSED_MSG,
17
20
  EXEC_STATE_QUEUED,
18
21
  EXEC_STATE_ERROR,
19
22
  EXEC_STATE_ERROR_START,
20
23
  EXEC_STATE_NOT_RUN,
21
- EXECUTION_STATE_MESSAGE_DICT, VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE,
24
+ EXECUTION_STATE_MESSAGE_DICT,
25
+ PLANNER_ROLE,
26
+ ADMIN_ROLE,
22
27
  )
23
28
  from cornflow.shared.exceptions import (
24
29
  AirflowError,
25
30
  ObjectDoesNotExist,
26
31
  InvalidUsage,
27
- InvalidData
32
+ InvalidData,
28
33
  )
29
34
  from cornflow.shared.validators import json_schema_validate_as_string
30
35
 
31
36
 
37
+ def _run_airflow_data_check(
38
+ af_client: Airflow,
39
+ execution: ExecutionModel,
40
+ schema: str,
41
+ user,
42
+ context_str: str,
43
+ **run_dag_kwargs,
44
+ ):
45
+ """
46
+ Helper function to check Airflow status and run a data check DAG.
47
+
48
+ :param af_client: Initialized Airflow client.
49
+ :param execution: The ExecutionModel object to update.
50
+ :param schema: The name of the schema/DAG to run.
51
+ :param user: The user object performing the action.
52
+ :param context_str: A string describing the context (e.g., "execution {id}") for logging.
53
+ :param run_dag_kwargs: Additional keyword arguments for af_client.run_dag.
54
+ :return: None. Updates execution object in place and raises AirflowError on failure.
55
+ """
56
+ # Check Airflow liveness
57
+ if not af_client.is_alive():
58
+ current_app.logger.error(AIRFLOW_NOT_REACHABLE_MSG)
59
+ execution.update_state(EXEC_STATE_ERROR_START)
60
+ raise AirflowError(
61
+ error=AIRFLOW_NOT_REACHABLE_MSG,
62
+ payload=dict(
63
+ message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
64
+ state=EXEC_STATE_ERROR_START,
65
+ ),
66
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
67
+ + AIRFLOW_NOT_REACHABLE_MSG,
68
+ )
69
+
70
+ # Check if DAG is paused
71
+ schema_info = af_client.get_dag_info(schema)
72
+ info = schema_info.json()
73
+ if info.get("is_paused", False):
74
+ current_app.logger.error(DAG_PAUSED_MSG)
75
+ execution.update_state(EXEC_STATE_ERROR_START)
76
+ raise AirflowError(
77
+ error=DAG_PAUSED_MSG,
78
+ payload=dict(
79
+ message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
80
+ state=EXEC_STATE_ERROR_START,
81
+ ),
82
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
83
+ + DAG_PAUSED_MSG,
84
+ )
85
+
86
+ # Run the DAG
87
+ try:
88
+ response = af_client.run_dag(
89
+ execution.id, dag_name=schema, checks_only=True, **run_dag_kwargs
90
+ )
91
+ except AirflowError as err:
92
+ error = f"{AIRFLOW_ERROR_MSG} {err}"
93
+ current_app.logger.error(error)
94
+ execution.update_state(EXEC_STATE_ERROR)
95
+ raise AirflowError(
96
+ error=error,
97
+ payload=dict(
98
+ message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
99
+ state=EXEC_STATE_ERROR,
100
+ ),
101
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
102
+ + error,
103
+ )
104
+
105
+ # Update execution on success
106
+ af_data = response.json()
107
+ execution.dag_run_id = af_data.get("dag_run_id")
108
+ execution.update_state(EXEC_STATE_QUEUED)
109
+
110
+
32
111
  class DataCheckExecutionEndpoint(BaseMetaResource):
33
112
  """
34
113
  Endpoint used to execute the instance and solution checks on an execution
35
114
  """
115
+
36
116
  ROLES_WITH_ACCESS = [PLANNER_ROLE, ADMIN_ROLE]
37
117
 
38
118
  def __init__(self):
@@ -56,13 +136,16 @@ class DataCheckExecutionEndpoint(BaseMetaResource):
56
136
  :rtype: Tuple(dict, integer)
57
137
  """
58
138
  config = current_app.config
139
+ user = self.get_user()
140
+ context_str = f"execution {idx}"
59
141
 
60
142
  execution = ExecutionModel.get_one_object(user=self.get_user(), idx=idx)
61
143
  if execution is None:
62
144
  err = "The execution to check does not exist"
63
145
  raise ObjectDoesNotExist(
64
146
  error=err,
65
- log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + err
147
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
148
+ + err,
66
149
  )
67
150
 
68
151
  schema = execution.schema
@@ -72,7 +155,8 @@ class DataCheckExecutionEndpoint(BaseMetaResource):
72
155
  err = "The execution is still running"
73
156
  raise InvalidUsage(
74
157
  error=err,
75
- log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + err
158
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
159
+ + err,
76
160
  )
77
161
 
78
162
  # this allows testing without airflow interaction:
@@ -80,58 +164,12 @@ class DataCheckExecutionEndpoint(BaseMetaResource):
80
164
  execution.update_state(EXEC_STATE_NOT_RUN)
81
165
  return execution, 201
82
166
 
83
- # We now try to launch the task in airflow
167
+ # Initialize Airflow client
84
168
  af_client = Airflow.from_config(config)
85
- if not af_client.is_alive():
86
- err = "Airflow is not accessible"
87
- current_app.logger.error(err)
88
- execution.update_state(EXEC_STATE_ERROR_START)
89
- raise AirflowError(
90
- error=err,
91
- payload=dict(
92
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
93
- state=EXEC_STATE_ERROR_START,
94
- ),
95
- log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + err
96
- )
97
- # ask airflow if dag_name exists
98
- schema_info = af_client.get_dag_info(schema)
99
-
100
- info = schema_info.json()
101
- if info["is_paused"]:
102
- err = "The dag exists but it is paused in airflow"
103
- current_app.logger.error(err)
104
- execution.update_state(EXEC_STATE_ERROR_START)
105
- raise AirflowError(
106
- error=err,
107
- payload=dict(
108
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
109
- state=EXEC_STATE_ERROR_START,
110
- ),
111
- log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + err
112
- )
113
169
 
114
- try:
115
- response = af_client.run_dag(
116
- execution.id, dag_name=schema, checks_only=True
117
- )
118
- except AirflowError as err:
119
- error = "Airflow responded with an error: {}".format(err)
120
- current_app.logger.error(error)
121
- execution.update_state(EXEC_STATE_ERROR)
122
- raise AirflowError(
123
- error=error,
124
- payload=dict(
125
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
126
- state=EXEC_STATE_ERROR,
127
- ),
128
- log_txt=f"Error while user {self.get_user()} tries to run data checks on execution {idx}. " + error
129
- )
170
+ # Call the shared Airflow execution logic
171
+ _run_airflow_data_check(af_client, execution, schema, user, context_str)
130
172
 
131
- # if we succeed, we register the dag_run_id in the execution table:
132
- af_data = response.json()
133
- execution.dag_run_id = af_data["dag_run_id"]
134
- execution.update_state(EXEC_STATE_QUEUED)
135
173
  current_app.logger.info(
136
174
  "User {} launches checks of execution {}".format(
137
175
  self.get_user_id(), execution.id
@@ -144,7 +182,9 @@ class DataCheckInstanceEndpoint(BaseMetaResource):
144
182
  """
145
183
  Endpoint used to execute the instance and solution checks on an execution
146
184
  """
185
+
147
186
  ROLES_WITH_ACCESS = [PLANNER_ROLE, ADMIN_ROLE]
187
+
148
188
  def __init__(self):
149
189
  super().__init__()
150
190
  self.model = ExecutionModel
@@ -169,13 +209,16 @@ class DataCheckInstanceEndpoint(BaseMetaResource):
169
209
  :rtype: Tuple(dict, integer)
170
210
  """
171
211
  config = current_app.config
212
+ user = self.get_user()
213
+ context_str = f"instance {idx}"
172
214
 
173
215
  instance = InstanceModel.get_one_object(user=self.get_user(), idx=idx)
174
216
  if instance is None:
175
217
  err = "The instance to check does not exist"
176
218
  raise ObjectDoesNotExist(
177
219
  error=err,
178
- log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + err
220
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
221
+ + err,
179
222
  )
180
223
  payload = dict(
181
224
  config=dict(checks_only=True),
@@ -185,67 +228,19 @@ class DataCheckInstanceEndpoint(BaseMetaResource):
185
228
  )
186
229
  schema = instance.schema
187
230
 
188
- execution, status_code = self.post_list(data=payload)
231
+ execution, _ = self.post_list(data=payload)
189
232
 
190
233
  # this allows testing without airflow interaction:
191
234
  if request.args.get("run", "1") == "0":
192
235
  execution.update_state(EXEC_STATE_NOT_RUN)
193
236
  return execution, 201
194
237
 
195
- # We now try to launch the task in airflow
238
+ # Initialize Airflow client
196
239
  af_client = Airflow.from_config(config)
197
- if not af_client.is_alive():
198
- err = "Airflow is not accessible"
199
- current_app.logger.error(err)
200
- execution.update_state(EXEC_STATE_ERROR_START)
201
- raise AirflowError(
202
- error=err,
203
- payload=dict(
204
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
205
- state=EXEC_STATE_ERROR_START,
206
- ),
207
- log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + err
208
240
 
209
- )
210
- # ask airflow if dag_name exists
211
- schema_info = af_client.get_dag_info(schema)
212
-
213
- info = schema_info.json()
214
- if info["is_paused"]:
215
- err = "The dag exists but it is paused in airflow"
216
- current_app.logger.error(err)
217
- execution.update_state(EXEC_STATE_ERROR_START)
218
- raise AirflowError(
219
- error=err,
220
- payload=dict(
221
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
222
- state=EXEC_STATE_ERROR_START,
223
- ),
224
- log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + err
225
-
226
- )
241
+ # Call the shared Airflow execution logic
242
+ _run_airflow_data_check(af_client, execution, schema, user, context_str)
227
243
 
228
- try:
229
- response = af_client.run_dag(
230
- execution.id, dag_name=schema, checks_only=True
231
- )
232
- except AirflowError as err:
233
- error = "Airflow responded with an error: {}".format(err)
234
- current_app.logger.error(error)
235
- execution.update_state(EXEC_STATE_ERROR)
236
- raise AirflowError(
237
- error=error,
238
- payload=dict(
239
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
240
- state=EXEC_STATE_ERROR,
241
- ),
242
- log_txt=f"Error while user {self.get_user()} tries to run data checks on instance {idx}. " + error
243
- )
244
-
245
- # if we succeed, we register the dag_run_id in the execution table:
246
- af_data = response.json()
247
- execution.dag_run_id = af_data["dag_run_id"]
248
- execution.update_state(EXEC_STATE_QUEUED)
249
244
  current_app.logger.info(
250
245
  "User {} creates instance check execution {}".format(
251
246
  self.get_user_id(), execution.id
@@ -258,7 +253,9 @@ class DataCheckCaseEndpoint(BaseMetaResource):
258
253
  """
259
254
  Endpoint used to execute the instance and solution checks on an execution
260
255
  """
256
+
261
257
  ROLES_WITH_ACCESS = [PLANNER_ROLE, ADMIN_ROLE]
258
+
262
259
  def __init__(self):
263
260
  super().__init__()
264
261
  self.model = ExecutionModel
@@ -283,13 +280,16 @@ class DataCheckCaseEndpoint(BaseMetaResource):
283
280
  :rtype: Tuple(dict, integer)
284
281
  """
285
282
  config = current_app.config
283
+ user = self.get_user()
284
+ context_str = f"case {idx}"
286
285
 
287
286
  case = CaseModel.get_one_object(user=self.get_user(), idx=idx)
288
287
  if case is None:
289
288
  err = "The case to check does not exist"
290
289
  raise ObjectDoesNotExist(
291
290
  error=err,
292
- log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + err
291
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
292
+ + err,
293
293
  )
294
294
 
295
295
  schema = case.schema or "solve_model_dag"
@@ -309,14 +309,18 @@ class DataCheckCaseEndpoint(BaseMetaResource):
309
309
  if schema == "pulp":
310
310
  validation_schema = "solve_model_dag"
311
311
 
312
- data_jsonschema = DeployedDAG.get_one_schema(config, validation_schema, INSTANCE_SCHEMA)
313
- validation_errors = json_schema_validate_as_string(data_jsonschema, instance_payload["data"])
312
+ data_jsonschema = DeployedDAG.get_one_schema(
313
+ config, validation_schema, INSTANCE_SCHEMA
314
+ )
315
+ validation_errors = json_schema_validate_as_string(
316
+ data_jsonschema, instance_payload["data"]
317
+ )
314
318
 
315
319
  if validation_errors:
316
320
  raise InvalidData(
317
321
  payload=dict(jsonschema_errors=validation_errors),
318
- log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. "
319
- f"Instance data does not match the jsonschema.",
322
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
323
+ f"Instance data does not match the jsonschema.",
320
324
  )
321
325
 
322
326
  instance, _ = self.post_list(data=instance_payload)
@@ -334,14 +338,18 @@ class DataCheckCaseEndpoint(BaseMetaResource):
334
338
 
335
339
  payload["data"] = case.solution
336
340
 
337
- data_jsonschema = DeployedDAG.get_one_schema(config, validation_schema, SOLUTION_SCHEMA)
338
- validation_errors = json_schema_validate_as_string(data_jsonschema, payload["data"])
341
+ data_jsonschema = DeployedDAG.get_one_schema(
342
+ config, validation_schema, SOLUTION_SCHEMA
343
+ )
344
+ validation_errors = json_schema_validate_as_string(
345
+ data_jsonschema, payload["data"]
346
+ )
339
347
 
340
348
  if validation_errors:
341
349
  raise InvalidData(
342
350
  payload=dict(jsonschema_errors=validation_errors),
343
- log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. "
344
- f"Solution data does not match the jsonschema.",
351
+ log_txt=f"Error while user {user} tries to run data checks on {context_str}. "
352
+ f"Solution data does not match the jsonschema.",
345
353
  )
346
354
 
347
355
  self.data_model = ExecutionModel
@@ -354,59 +362,14 @@ class DataCheckCaseEndpoint(BaseMetaResource):
354
362
  execution.update_state(EXEC_STATE_NOT_RUN)
355
363
  return execution, 201
356
364
 
357
- # We now try to launch the task in airflow
365
+ # Initialize Airflow client
358
366
  af_client = Airflow.from_config(config)
359
- if not af_client.is_alive():
360
- err = "Airflow is not accessible"
361
- current_app.logger.error(err)
362
- execution.update_state(EXEC_STATE_ERROR_START)
363
- raise AirflowError(
364
- error=err,
365
- payload=dict(
366
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
367
- state=EXEC_STATE_ERROR_START,
368
- ),
369
- log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + err
370
- )
371
- # ask airflow if dag_name exists
372
- schema_info = af_client.get_dag_info(schema)
373
-
374
- info = schema_info.json()
375
- if info["is_paused"]:
376
- err = "The dag exists but it is paused in airflow"
377
- current_app.logger.error(err)
378
- execution.update_state(EXEC_STATE_ERROR_START)
379
- raise AirflowError(
380
- error=err,
381
- payload=dict(
382
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
383
- state=EXEC_STATE_ERROR_START,
384
- ),
385
- log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + err
386
- )
387
-
388
- try:
389
- response = af_client.run_dag(
390
- execution.id, dag_name=schema, checks_only=True, case_id=idx
391
- )
392
367
 
393
- except AirflowError as err:
394
- error = "Airflow responded with an error: {}".format(err)
395
- current_app.logger.error(error)
396
- execution.update_state(EXEC_STATE_ERROR)
397
- raise AirflowError(
398
- error=error,
399
- payload=dict(
400
- message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
401
- state=EXEC_STATE_ERROR,
402
- ),
403
- log_txt=f"Error while user {self.get_user()} tries to run data checks on case {idx}. " + error
404
- )
368
+ # Call the shared Airflow execution logic, passing case_id
369
+ _run_airflow_data_check(
370
+ af_client, execution, schema, user, context_str, case_id=idx
371
+ )
405
372
 
406
- # if we succeed, we register the dag_run_id in the execution table:
407
- af_data = response.json()
408
- execution.dag_run_id = af_data["dag_run_id"]
409
- execution.update_state(EXEC_STATE_QUEUED)
410
373
  current_app.logger.info(
411
374
  "User {} creates case check execution {}".format(
412
375
  self.get_user_id(), execution.id
@@ -1,6 +1,7 @@
1
1
  """
2
2
  Endpoints to get the example data from a DAG
3
3
  """
4
+
4
5
  import json
5
6
 
6
7
  from cornflow_client.airflow.api import Airflow
@@ -11,7 +12,12 @@ from cornflow.endpoints.meta_resource import BaseMetaResource
11
12
  from cornflow.models import PermissionsDAG
12
13
  from cornflow.schemas.example_data import ExampleListData, ExampleDetailData
13
14
  from cornflow.shared.authentication import Auth, authenticate
14
- from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE
15
+ from cornflow.shared.const import (
16
+ AIRFLOW_NOT_REACHABLE_MSG,
17
+ VIEWER_ROLE,
18
+ PLANNER_ROLE,
19
+ ADMIN_ROLE,
20
+ )
15
21
  from cornflow.shared.exceptions import AirflowError, NoPermission, ObjectDoesNotExist
16
22
 
17
23
 
@@ -44,7 +50,7 @@ class ExampleDataListEndpoint(BaseMetaResource):
44
50
  current_app.logger.error(
45
51
  "Airflow not accessible when getting data {}".format(dag_name)
46
52
  )
47
- raise AirflowError(error="Airflow is not accessible")
53
+ raise AirflowError(error=f"{AIRFLOW_NOT_REACHABLE_MSG}")
48
54
 
49
55
  # try airflow and see if dag_name exists
50
56
  af_client.get_dag_info(dag_name)
@@ -87,7 +93,7 @@ class ExampleDataDetailEndpoint(BaseMetaResource):
87
93
  current_app.logger.error(
88
94
  "Airflow not accessible when getting data {}".format(dag_name)
89
95
  )
90
- raise AirflowError(error="Airflow is not accessible")
96
+ raise AirflowError(error=f"{AIRFLOW_NOT_REACHABLE_MSG}")
91
97
 
92
98
  # try airflow and see if dag_name exists
93
99
  af_client.get_dag_info(dag_name)
@@ -24,11 +24,14 @@ from cornflow.schemas.execution import (
24
24
  ExecutionEditRequest,
25
25
  QueryFiltersExecution,
26
26
  ReLaunchExecutionRequest,
27
- ExecutionDetailsWithIndicatorsAndLogResponse
27
+ ExecutionDetailsWithIndicatorsAndLogResponse,
28
28
  )
29
29
  from cornflow.shared.authentication import Auth, authenticate
30
30
  from cornflow.shared.compress import compressed
31
31
  from cornflow.shared.const import (
32
+ AIRFLOW_ERROR_MSG,
33
+ AIRFLOW_NOT_REACHABLE_MSG,
34
+ DAG_PAUSED_MSG,
32
35
  EXEC_STATE_RUNNING,
33
36
  EXEC_STATE_ERROR,
34
37
  EXEC_STATE_ERROR_START,
@@ -100,8 +103,8 @@ class ExecutionEndpoint(BaseMetaResource):
100
103
  af_client = Airflow.from_config(current_app.config)
101
104
  if not af_client.is_alive():
102
105
  current_app.logger.warning(
103
- "Error while the app tried to update the status of all running executions."
104
- "Airflow is not accessible."
106
+ f"Error while the app tried to update the status of all running executions."
107
+ f"{AIRFLOW_NOT_REACHABLE_MSG}"
105
108
  )
106
109
  continue
107
110
 
@@ -112,7 +115,7 @@ class ExecutionEndpoint(BaseMetaResource):
112
115
  except AirflowError as err:
113
116
  current_app.logger.warning(
114
117
  "Error while the app tried to update the status of all running executions."
115
- f"Airflow responded with an error: {err}"
118
+ f"{AIRFLOW_ERROR_MSG} {err}"
116
119
  )
117
120
  continue
118
121
 
@@ -137,18 +140,21 @@ class ExecutionEndpoint(BaseMetaResource):
137
140
  the reference_id for the newly created execution if successful) and a integer wit the HTTP status code
138
141
  :rtype: Tuple(dict, integer)
139
142
  """
140
- # TODO: should validation should be done even if the execution is not going to be run?
141
- # TODO: should the schema field be cross validated with the instance schema field?
143
+
142
144
  config = current_app.config
143
145
 
144
146
  if "schema" not in kwargs:
145
147
  kwargs["schema"] = "solve_model_dag"
146
148
 
147
- execution, status_code = self.post_list(data=kwargs)
149
+ execution, _ = self.post_list(data=kwargs)
148
150
  instance = InstanceModel.get_one_object(
149
151
  user=self.get_user(), idx=execution.instance_id
150
152
  )
151
153
 
154
+ if execution.schema != instance.schema:
155
+ execution.delete()
156
+ raise InvalidData(error="Instance and execution schema mismatch")
157
+
152
158
  current_app.logger.debug(f"The request is: {request.args.get('run')}")
153
159
  # this allows testing without airflow interaction:
154
160
  if request.args.get("run", "1") == "0":
@@ -161,17 +167,17 @@ class ExecutionEndpoint(BaseMetaResource):
161
167
  # We now try to launch the task in airflow
162
168
  af_client = Airflow.from_config(config)
163
169
  if not af_client.is_alive():
164
- err = "Airflow is not accessible"
165
- current_app.logger.error(err)
170
+
171
+ current_app.logger.error(AIRFLOW_NOT_REACHABLE_MSG)
166
172
  execution.update_state(EXEC_STATE_ERROR_START)
167
173
  raise AirflowError(
168
- error=err,
174
+ error=AIRFLOW_NOT_REACHABLE_MSG,
169
175
  payload=dict(
170
176
  message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
171
177
  state=EXEC_STATE_ERROR_START,
172
178
  ),
173
179
  log_txt=f"Error while user {self.get_user()} tries to create an execution "
174
- + err,
180
+ + AIRFLOW_NOT_REACHABLE_MSG,
175
181
  )
176
182
  # ask airflow if dag_name exists
177
183
  schema = execution.schema
@@ -231,23 +237,23 @@ class ExecutionEndpoint(BaseMetaResource):
231
237
 
232
238
  info = schema_info.json()
233
239
  if info["is_paused"]:
234
- err = "The dag exists but it is paused in airflow"
235
- current_app.logger.error(err)
240
+
241
+ current_app.logger.error(DAG_PAUSED_MSG)
236
242
  execution.update_state(EXEC_STATE_ERROR_START)
237
243
  raise AirflowError(
238
- error=err,
244
+ error=DAG_PAUSED_MSG,
239
245
  payload=dict(
240
246
  message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
241
247
  state=EXEC_STATE_ERROR_START,
242
248
  ),
243
249
  log_txt=f"Error while user {self.get_user()} tries to create an execution. "
244
- + err,
250
+ + DAG_PAUSED_MSG,
245
251
  )
246
252
 
247
253
  try:
248
254
  response = af_client.run_dag(execution.id, dag_name=schema)
249
255
  except AirflowError as err:
250
- error = "Airflow responded with an error: {}".format(err)
256
+ error = f"{AIRFLOW_ERROR_MSG} {err}"
251
257
  current_app.logger.error(error)
252
258
  execution.update_state(EXEC_STATE_ERROR)
253
259
  raise AirflowError(
@@ -339,17 +345,17 @@ class ExecutionRelaunchEndpoint(BaseMetaResource):
339
345
  # We now try to launch the task in airflow
340
346
  af_client = Airflow.from_config(config)
341
347
  if not af_client.is_alive():
342
- err = "Airflow is not accessible"
343
- current_app.logger.error(err)
348
+
349
+ current_app.logger.error(AIRFLOW_NOT_REACHABLE_MSG)
344
350
  execution.update_state(EXEC_STATE_ERROR_START)
345
351
  raise AirflowError(
346
- error=err,
352
+ error=AIRFLOW_NOT_REACHABLE_MSG,
347
353
  payload=dict(
348
354
  message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
349
355
  state=EXEC_STATE_ERROR_START,
350
356
  ),
351
357
  log_txt=f"Error while user {self.get_user()} tries to relaunch execution {idx}. "
352
- + err,
358
+ + AIRFLOW_NOT_REACHABLE_MSG,
353
359
  )
354
360
  # ask airflow if dag_name exists
355
361
  schema = execution.schema
@@ -357,23 +363,23 @@ class ExecutionRelaunchEndpoint(BaseMetaResource):
357
363
 
358
364
  info = schema_info.json()
359
365
  if info["is_paused"]:
360
- err = "The dag exists but it is paused in airflow"
361
- current_app.logger.error(err)
366
+
367
+ current_app.logger.error(DAG_PAUSED_MSG)
362
368
  execution.update_state(EXEC_STATE_ERROR_START)
363
369
  raise AirflowError(
364
- error=err,
370
+ error=DAG_PAUSED_MSG,
365
371
  payload=dict(
366
372
  message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR_START],
367
373
  state=EXEC_STATE_ERROR_START,
368
374
  ),
369
375
  log_txt=f"Error while user {self.get_user()} tries to relaunch execution {idx}. "
370
- + err,
376
+ + DAG_PAUSED_MSG,
371
377
  )
372
378
 
373
379
  try:
374
380
  response = af_client.run_dag(execution.id, dag_name=schema)
375
381
  except AirflowError as err:
376
- error = "Airflow responded with an error: {}".format(err)
382
+ error = f"{AIRFLOW_ERROR_MSG} {err}"
377
383
  current_app.logger.error(error)
378
384
  execution.update_state(EXEC_STATE_ERROR)
379
385
  raise AirflowError(
@@ -490,13 +496,13 @@ class ExecutionDetailsEndpoint(ExecutionDetailsEndpointBase):
490
496
  )
491
497
  af_client = Airflow.from_config(current_app.config)
492
498
  if not af_client.is_alive():
493
- err = "Airflow is not accessible"
499
+
494
500
  raise AirflowError(
495
- error=err,
501
+ error=AIRFLOW_NOT_REACHABLE_MSG,
496
502
  log_txt=f"Error while user {self.get_user()} tries to stop execution {idx}. "
497
- + err,
503
+ + AIRFLOW_NOT_REACHABLE_MSG,
498
504
  )
499
- response = af_client.set_dag_run_to_fail(
505
+ af_client.set_dag_run_to_fail(
500
506
  dag_name=execution.schema, dag_run_id=execution.dag_run_id
501
507
  )
502
508
  execution.update_state(EXEC_STATE_STOPPED)
@@ -563,21 +569,21 @@ class ExecutionStatusEndpoint(BaseMetaResource):
563
569
 
564
570
  af_client = Airflow.from_config(current_app.config)
565
571
  if not af_client.is_alive():
566
- err = "Airflow is not accessible"
572
+
567
573
  _raise_af_error(
568
574
  execution,
569
- err,
575
+ AIRFLOW_NOT_REACHABLE_MSG,
570
576
  log_txt=f"Error while user {self.get_user()} tries to get the status of execution {idx}. "
571
- + err,
577
+ + AIRFLOW_NOT_REACHABLE_MSG,
572
578
  )
573
579
 
574
580
  try:
575
- # TODO: get the dag_name from somewhere!
581
+
576
582
  response = af_client.get_dag_run_status(
577
583
  dag_name=execution.schema, dag_run_id=dag_run_id
578
584
  )
579
585
  except AirflowError as err:
580
- error = f"Airflow responded with an error: {err}"
586
+ error = f"{AIRFLOW_ERROR_MSG} {err}"
581
587
  _raise_af_error(
582
588
  execution,
583
589
  error,