cornflow 2.0.0a11__py3-none-any.whl → 2.0.0a13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow_config/airflow_local_settings.py +1 -1
- cornflow/app.py +8 -3
- cornflow/cli/migrations.py +23 -3
- cornflow/cli/service.py +18 -18
- cornflow/cli/utils.py +16 -1
- cornflow/commands/dag.py +1 -1
- cornflow/config.py +13 -8
- cornflow/endpoints/__init__.py +8 -2
- cornflow/endpoints/alarms.py +66 -2
- cornflow/endpoints/data_check.py +53 -26
- cornflow/endpoints/execution.py +387 -132
- cornflow/endpoints/login.py +81 -63
- cornflow/endpoints/meta_resource.py +11 -3
- cornflow/migrations/versions/999b98e24225.py +34 -0
- cornflow/models/base_data_model.py +4 -32
- cornflow/models/execution.py +2 -3
- cornflow/models/meta_models.py +28 -22
- cornflow/models/user.py +7 -10
- cornflow/schemas/alarms.py +8 -0
- cornflow/schemas/execution.py +2 -1
- cornflow/schemas/query.py +2 -1
- cornflow/schemas/user.py +5 -20
- cornflow/shared/authentication/auth.py +201 -264
- cornflow/shared/const.py +3 -14
- cornflow/shared/databricks.py +5 -1
- cornflow/tests/const.py +2 -0
- cornflow/tests/custom_test_case.py +77 -26
- cornflow/tests/unit/test_actions.py +2 -2
- cornflow/tests/unit/test_alarms.py +55 -1
- cornflow/tests/unit/test_apiview.py +108 -3
- cornflow/tests/unit/test_cases.py +20 -29
- cornflow/tests/unit/test_cli.py +6 -5
- cornflow/tests/unit/test_commands.py +3 -3
- cornflow/tests/unit/test_dags.py +5 -6
- cornflow/tests/unit/test_executions.py +491 -118
- cornflow/tests/unit/test_instances.py +14 -2
- cornflow/tests/unit/test_instances_file.py +1 -1
- cornflow/tests/unit/test_licenses.py +1 -1
- cornflow/tests/unit/test_log_in.py +230 -207
- cornflow/tests/unit/test_permissions.py +8 -8
- cornflow/tests/unit/test_roles.py +48 -10
- cornflow/tests/unit/test_schemas.py +1 -1
- cornflow/tests/unit/test_tables.py +7 -7
- cornflow/tests/unit/test_token.py +19 -5
- cornflow/tests/unit/test_users.py +22 -6
- cornflow/tests/unit/tools.py +75 -10
- {cornflow-2.0.0a11.dist-info → cornflow-2.0.0a13.dist-info}/METADATA +16 -15
- {cornflow-2.0.0a11.dist-info → cornflow-2.0.0a13.dist-info}/RECORD +51 -51
- {cornflow-2.0.0a11.dist-info → cornflow-2.0.0a13.dist-info}/WHEEL +1 -1
- cornflow/endpoints/execution_databricks.py +0 -808
- {cornflow-2.0.0a11.dist-info → cornflow-2.0.0a13.dist-info}/entry_points.txt +0 -0
- {cornflow-2.0.0a11.dist-info → cornflow-2.0.0a13.dist-info}/top_level.txt +0 -0
@@ -6,6 +6,8 @@ Unit test for the executions endpoints
|
|
6
6
|
import json
|
7
7
|
from unittest.mock import patch
|
8
8
|
|
9
|
+
from cornflow.app import create_app
|
10
|
+
|
9
11
|
# Import from internal modules
|
10
12
|
from cornflow.models import ExecutionModel, InstanceModel
|
11
13
|
from cornflow.tests.const import (
|
@@ -17,10 +19,10 @@ from cornflow.tests.const import (
|
|
17
19
|
INSTANCE_URL,
|
18
20
|
DAG_URL,
|
19
21
|
BAD_EXECUTION_PATH,
|
20
|
-
EXECUTION_SOLUTION_PATH,
|
22
|
+
EXECUTION_SOLUTION_PATH, EDIT_EXECUTION_SOLUTION,
|
21
23
|
)
|
22
24
|
from cornflow.tests.custom_test_case import CustomTestCase, BaseTestCases
|
23
|
-
from cornflow.tests.unit.tools import patch_af_client
|
25
|
+
from cornflow.tests.unit.tools import patch_af_client, patch_db_client
|
24
26
|
|
25
27
|
|
26
28
|
class TestExecutionsListEndpoint(BaseTestCases.ListFilters):
|
@@ -62,13 +64,13 @@ class TestExecutionsListEndpoint(BaseTestCases.ListFilters):
|
|
62
64
|
def test_new_execution(self):
|
63
65
|
self.create_new_row(self.url, self.model, payload=self.payload)
|
64
66
|
|
65
|
-
@patch("cornflow.endpoints.
|
67
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
66
68
|
def test_new_execution_run(self, af_client_class):
|
67
69
|
patch_af_client(af_client_class)
|
68
70
|
|
69
71
|
self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
70
72
|
|
71
|
-
@patch("cornflow.endpoints.
|
73
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
72
74
|
def test_new_execution_bad_config(self, af_client_class):
|
73
75
|
patch_af_client(af_client_class)
|
74
76
|
response = self.create_new_row(
|
@@ -81,7 +83,7 @@ class TestExecutionsListEndpoint(BaseTestCases.ListFilters):
|
|
81
83
|
self.assertIn("error", response)
|
82
84
|
self.assertIn("jsonschema_errors", response)
|
83
85
|
|
84
|
-
@patch("cornflow.endpoints.
|
86
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
85
87
|
def test_new_execution_partial_config(self, af_client_class):
|
86
88
|
patch_af_client(af_client_class)
|
87
89
|
self.payload["config"].pop("solver")
|
@@ -91,7 +93,7 @@ class TestExecutionsListEndpoint(BaseTestCases.ListFilters):
|
|
91
93
|
self.assertIn("solver", response["config"])
|
92
94
|
self.assertEqual(response["config"]["solver"], "cbc")
|
93
95
|
|
94
|
-
@patch("cornflow.endpoints.
|
96
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
95
97
|
def test_new_execution_with_solution(self, af_client_class):
|
96
98
|
patch_af_client(af_client_class)
|
97
99
|
self.payload["data"] = self.solution
|
@@ -102,7 +104,7 @@ class TestExecutionsListEndpoint(BaseTestCases.ListFilters):
|
|
102
104
|
check_payload=False,
|
103
105
|
)
|
104
106
|
|
105
|
-
@patch("cornflow.endpoints.
|
107
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
106
108
|
def test_new_execution_with_solution_bad(self, af_client_class):
|
107
109
|
patch_af_client(af_client_class)
|
108
110
|
patch_af_client(af_client_class)
|
@@ -144,6 +146,131 @@ class TestExecutionsListEndpoint(BaseTestCases.ListFilters):
|
|
144
146
|
self.assertEqual(len(rows.json), len(self.payloads))
|
145
147
|
|
146
148
|
|
149
|
+
class TestExecutionsListEndpointDatabricks(BaseTestCases.ListFilters):
|
150
|
+
def setUp(self):
|
151
|
+
super().setUp()
|
152
|
+
|
153
|
+
with open(INSTANCE_PATH) as f:
|
154
|
+
payload = json.load(f)
|
155
|
+
fk_id = self.create_new_row(INSTANCE_URL, InstanceModel, payload)
|
156
|
+
self.url = EXECUTION_URL_NORUN
|
157
|
+
self.model = ExecutionModel
|
158
|
+
|
159
|
+
def load_file_fk(_file):
|
160
|
+
with open(_file) as f:
|
161
|
+
temp = json.load(f)
|
162
|
+
temp["instance_id"] = fk_id
|
163
|
+
return temp
|
164
|
+
|
165
|
+
self.payload = load_file_fk(EXECUTION_PATH)
|
166
|
+
self.bad_payload = load_file_fk(BAD_EXECUTION_PATH)
|
167
|
+
self.payloads = [load_file_fk(f) for f in EXECUTIONS_LIST]
|
168
|
+
self.solution = load_file_fk(EXECUTION_SOLUTION_PATH)
|
169
|
+
self.keys_to_check = [
|
170
|
+
"data_hash",
|
171
|
+
"created_at",
|
172
|
+
"config",
|
173
|
+
"state",
|
174
|
+
"message",
|
175
|
+
"schema",
|
176
|
+
"description",
|
177
|
+
"id",
|
178
|
+
"user_id",
|
179
|
+
"log",
|
180
|
+
"instance_id",
|
181
|
+
"name",
|
182
|
+
"indicators",
|
183
|
+
]
|
184
|
+
|
185
|
+
def create_app(self):
|
186
|
+
app = create_app("testing-databricks")
|
187
|
+
return app
|
188
|
+
|
189
|
+
def test_new_execution(self):
|
190
|
+
self.create_new_row(self.url, self.model, payload=self.payload)
|
191
|
+
|
192
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
193
|
+
def test_new_execution_run_databricks(self, db_client_class):
|
194
|
+
patch_db_client(db_client_class)
|
195
|
+
|
196
|
+
self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
197
|
+
|
198
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
199
|
+
def test_new_execution_bad_config(self, db_client_class):
|
200
|
+
patch_db_client(db_client_class)
|
201
|
+
response = self.create_new_row(
|
202
|
+
EXECUTION_URL,
|
203
|
+
self.model,
|
204
|
+
payload=self.bad_payload,
|
205
|
+
expected_status=400,
|
206
|
+
check_payload=False,
|
207
|
+
)
|
208
|
+
self.assertIn("error", response)
|
209
|
+
self.assertIn("jsonschema_errors", response)
|
210
|
+
|
211
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
212
|
+
def test_new_execution_partial_config(self, db_client_class):
|
213
|
+
patch_db_client(db_client_class)
|
214
|
+
self.payload["config"].pop("solver")
|
215
|
+
response = self.create_new_row(
|
216
|
+
EXECUTION_URL, self.model, payload=self.payload, check_payload=False
|
217
|
+
)
|
218
|
+
self.assertIn("solver", response["config"])
|
219
|
+
self.assertEqual(response["config"]["solver"], "cbc")
|
220
|
+
|
221
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
222
|
+
def test_new_execution_with_solution(self, db_client_class):
|
223
|
+
patch_db_client(db_client_class)
|
224
|
+
self.payload["data"] = self.solution
|
225
|
+
response = self.create_new_row(
|
226
|
+
EXECUTION_URL,
|
227
|
+
self.model,
|
228
|
+
payload=self.payload,
|
229
|
+
check_payload=False,
|
230
|
+
)
|
231
|
+
|
232
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
233
|
+
def test_new_execution_with_solution_bad(self, db_client_class):
|
234
|
+
patch_db_client(db_client_class)
|
235
|
+
patch_db_client(db_client_class)
|
236
|
+
self.payload["data"] = {"message": "THIS IS NOT A VALID SOLUTION"}
|
237
|
+
response = self.create_new_row(
|
238
|
+
EXECUTION_URL,
|
239
|
+
self.model,
|
240
|
+
payload=self.payload,
|
241
|
+
check_payload=False,
|
242
|
+
expected_status=400,
|
243
|
+
)
|
244
|
+
self.assertIn("error", response)
|
245
|
+
self.assertIn("jsonschema_errors", response)
|
246
|
+
|
247
|
+
def test_new_execution_no_instance(self):
|
248
|
+
payload = dict(self.payload)
|
249
|
+
payload["instance_id"] = "bad_id"
|
250
|
+
response = self.client.post(
|
251
|
+
self.url,
|
252
|
+
data=json.dumps(payload),
|
253
|
+
follow_redirects=True,
|
254
|
+
headers=self.get_header_with_auth(self.token),
|
255
|
+
)
|
256
|
+
self.assertEqual(404, response.status_code)
|
257
|
+
self.assertTrue("error" in response.json)
|
258
|
+
|
259
|
+
def test_get_executions(self):
|
260
|
+
self.get_rows(self.url, self.payloads, keys_to_check=self.keys_to_check)
|
261
|
+
|
262
|
+
def test_get_no_executions(self):
|
263
|
+
self.get_no_rows(self.url)
|
264
|
+
|
265
|
+
def test_get_executions_superadmin(self):
|
266
|
+
self.get_rows(self.url, self.payloads, keys_to_check=self.keys_to_check)
|
267
|
+
token = self.create_service_user()
|
268
|
+
rows = self.client.get(
|
269
|
+
self.url, follow_redirects=True, headers=self.get_header_with_auth(token)
|
270
|
+
)
|
271
|
+
self.assertEqual(len(rows.json), len(self.payloads))
|
272
|
+
|
273
|
+
|
147
274
|
class TestExecutionRelaunchEndpoint(CustomTestCase):
|
148
275
|
def setUp(self):
|
149
276
|
super().setUp()
|
@@ -193,7 +320,7 @@ class TestExecutionRelaunchEndpoint(CustomTestCase):
|
|
193
320
|
self.assertEqual(row["config"], self.payload["config"])
|
194
321
|
self.assertIsNone(row["checks"])
|
195
322
|
|
196
|
-
@patch("cornflow.endpoints.
|
323
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
197
324
|
def test_relaunch_execution_run(self, af_client_class):
|
198
325
|
patch_af_client(af_client_class)
|
199
326
|
|
@@ -240,12 +367,113 @@ class TestExecutionRelaunchEndpoint(CustomTestCase):
|
|
240
367
|
self.assertEqual(404, response.status_code)
|
241
368
|
|
242
369
|
|
370
|
+
class TestExecutionRelaunchEndpointDatabricks(CustomTestCase):
|
371
|
+
def setUp(self):
|
372
|
+
super().setUp()
|
373
|
+
|
374
|
+
with open(INSTANCE_PATH) as f:
|
375
|
+
payload = json.load(f)
|
376
|
+
fk_id = self.create_new_row(INSTANCE_URL, InstanceModel, payload)
|
377
|
+
self.url = EXECUTION_URL_NORUN
|
378
|
+
self.model = ExecutionModel
|
379
|
+
|
380
|
+
def load_file_fk(_file):
|
381
|
+
with open(_file) as f:
|
382
|
+
temp = json.load(f)
|
383
|
+
temp["instance_id"] = fk_id
|
384
|
+
return temp
|
385
|
+
|
386
|
+
self.payload = load_file_fk(EXECUTION_PATH)
|
387
|
+
|
388
|
+
def create_app(self):
|
389
|
+
app = create_app("testing-databricks")
|
390
|
+
return app
|
391
|
+
|
392
|
+
def test_relaunch_execution(self):
|
393
|
+
idx = self.create_new_row(self.url, self.model, payload=self.payload)
|
394
|
+
|
395
|
+
# Add solution checks to see if they are deleted correctly
|
396
|
+
token = self.create_service_user()
|
397
|
+
self.update_row(
|
398
|
+
url=DAG_URL + idx + "/",
|
399
|
+
payload_to_check=dict(),
|
400
|
+
change=dict(solution_schema="_data_checks", checks=dict(check_1=[])),
|
401
|
+
token=token,
|
402
|
+
check_payload=False,
|
403
|
+
)
|
404
|
+
|
405
|
+
url = EXECUTION_URL + idx + "/relaunch/?run=0"
|
406
|
+
self.payload["config"]["warmStart"] = False
|
407
|
+
response = self.client.post(
|
408
|
+
url,
|
409
|
+
data=json.dumps({"config": self.payload["config"]}),
|
410
|
+
follow_redirects=True,
|
411
|
+
headers=self.get_header_with_auth(self.token),
|
412
|
+
)
|
413
|
+
self.assertEqual(201, response.status_code)
|
414
|
+
|
415
|
+
url = EXECUTION_URL + idx + "/data"
|
416
|
+
row = self.client.get(
|
417
|
+
url, follow_redirects=True, headers=self.get_header_with_auth(self.token)
|
418
|
+
).json
|
419
|
+
|
420
|
+
self.assertEqual(row["config"], self.payload["config"])
|
421
|
+
self.assertIsNone(row["checks"])
|
422
|
+
|
423
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
424
|
+
def test_relaunch_execution_run(self, db_client_class):
|
425
|
+
patch_db_client(db_client_class)
|
426
|
+
|
427
|
+
idx = self.create_new_row(self.url, self.model, payload=self.payload)
|
428
|
+
|
429
|
+
# Add solution checks to see if they are deleted correctly
|
430
|
+
token = self.create_service_user()
|
431
|
+
self.update_row(
|
432
|
+
url=DAG_URL + idx + "/",
|
433
|
+
payload_to_check=dict(),
|
434
|
+
change=dict(solution_schema="_data_checks", checks=dict(check_1=[])),
|
435
|
+
token=token,
|
436
|
+
check_payload=False,
|
437
|
+
)
|
438
|
+
|
439
|
+
url = EXECUTION_URL + idx + "/relaunch/"
|
440
|
+
self.payload["config"]["warmStart"] = False
|
441
|
+
response = self.client.post(
|
442
|
+
url,
|
443
|
+
data=json.dumps({"config": self.payload["config"]}),
|
444
|
+
follow_redirects=True,
|
445
|
+
headers=self.get_header_with_auth(self.token),
|
446
|
+
)
|
447
|
+
self.assertEqual(201, response.status_code)
|
448
|
+
|
449
|
+
url = EXECUTION_URL + idx + "/data"
|
450
|
+
row = self.client.get(
|
451
|
+
url, follow_redirects=True, headers=self.get_header_with_auth(self.token)
|
452
|
+
).json
|
453
|
+
|
454
|
+
self.assertEqual(row["config"], self.payload["config"])
|
455
|
+
self.assertIsNone(row["checks"])
|
456
|
+
|
457
|
+
def test_relaunch_invalid_execution(self):
|
458
|
+
idx = "thisIsAnInvalidExecutionId"
|
459
|
+
url = EXECUTION_URL + idx + "/relaunch/?run=0"
|
460
|
+
self.payload["config"]["warmStart"] = False
|
461
|
+
response = self.client.post(
|
462
|
+
url,
|
463
|
+
data=json.dumps({"config": self.payload["config"]}),
|
464
|
+
follow_redirects=True,
|
465
|
+
headers=self.get_header_with_auth(self.token),
|
466
|
+
)
|
467
|
+
self.assertEqual(404, response.status_code)
|
468
|
+
|
469
|
+
|
243
470
|
class TestExecutionsDetailEndpointMock(CustomTestCase):
|
244
471
|
def setUp(self):
|
245
472
|
super().setUp()
|
246
473
|
with open(INSTANCE_PATH) as f:
|
247
474
|
payload = json.load(f)
|
248
475
|
fk_id = self.create_new_row(INSTANCE_URL, InstanceModel, payload)
|
476
|
+
self.instance_payload = payload
|
249
477
|
self.model = ExecutionModel
|
250
478
|
self.response_items = {
|
251
479
|
"id",
|
@@ -269,7 +497,7 @@ class TestExecutionsDetailEndpointMock(CustomTestCase):
|
|
269
497
|
self.payload["instance_id"] = fk_id
|
270
498
|
|
271
499
|
|
272
|
-
class
|
500
|
+
class TestExecutionsDetailEndpointAirflow(
|
273
501
|
TestExecutionsDetailEndpointMock, BaseTestCases.DetailEndpoint
|
274
502
|
):
|
275
503
|
def setUp(self):
|
@@ -277,115 +505,193 @@ class TestExecutionsDetailEndpoint(
|
|
277
505
|
self.url = self.url
|
278
506
|
self.query_arguments = {"run": 0}
|
279
507
|
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
508
|
+
def create_app(self):
|
509
|
+
app = create_app("testing") # configuración para Airflow
|
510
|
+
return app
|
511
|
+
|
512
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
513
|
+
def test_stop_execution(self, af_client_class):
|
514
|
+
patch_af_client(af_client_class)
|
515
|
+
|
516
|
+
idx = self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
517
|
+
|
518
|
+
response = self.client.post(
|
519
|
+
self.url + str(idx) + "/",
|
520
|
+
follow_redirects=True,
|
521
|
+
headers=self.get_header_with_auth(self.token),
|
289
522
|
)
|
290
523
|
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
"config",
|
299
|
-
"instance_id",
|
300
|
-
"user_id",
|
301
|
-
"indicators",
|
302
|
-
"description",
|
303
|
-
"name",
|
304
|
-
"created_at",
|
305
|
-
"state",
|
306
|
-
]
|
307
|
-
execution = self.get_one_row(
|
308
|
-
self.url + idx,
|
309
|
-
payload={**self.payload, **dict(id=idx)},
|
310
|
-
keys_to_check=keys_to_check,
|
524
|
+
self.assertEqual(200, response.status_code)
|
525
|
+
self.assertEqual(response.json["message"], "The execution has been stopped")
|
526
|
+
|
527
|
+
def test_edit_execution(self):
|
528
|
+
|
529
|
+
id_new_instance = self.create_new_row(
|
530
|
+
INSTANCE_URL, InstanceModel, self.instance_payload
|
311
531
|
)
|
312
|
-
self.
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
"
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
532
|
+
idx = self.create_new_row(
|
533
|
+
self.url_with_query_arguments(), self.model, self.payload
|
534
|
+
)
|
535
|
+
|
536
|
+
# Extract the data from data/edit_execution_solution.json
|
537
|
+
with open(EDIT_EXECUTION_SOLUTION) as f:
|
538
|
+
data = json.load(f)
|
539
|
+
|
540
|
+
data = {
|
541
|
+
"name": "new_name",
|
542
|
+
"description": "Updated description",
|
543
|
+
"data": data,
|
544
|
+
"instance_id": id_new_instance,
|
545
|
+
}
|
546
|
+
payload_to_check = {
|
547
|
+
"id": idx,
|
548
|
+
"name":"new_name",
|
549
|
+
"description":"Updated description",
|
550
|
+
"data_hash":"74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b",
|
551
|
+
"instance_id":"805bad3280c95e45384dc6bd91a41317f9a7858c",
|
552
|
+
}
|
553
|
+
self.update_row(
|
554
|
+
self.url + str(idx) + "/",
|
555
|
+
data,
|
556
|
+
payload_to_check,
|
557
|
+
)
|
558
|
+
|
559
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
560
|
+
def test_get_one_status(self, af_client_class):
|
561
|
+
patch_af_client(af_client_class)
|
562
|
+
idx = self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
563
|
+
payload = dict(self.payload)
|
564
|
+
payload["id"] = idx
|
565
|
+
keys_to_check = ["state", "message", "id", "data_hash"]
|
566
|
+
data = self.get_one_row(
|
567
|
+
EXECUTION_URL + idx + "/status/",
|
568
|
+
payload,
|
327
569
|
check_payload=False,
|
328
570
|
keys_to_check=keys_to_check,
|
329
571
|
)
|
330
|
-
|
331
|
-
self.assertFalse(idx in executions)
|
572
|
+
self.assertEqual(data["state"], 1)
|
332
573
|
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
payload =
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
self.get_one_row(
|
346
|
-
self.url + idx, payload={}, expected_status=404, check_payload=False
|
574
|
+
@patch("cornflow.endpoints.execution.Airflow")
|
575
|
+
def test_put_one_status(self, af_client_class):
|
576
|
+
patch_af_client(af_client_class)
|
577
|
+
|
578
|
+
idx = self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
579
|
+
payload = dict(self.payload)
|
580
|
+
payload["id"] = idx
|
581
|
+
response = self.client.put(
|
582
|
+
EXECUTION_URL + idx + "/status/",
|
583
|
+
data=json.dumps({"status": 0}),
|
584
|
+
follow_redirects=True,
|
585
|
+
headers=self.get_header_with_auth(self.token),
|
347
586
|
)
|
348
587
|
|
349
|
-
|
588
|
+
self.assertEqual(200, response.status_code)
|
589
|
+
self.assertEqual(f"execution {idx} updated correctly", response.json["message"])
|
590
|
+
|
591
|
+
class TestExecutionsDetailEndpointDatabricks(
|
592
|
+
TestExecutionsDetailEndpointMock, BaseTestCases.DetailEndpoint
|
593
|
+
):
|
594
|
+
def setUp(self):
|
595
|
+
super().setUp()
|
596
|
+
self.url = self.url
|
597
|
+
self.query_arguments = {"run": 0}
|
598
|
+
|
599
|
+
def create_app(self):
|
600
|
+
app = create_app("testing-databricks")
|
601
|
+
return app
|
602
|
+
|
603
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
604
|
+
def test_stop_execution(self, db_client_class):
|
605
|
+
patch_db_client(db_client_class)
|
606
|
+
|
607
|
+
idx = self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
608
|
+
|
609
|
+
response = self.client.post(
|
610
|
+
self.url + str(idx) + "/",
|
611
|
+
follow_redirects=True,
|
612
|
+
headers=self.get_header_with_auth(self.token),
|
613
|
+
)
|
614
|
+
|
615
|
+
self.assertEqual(200, response.status_code)
|
616
|
+
self.assertEqual(
|
617
|
+
response.json["message"], "This feature is not available for Databricks"
|
618
|
+
)
|
619
|
+
def test_edit_execution(self):
|
620
|
+
|
621
|
+
id_new_instance = self.create_new_row(
|
622
|
+
INSTANCE_URL, InstanceModel, self.instance_payload
|
623
|
+
)
|
350
624
|
idx = self.create_new_row(
|
351
625
|
self.url_with_query_arguments(), self.model, self.payload
|
352
626
|
)
|
353
|
-
with open(INSTANCE_PATH) as f:
|
354
|
-
payload = json.load(f)
|
355
|
-
payload["data"]["parameters"]["name"] = "NewName"
|
356
627
|
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
628
|
+
# Extract the data from data/edit_execution_solution.json
|
629
|
+
with open(EDIT_EXECUTION_SOLUTION) as f:
|
630
|
+
data = json.load(f)
|
631
|
+
|
632
|
+
data = {
|
633
|
+
"name": "new_name",
|
634
|
+
"description": "Updated description",
|
635
|
+
"data": data,
|
636
|
+
"instance_id": id_new_instance,
|
361
637
|
}
|
638
|
+
payload_to_check = {
|
639
|
+
"id": idx,
|
640
|
+
"name":"new_name",
|
641
|
+
"description":"Updated description",
|
642
|
+
"data_hash":"74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b",
|
643
|
+
"instance_id":"805bad3280c95e45384dc6bd91a41317f9a7858c",
|
644
|
+
}
|
362
645
|
self.update_row(
|
363
|
-
url,
|
364
|
-
|
365
|
-
|
646
|
+
self.url + str(idx) + "/",
|
647
|
+
data,
|
648
|
+
payload_to_check,
|
366
649
|
)
|
367
650
|
|
368
|
-
url += "data/"
|
369
|
-
row = self.client.get(
|
370
|
-
url, follow_redirects=True, headers=self.get_header_with_auth(self.token)
|
371
|
-
)
|
372
651
|
|
373
|
-
self.assertEqual(row.json["checks"], None)
|
374
652
|
|
375
|
-
@patch("cornflow.endpoints.execution_databricks.Airflow")
|
376
|
-
def test_stop_execution(self, af_client_class):
|
377
|
-
patch_af_client(af_client_class)
|
378
653
|
|
654
|
+
class TestExecutionsStatusEndpointDatabricks(TestExecutionsDetailEndpointMock):
|
655
|
+
def setUp(self):
|
656
|
+
super().setUp()
|
657
|
+
self.response_items = {"id", "name", "status"}
|
658
|
+
self.items_to_check = []
|
659
|
+
|
660
|
+
def create_app(self):
|
661
|
+
app = create_app("testing-databricks")
|
662
|
+
return app
|
663
|
+
|
664
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
665
|
+
def test_get_one_status(self, db_client_class):
|
666
|
+
patch_db_client(db_client_class)
|
379
667
|
idx = self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
668
|
+
payload = dict(self.payload)
|
669
|
+
payload["id"] = idx
|
670
|
+
keys_to_check = ["state", "message", "id", "data_hash"]
|
671
|
+
data = self.get_one_row(
|
672
|
+
EXECUTION_URL + idx + "/status/",
|
673
|
+
payload,
|
674
|
+
check_payload=False,
|
675
|
+
keys_to_check=keys_to_check,
|
676
|
+
)
|
677
|
+
self.assertEqual(data["state"], -1)
|
380
678
|
|
381
|
-
|
382
|
-
|
679
|
+
@patch("cornflow.endpoints.execution.Databricks")
|
680
|
+
def test_put_one_status(self, db_client_class):
|
681
|
+
patch_db_client(db_client_class)
|
682
|
+
|
683
|
+
idx = self.create_new_row(EXECUTION_URL, self.model, payload=self.payload)
|
684
|
+
payload = dict(self.payload)
|
685
|
+
payload["id"] = idx
|
686
|
+
response = self.client.put(
|
687
|
+
EXECUTION_URL + idx + "/status/",
|
688
|
+
data=json.dumps({"status": 0}),
|
383
689
|
follow_redirects=True,
|
384
690
|
headers=self.get_header_with_auth(self.token),
|
385
691
|
)
|
386
692
|
|
387
693
|
self.assertEqual(200, response.status_code)
|
388
|
-
self.assertEqual(response.json["message"]
|
694
|
+
self.assertEqual(f"execution {idx} updated correctly", response.json["message"])
|
389
695
|
|
390
696
|
|
391
697
|
class TestExecutionsDataEndpoint(TestExecutionsDetailEndpointMock):
|
@@ -430,6 +736,52 @@ class TestExecutionsDataEndpoint(TestExecutionsDetailEndpointMock):
|
|
430
736
|
)
|
431
737
|
|
432
738
|
|
739
|
+
class TestExecutionsDataEndpointDatabricks(TestExecutionsDetailEndpointMock):
|
740
|
+
def setUp(self):
|
741
|
+
super().setUp()
|
742
|
+
self.response_items = {"id", "name", "data"}
|
743
|
+
self.items_to_check = ["name"]
|
744
|
+
self.keys_to_check = [
|
745
|
+
"created_at",
|
746
|
+
"checks",
|
747
|
+
"instance_id",
|
748
|
+
"schema",
|
749
|
+
"data",
|
750
|
+
"user_id",
|
751
|
+
"message",
|
752
|
+
"data_hash",
|
753
|
+
"log",
|
754
|
+
"config",
|
755
|
+
"description",
|
756
|
+
"state",
|
757
|
+
"name",
|
758
|
+
"id",
|
759
|
+
]
|
760
|
+
|
761
|
+
def create_app(self):
|
762
|
+
app = create_app("testing-databricks")
|
763
|
+
return app
|
764
|
+
|
765
|
+
def test_get_one_execution(self):
|
766
|
+
idx = self.create_new_row(EXECUTION_URL_NORUN, self.model, self.payload)
|
767
|
+
self.url = EXECUTION_URL + idx + "/data/"
|
768
|
+
payload = dict(self.payload)
|
769
|
+
payload["id"] = idx
|
770
|
+
self.get_one_row(self.url, payload, keys_to_check=self.keys_to_check)
|
771
|
+
|
772
|
+
def test_get_one_execution_superadmin(self):
|
773
|
+
idx = self.create_new_row(EXECUTION_URL_NORUN, self.model, self.payload)
|
774
|
+
payload = dict(self.payload)
|
775
|
+
payload["id"] = idx
|
776
|
+
token = self.create_service_user()
|
777
|
+
self.get_one_row(
|
778
|
+
EXECUTION_URL + idx + "/data/",
|
779
|
+
payload,
|
780
|
+
token=token,
|
781
|
+
keys_to_check=self.keys_to_check,
|
782
|
+
)
|
783
|
+
|
784
|
+
|
433
785
|
class TestExecutionsLogEndpoint(TestExecutionsDetailEndpointMock):
|
434
786
|
def setUp(self):
|
435
787
|
super().setUp()
|
@@ -473,45 +825,52 @@ class TestExecutionsLogEndpoint(TestExecutionsDetailEndpointMock):
|
|
473
825
|
)
|
474
826
|
|
475
827
|
|
476
|
-
class
|
828
|
+
class TestExecutionsLogEndpointDatabricks(TestExecutionsDetailEndpointMock):
|
477
829
|
def setUp(self):
|
478
830
|
super().setUp()
|
479
|
-
self.response_items = {"id", "name", "
|
480
|
-
self.items_to_check = []
|
831
|
+
self.response_items = {"id", "name", "log", "indicators"}
|
832
|
+
self.items_to_check = ["name"]
|
833
|
+
self.keys_to_check = [
|
834
|
+
"created_at",
|
835
|
+
"id",
|
836
|
+
"log_text",
|
837
|
+
"instance_id",
|
838
|
+
"state",
|
839
|
+
"message",
|
840
|
+
"description",
|
841
|
+
"data_hash",
|
842
|
+
"name",
|
843
|
+
"log",
|
844
|
+
"schema",
|
845
|
+
"user_id",
|
846
|
+
"config",
|
847
|
+
"indicators",
|
848
|
+
]
|
481
849
|
|
482
|
-
|
483
|
-
|
484
|
-
|
850
|
+
def create_app(self):
|
851
|
+
app = create_app("testing-databricks")
|
852
|
+
return app
|
485
853
|
|
486
|
-
|
854
|
+
def test_get_one_execution(self):
|
855
|
+
idx = self.create_new_row(EXECUTION_URL_NORUN, self.model, self.payload)
|
487
856
|
payload = dict(self.payload)
|
488
857
|
payload["id"] = idx
|
489
|
-
|
490
|
-
|
491
|
-
EXECUTION_URL + idx + "/status/",
|
492
|
-
payload,
|
493
|
-
check_payload=False,
|
494
|
-
keys_to_check=keys_to_check,
|
858
|
+
self.get_one_row(
|
859
|
+
EXECUTION_URL + idx + "/log/", payload, keys_to_check=self.keys_to_check
|
495
860
|
)
|
496
|
-
self.assertEqual(data["state"], 1)
|
497
861
|
|
498
|
-
|
499
|
-
|
500
|
-
patch_af_client(af_client_class)
|
501
|
-
|
502
|
-
idx = self.create_new_row(EXECUTION_URL, self.model, self.payload)
|
862
|
+
def test_get_one_execution_superadmin(self):
|
863
|
+
idx = self.create_new_row(EXECUTION_URL_NORUN, self.model, self.payload)
|
503
864
|
payload = dict(self.payload)
|
504
865
|
payload["id"] = idx
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
866
|
+
token = self.create_service_user()
|
867
|
+
self.get_one_row(
|
868
|
+
EXECUTION_URL + idx + "/log/",
|
869
|
+
payload,
|
870
|
+
token=token,
|
871
|
+
keys_to_check=self.keys_to_check,
|
510
872
|
)
|
511
873
|
|
512
|
-
self.assertEqual(200, response.status_code)
|
513
|
-
self.assertEqual(f"execution {idx} updated correctly", response.json["message"])
|
514
|
-
|
515
874
|
|
516
875
|
class TestExecutionsModel(TestExecutionsDetailEndpointMock):
|
517
876
|
def test_repr_method(self):
|
@@ -521,3 +880,17 @@ class TestExecutionsModel(TestExecutionsDetailEndpointMock):
|
|
521
880
|
def test_str_method(self):
|
522
881
|
idx = self.create_new_row(self.url + "?run=0", self.model, self.payload)
|
523
882
|
self.str_method(idx, f"<Execution {idx}>")
|
883
|
+
|
884
|
+
|
885
|
+
class TestExecutionsModelDatabricks(TestExecutionsDetailEndpointMock):
|
886
|
+
def create_app(self):
|
887
|
+
app = create_app("testing-databricks")
|
888
|
+
return app
|
889
|
+
|
890
|
+
def test_repr_method(self):
|
891
|
+
idx = self.create_new_row(self.url + "?run=0", self.model, self.payload)
|
892
|
+
self.repr_method(idx, f"<Execution {idx}>")
|
893
|
+
|
894
|
+
def test_str_method(self):
|
895
|
+
idx = self.create_new_row(self.url + "?run=0", self.model, self.payload)
|
896
|
+
self.str_method(idx, f"<Execution {idx}>")
|