cornflow 1.1.1a1__py3-none-any.whl → 1.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cornflow/app.py +0 -4
- cornflow/cli/utils.py +1 -1
- cornflow/config.py +2 -10
- cornflow/endpoints/__init__.py +0 -14
- cornflow/endpoints/execution.py +1 -1
- cornflow/endpoints/login.py +1 -8
- cornflow/models/__init__.py +0 -2
- cornflow/models/execution.py +0 -8
- cornflow/models/meta_models.py +12 -23
- cornflow/schemas/execution.py +0 -3
- cornflow/shared/const.py +0 -21
- cornflow/shared/exceptions.py +9 -20
- cornflow/tests/const.py +0 -7
- cornflow/tests/{custom_live_server.py → custom_liveServer.py} +1 -3
- cornflow/tests/custom_test_case.py +3 -2
- cornflow/tests/integration/test_commands.py +1 -1
- cornflow/tests/integration/test_cornflowclient.py +28 -116
- cornflow/tests/unit/test_alarms.py +9 -22
- cornflow/tests/unit/test_cli.py +5 -10
- cornflow/tests/unit/test_commands.py +2 -6
- cornflow/tests/unit/test_executions.py +0 -5
- cornflow/tests/unit/test_main_alarms.py +0 -8
- cornflow/tests/unit/test_users.py +2 -5
- {cornflow-1.1.1a1.dist-info → cornflow-1.1.4.dist-info}/METADATA +7 -7
- {cornflow-1.1.1a1.dist-info → cornflow-1.1.4.dist-info}/RECORD +28 -35
- {cornflow-1.1.1a1.dist-info → cornflow-1.1.4.dist-info}/WHEEL +1 -1
- cornflow/endpoints/reports.py +0 -283
- cornflow/migrations/versions/83164be03c23_.py +0 -40
- cornflow/migrations/versions/96f00d0961d1_reports_table.py +0 -50
- cornflow/models/reports.py +0 -119
- cornflow/schemas/reports.py +0 -48
- cornflow/static/v1.json +0 -3854
- cornflow/tests/unit/test_reports.py +0 -308
- {cornflow-1.1.1a1.dist-info → cornflow-1.1.4.dist-info}/entry_points.txt +0 -0
- {cornflow-1.1.1a1.dist-info → cornflow-1.1.4.dist-info}/top_level.txt +0 -0
cornflow/app.py
CHANGED
@@ -46,9 +46,6 @@ def create_app(env_name="development", dataconn=None):
|
|
46
46
|
:return: the application that is going to be running :class:`Flask`
|
47
47
|
:rtype: :class:`Flask`
|
48
48
|
"""
|
49
|
-
if os.getenv("FLASK_ENV", None) is not None:
|
50
|
-
env_name = os.getenv("FLASK_ENV")
|
51
|
-
|
52
49
|
dictConfig(log_config(app_config[env_name].LOG_LEVEL))
|
53
50
|
|
54
51
|
app = Flask(__name__)
|
@@ -77,7 +74,6 @@ def create_app(env_name="development", dataconn=None):
|
|
77
74
|
api = Api(app)
|
78
75
|
for res in resources:
|
79
76
|
api.add_resource(res["resource"], res["urls"], endpoint=res["endpoint"])
|
80
|
-
|
81
77
|
if app.config["ALARMS_ENDPOINTS"]:
|
82
78
|
for res in alarms_resources:
|
83
79
|
api.add_resource(res["resource"], res["urls"], endpoint=res["endpoint"])
|
cornflow/cli/utils.py
CHANGED
@@ -6,7 +6,7 @@ import warnings
|
|
6
6
|
|
7
7
|
def get_app():
|
8
8
|
env = os.getenv("FLASK_ENV", "development")
|
9
|
-
data_conn = os.getenv("DATABASE_URL")
|
9
|
+
data_conn = os.getenv("DATABASE_URL", "sqlite:///cornflow.db")
|
10
10
|
if env == "production":
|
11
11
|
warnings.filterwarnings("ignore")
|
12
12
|
external = int(os.getenv("EXTERNAL_APP", 0))
|
cornflow/config.py
CHANGED
@@ -22,14 +22,6 @@ class DefaultConfig(object):
|
|
22
22
|
SIGNUP_ACTIVATED = int(os.getenv("SIGNUP_ACTIVATED", 1))
|
23
23
|
CORNFLOW_SERVICE_USER = os.getenv("CORNFLOW_SERVICE_USER", "service_user")
|
24
24
|
|
25
|
-
# file support for reports
|
26
|
-
FILE_BACKEND = os.getenv("FILE_BACKEND", "local")
|
27
|
-
UPLOAD_FOLDER = os.getenv(
|
28
|
-
"UPLOAD_FOLDER",
|
29
|
-
os.path.abspath(os.path.join(os.path.dirname(__file__), "./static")),
|
30
|
-
)
|
31
|
-
ALLOWED_EXTENSIONS = os.getenv("ALLOWED_EXTENSIONS", ["pdf", "html"])
|
32
|
-
|
33
25
|
# Open deployment (all dags accessible to all users)
|
34
26
|
OPEN_DEPLOYMENT = os.getenv("OPEN_DEPLOYMENT", 1)
|
35
27
|
|
@@ -92,6 +84,7 @@ class DefaultConfig(object):
|
|
92
84
|
|
93
85
|
|
94
86
|
class Development(DefaultConfig):
|
87
|
+
|
95
88
|
""" """
|
96
89
|
|
97
90
|
ENV = "development"
|
@@ -102,7 +95,7 @@ class Testing(DefaultConfig):
|
|
102
95
|
|
103
96
|
ENV = "testing"
|
104
97
|
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
105
|
-
DEBUG =
|
98
|
+
DEBUG = False
|
106
99
|
TESTING = True
|
107
100
|
PROPAGATE_EXCEPTIONS = True
|
108
101
|
SECRET_TOKEN_KEY = "TESTINGSECRETKEY"
|
@@ -126,7 +119,6 @@ class Production(DefaultConfig):
|
|
126
119
|
# needs to be on to avoid getting only 500 codes:
|
127
120
|
# and https://medium.com/@johanesriandy/flask-error-handler-not-working-on-production-mode-3adca4c7385c
|
128
121
|
PROPAGATE_EXCEPTIONS = True
|
129
|
-
UPLOAD_FOLDER = os.getenv("UPLOAD_FOLDER", "/usr/src/app/static")
|
130
122
|
|
131
123
|
|
132
124
|
app_config = {"development": Development, "testing": Testing, "production": Production}
|
cornflow/endpoints/__init__.py
CHANGED
@@ -37,8 +37,6 @@ from .execution import (
|
|
37
37
|
ExecutionLogEndpoint,
|
38
38
|
ExecutionRelaunchEndpoint,
|
39
39
|
)
|
40
|
-
|
41
|
-
from .reports import ReportEndpoint, ReportDetailsEndpoint, ReportDetailsEditEndpoint
|
42
40
|
from .health import HealthEndpoint
|
43
41
|
from .instance import (
|
44
42
|
InstanceEndpoint,
|
@@ -49,7 +47,6 @@ from .instance import (
|
|
49
47
|
from .licenses import LicensesEndpoint
|
50
48
|
from .main_alarms import MainAlarmsEndpoint
|
51
49
|
from .permission import PermissionsViewRoleEndpoint, PermissionsViewRoleDetailEndpoint
|
52
|
-
from .reports import ReportEndpoint, ReportDetailsEndpoint
|
53
50
|
from .roles import RolesListEndpoint, RoleDetailEndpoint
|
54
51
|
from .schemas import SchemaDetailsEndpoint, SchemaEndpoint
|
55
52
|
from .tables import TablesEndpoint, TablesDetailsEndpoint
|
@@ -219,17 +216,6 @@ resources = [
|
|
219
216
|
urls="/table/<string:table_name>/<string:idx>/",
|
220
217
|
endpoint="tables-detail",
|
221
218
|
),
|
222
|
-
dict(
|
223
|
-
resource=ReportDetailsEndpoint,
|
224
|
-
urls="/report/<int:idx>/",
|
225
|
-
endpoint="report-detail",
|
226
|
-
),
|
227
|
-
dict(
|
228
|
-
resource=ReportDetailsEditEndpoint,
|
229
|
-
urls="/report/<int:idx>/edit/",
|
230
|
-
endpoint="report-detail-edit",
|
231
|
-
),
|
232
|
-
dict(resource=ReportEndpoint, urls="/report/", endpoint="report"),
|
233
219
|
]
|
234
220
|
|
235
221
|
|
cornflow/endpoints/execution.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
External endpoints to manage the executions: create new ones, list all of them, get one in particular
|
3
3
|
or check the status of an ongoing one
|
4
|
-
These endpoints
|
4
|
+
These endpoints hve different access url, but manage the same data entities
|
5
5
|
"""
|
6
6
|
|
7
7
|
# Import from libraries
|
cornflow/endpoints/login.py
CHANGED
@@ -64,7 +64,6 @@ class LoginBaseEndpoint(BaseMetaResource):
|
|
64
64
|
try:
|
65
65
|
token = self.auth_class.generate_token(user.id)
|
66
66
|
except Exception as e:
|
67
|
-
current_app.logger.error(f"Error in generating user token: {str(e)}")
|
68
67
|
raise InvalidUsage(f"Error in generating user token: {str(e)}", 400)
|
69
68
|
|
70
69
|
response.update({"token": token, "id": user.id})
|
@@ -83,11 +82,9 @@ class LoginBaseEndpoint(BaseMetaResource):
|
|
83
82
|
user = self.data_model.get_one_object(username=username)
|
84
83
|
|
85
84
|
if not user:
|
86
|
-
current_app.logger.error(f"Error on login user does not exist")
|
87
85
|
raise InvalidCredentials()
|
88
86
|
|
89
87
|
if not user.check_hash(password):
|
90
|
-
current_app.logger.error(f"Error on login invalid credentials")
|
91
88
|
raise InvalidCredentials()
|
92
89
|
|
93
90
|
return user
|
@@ -226,11 +223,7 @@ class LoginEndpoint(LoginBaseEndpoint):
|
|
226
223
|
:rtype: Tuple(dict, integer)
|
227
224
|
"""
|
228
225
|
|
229
|
-
|
230
|
-
return self.log_in(**kwargs)
|
231
|
-
except Exception as e:
|
232
|
-
current_app.logger.error(f"Final exception: {str(e)}")
|
233
|
-
raise e
|
226
|
+
return self.log_in(**kwargs)
|
234
227
|
|
235
228
|
|
236
229
|
class LoginOpenAuthEndpoint(LoginBaseEndpoint):
|
cornflow/models/__init__.py
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
"""
|
2
2
|
Initialization file for the models module
|
3
3
|
"""
|
4
|
-
|
5
4
|
from .action import ActionModel
|
6
5
|
from .alarms import AlarmsModel
|
7
6
|
from .case import CaseModel
|
@@ -15,4 +14,3 @@ from .role import RoleModel
|
|
15
14
|
from .user import UserModel
|
16
15
|
from .user_role import UserRoleModel
|
17
16
|
from .view import ViewModel
|
18
|
-
from .reports import ReportModel
|
cornflow/models/execution.py
CHANGED
@@ -64,14 +64,6 @@ class ExecutionModel(BaseDataModel):
|
|
64
64
|
default=EXECUTION_STATE_MESSAGE_DICT[DEFAULT_EXECUTION_CODE],
|
65
65
|
nullable=True,
|
66
66
|
)
|
67
|
-
reports = db.relationship(
|
68
|
-
"ReportModel",
|
69
|
-
backref="executions",
|
70
|
-
lazy=True,
|
71
|
-
primaryjoin="and_(ExecutionModel.id==ReportModel.execution_id, "
|
72
|
-
"ReportModel.deleted_at==None)",
|
73
|
-
cascade="all,delete",
|
74
|
-
)
|
75
67
|
|
76
68
|
def __init__(self, data):
|
77
69
|
super().__init__(data)
|
cornflow/models/meta_models.py
CHANGED
@@ -33,29 +33,17 @@ class EmptyBaseModel(db.Model):
|
|
33
33
|
|
34
34
|
try:
|
35
35
|
db.session.commit()
|
36
|
-
current_app.logger.debug(
|
37
|
-
f"Transaction type: {action}, performed correctly on {self}"
|
38
|
-
)
|
39
|
-
|
36
|
+
current_app.logger.debug(f"Transaction type: {action}, performed correctly on {self}")
|
40
37
|
except IntegrityError as err:
|
41
38
|
db.session.rollback()
|
42
39
|
current_app.logger.error(f"Integrity error on {action} data: {err}")
|
43
40
|
current_app.logger.error(f"Data: {self.__dict__}")
|
44
|
-
|
45
|
-
if "FOREIGN KEY" in str(err):
|
46
|
-
message = f"Foreign key constraint error while {action} on {self.__class__.__tablename__} table"
|
47
|
-
raise InvalidData(message)
|
48
|
-
else:
|
49
|
-
raise InvalidData(f"Integrity error on {action} with data {self}")
|
50
|
-
|
41
|
+
raise InvalidData(f"Integrity error on {action} with data {self}")
|
51
42
|
except DBAPIError as err:
|
52
43
|
db.session.rollback()
|
53
|
-
current_app.logger.error(
|
54
|
-
f"Unknown database error on {action} data: {type(err)}"
|
55
|
-
)
|
44
|
+
current_app.logger.error(f"Unknown database error on {action} data: {err}")
|
56
45
|
current_app.logger.error(f"Data: {self.__dict__}")
|
57
46
|
raise InvalidData(f"Unknown database error on {action} with data {self}")
|
58
|
-
|
59
47
|
except Exception as err:
|
60
48
|
db.session.rollback()
|
61
49
|
current_app.logger.error(f"Unknown error on {action} data: {err}")
|
@@ -111,9 +99,7 @@ class EmptyBaseModel(db.Model):
|
|
111
99
|
action = "bulk create"
|
112
100
|
try:
|
113
101
|
db.session.commit()
|
114
|
-
current_app.logger.debug(
|
115
|
-
f"Transaction type: {action}, performed correctly on {cls}"
|
116
|
-
)
|
102
|
+
current_app.logger.debug(f"Transaction type: {action}, performed correctly on {cls}")
|
117
103
|
except IntegrityError as err:
|
118
104
|
db.session.rollback()
|
119
105
|
current_app.logger.error(f"Integrity error on {action} data: {err}")
|
@@ -134,9 +120,7 @@ class EmptyBaseModel(db.Model):
|
|
134
120
|
action = "bulk create update"
|
135
121
|
try:
|
136
122
|
db.session.commit()
|
137
|
-
current_app.logger.debug(
|
138
|
-
f"Transaction type: {action}, performed correctly on {cls}"
|
139
|
-
)
|
123
|
+
current_app.logger.debug(f"Transaction type: {action}, performed correctly on {cls}")
|
140
124
|
except IntegrityError as err:
|
141
125
|
db.session.rollback()
|
142
126
|
current_app.logger.error(f"Integrity error on {action} data: {err}")
|
@@ -152,7 +136,12 @@ class EmptyBaseModel(db.Model):
|
|
152
136
|
return instances
|
153
137
|
|
154
138
|
@classmethod
|
155
|
-
def get_all_objects(
|
139
|
+
def get_all_objects(
|
140
|
+
cls,
|
141
|
+
offset=0,
|
142
|
+
limit=None,
|
143
|
+
**kwargs
|
144
|
+
):
|
156
145
|
"""
|
157
146
|
Method to get all the objects from the database applying the filters passed as keyword arguments
|
158
147
|
|
@@ -272,7 +261,7 @@ class TraceAttributesModel(EmptyBaseModel):
|
|
272
261
|
update_date_lte=None,
|
273
262
|
offset=0,
|
274
263
|
limit=None,
|
275
|
-
**kwargs
|
264
|
+
**kwargs
|
276
265
|
):
|
277
266
|
"""
|
278
267
|
Method to get all the objects from the database applying the filters passed as keyword arguments
|
cornflow/schemas/execution.py
CHANGED
@@ -5,7 +5,6 @@ from marshmallow import fields, Schema, validate
|
|
5
5
|
from cornflow.shared.const import MIN_EXECUTION_STATUS_CODE, MAX_EXECUTION_STATUS_CODE
|
6
6
|
from .common import QueryFilters, BaseDataEndpointResponse
|
7
7
|
from .solution_log import LogSchema, BasicLogSchema
|
8
|
-
from .reports import ReportSchemaBase
|
9
8
|
|
10
9
|
|
11
10
|
class QueryFiltersExecution(QueryFilters):
|
@@ -31,7 +30,6 @@ class ConfigSchema(Schema):
|
|
31
30
|
threads = fields.Int(required=False)
|
32
31
|
logPath = fields.Str(required=False)
|
33
32
|
MIPGap = fields.Float(required=False)
|
34
|
-
report = fields.Raw(required=False)
|
35
33
|
|
36
34
|
|
37
35
|
class ConfigSchemaResponse(ConfigSchema):
|
@@ -97,7 +95,6 @@ class ExecutionDagPostRequest(ExecutionRequest, ExecutionDagRequest):
|
|
97
95
|
|
98
96
|
|
99
97
|
class ExecutionDetailsEndpointResponse(BaseDataEndpointResponse):
|
100
|
-
reports = fields.Nested(ReportSchemaBase, many=True)
|
101
98
|
config = fields.Nested(ConfigSchemaResponse)
|
102
99
|
instance_id = fields.Str()
|
103
100
|
state = fields.Int()
|
cornflow/shared/const.py
CHANGED
@@ -43,25 +43,6 @@ AIRFLOW_TO_STATE_MAP = dict(
|
|
43
43
|
queued=EXEC_STATE_QUEUED,
|
44
44
|
)
|
45
45
|
|
46
|
-
# Reports codes
|
47
|
-
|
48
|
-
|
49
|
-
class REPORT_STATE:
|
50
|
-
RUNNING = 0
|
51
|
-
CORRECT = 1
|
52
|
-
ERROR = -1
|
53
|
-
UNKNOWN = -5
|
54
|
-
ERROR_NO_QUARTO = -10
|
55
|
-
|
56
|
-
|
57
|
-
REPORT_STATE_MSG = {
|
58
|
-
REPORT_STATE.RUNNING: "The report is currently running.",
|
59
|
-
REPORT_STATE.CORRECT: "The report has been solved correctly.",
|
60
|
-
REPORT_STATE.ERROR: "The report has an error.",
|
61
|
-
REPORT_STATE.UNKNOWN: "The report has an unknown error.",
|
62
|
-
REPORT_STATE.ERROR_NO_QUARTO: "The report failed because Quarto was not found.",
|
63
|
-
}
|
64
|
-
|
65
46
|
# These codes and names are inherited from flask app builder in order to have the same names and values
|
66
47
|
# as this library that is the base of airflow
|
67
48
|
AUTH_DB = 1
|
@@ -141,6 +122,4 @@ BASE_PERMISSION_ASSIGNATION = [
|
|
141
122
|
|
142
123
|
EXTRA_PERMISSION_ASSIGNATION = [
|
143
124
|
(VIEWER_ROLE, PUT_ACTION, "user-detail"),
|
144
|
-
(VIEWER_ROLE, GET_ACTION, "report"),
|
145
|
-
(PLANNER_ROLE, GET_ACTION, "report"),
|
146
125
|
]
|
cornflow/shared/exceptions.py
CHANGED
@@ -21,10 +21,7 @@ class InvalidUsage(Exception):
|
|
21
21
|
def __init__(self, error=None, status_code=None, payload=None, log_txt=None):
|
22
22
|
Exception.__init__(self, error)
|
23
23
|
if error is not None:
|
24
|
-
|
25
|
-
self.error = str(error)
|
26
|
-
else:
|
27
|
-
self.error = error
|
24
|
+
self.error = error
|
28
25
|
if status_code is not None:
|
29
26
|
self.status_code = status_code
|
30
27
|
self.payload = payload
|
@@ -125,21 +122,10 @@ class ConfigurationError(InvalidUsage):
|
|
125
122
|
error = "No authentication method configured on the server"
|
126
123
|
|
127
124
|
|
128
|
-
class FileError(InvalidUsage):
|
129
|
-
"""
|
130
|
-
Exception used when there is an error regarding the upload of a file to the server
|
131
|
-
"""
|
132
|
-
|
133
|
-
status_code = 400
|
134
|
-
error = "Error uploading the file"
|
135
|
-
|
136
|
-
|
137
125
|
INTERNAL_SERVER_ERROR_MESSAGE = "500 Internal Server Error"
|
138
|
-
INTERNAL_SERVER_ERROR_MESSAGE_DETAIL =
|
139
|
-
|
140
|
-
|
141
|
-
"there is an error in the application."
|
142
|
-
)
|
126
|
+
INTERNAL_SERVER_ERROR_MESSAGE_DETAIL = "The server encountered an internal error and was unable " \
|
127
|
+
"to complete your request. Either the server is overloaded or " \
|
128
|
+
"there is an error in the application."
|
143
129
|
|
144
130
|
|
145
131
|
def initialize_errorhandlers(app):
|
@@ -201,7 +187,10 @@ def initialize_errorhandlers(app):
|
|
201
187
|
status_code = error.code or status_code
|
202
188
|
error_msg = f"{status_code} {error.name or INTERNAL_SERVER_ERROR_MESSAGE}"
|
203
189
|
error_str = f"{error_msg}. {str(error.description or '') or INTERNAL_SERVER_ERROR_MESSAGE_DETAIL}"
|
204
|
-
response_dict = {
|
190
|
+
response_dict = {
|
191
|
+
"message": error_msg,
|
192
|
+
"error": error_str
|
193
|
+
}
|
205
194
|
response = jsonify(response_dict)
|
206
195
|
|
207
196
|
elif app.config["ENV"] == "production":
|
@@ -213,7 +202,7 @@ def initialize_errorhandlers(app):
|
|
213
202
|
|
214
203
|
response_dict = {
|
215
204
|
"message": INTERNAL_SERVER_ERROR_MESSAGE,
|
216
|
-
"error": INTERNAL_SERVER_ERROR_MESSAGE_DETAIL
|
205
|
+
"error": INTERNAL_SERVER_ERROR_MESSAGE_DETAIL
|
217
206
|
}
|
218
207
|
response = jsonify(response_dict)
|
219
208
|
else:
|
cornflow/tests/const.py
CHANGED
@@ -14,7 +14,6 @@ INSTANCE_URL = PREFIX + "/instance/"
|
|
14
14
|
INSTANCE_MPS = _get_file("./data/test_mps.mps")
|
15
15
|
INSTANCE_GC_20 = _get_file("./data/gc_20_7.json")
|
16
16
|
INSTANCE_FILE_FAIL = _get_file("./unit/test_instances.py")
|
17
|
-
INSTANCE_TSP = _get_file("./data/tsp_instance.json")
|
18
17
|
|
19
18
|
EXECUTION_PATH = _get_file("./data/new_execution.json")
|
20
19
|
BAD_EXECUTION_PATH = _get_file("./data/bad_execution.json")
|
@@ -35,12 +34,6 @@ CASE_INSTANCE_URL = PREFIX + "/case/instance/"
|
|
35
34
|
FULL_CASE_PATH = _get_file("./data/full_case_raw.json")
|
36
35
|
FULL_CASE_LIST = [FULL_CASE_PATH, _get_file("./data/full_case_raw_2.json")]
|
37
36
|
|
38
|
-
REPORT_PATH = _get_file("./data/new_report.json")
|
39
|
-
REPORT_HTML_FILE_PATH = _get_file("./data/new_report.html")
|
40
|
-
REPORT_PDF_FILE_PATH = _get_file("./data/new_report_2.pdf")
|
41
|
-
BAD_REPORT_PATH = _get_file("./data/bad_report.json")
|
42
|
-
REPORT_URL = PREFIX + "/report/"
|
43
|
-
|
44
37
|
JSON_PATCH_GOOD_PATH = _get_file("./data/json_patch_good.json")
|
45
38
|
JSON_PATCH_BAD_PATH = _get_file("./data/json_patch_bad.json")
|
46
39
|
FULL_CASE_JSON_PATCH_1 = _get_file("./data/full_case_patch.json")
|
@@ -39,9 +39,7 @@ class CustomTestCaseLive(LiveServerTestCase):
|
|
39
39
|
if create_all:
|
40
40
|
db.create_all()
|
41
41
|
access_init_command(False)
|
42
|
-
register_deployed_dags_command_test(
|
43
|
-
verbose=False, dags=["solve_model_dag", "gc", "timer", "tsp"]
|
44
|
-
)
|
42
|
+
register_deployed_dags_command_test(verbose=False)
|
45
43
|
user_data = dict(
|
46
44
|
username="testname",
|
47
45
|
email="test@test.com",
|
@@ -90,8 +90,8 @@ class CustomTestCase(TestCase):
|
|
90
90
|
self.roles_with_access = []
|
91
91
|
|
92
92
|
@staticmethod
|
93
|
-
def get_header_with_auth(token
|
94
|
-
return {"Content-Type":
|
93
|
+
def get_header_with_auth(token):
|
94
|
+
return {"Content-Type": "application/json", "Authorization": "Bearer " + token}
|
95
95
|
|
96
96
|
def create_user(self, data):
|
97
97
|
return self.client.post(
|
@@ -169,6 +169,7 @@ class CustomTestCase(TestCase):
|
|
169
169
|
self.assertEqual(row.id, response.json["id"])
|
170
170
|
|
171
171
|
for key in self.get_keys_to_check(payload):
|
172
|
+
getattr(row, key)
|
172
173
|
if key in payload:
|
173
174
|
self.assertEqual(getattr(row, key), payload[key])
|
174
175
|
return row.id
|
@@ -3,7 +3,7 @@ from flask import current_app
|
|
3
3
|
from cornflow.commands.dag import register_deployed_dags_command
|
4
4
|
from cornflow.models import DeployedDAG
|
5
5
|
from cornflow.tests.const import PUBLIC_DAGS
|
6
|
-
from cornflow.tests.
|
6
|
+
from cornflow.tests.custom_liveServer import CustomTestCaseLive
|
7
7
|
|
8
8
|
|
9
9
|
class TestCornflowCommands(CustomTestCaseLive):
|
@@ -1,17 +1,17 @@
|
|
1
1
|
"""
|
2
|
-
Main script to run the integration tests of cornflow-server
|
3
|
-
"""
|
4
2
|
|
3
|
+
"""
|
4
|
+
# Full imports
|
5
5
|
import json
|
6
|
+
import pulp
|
6
7
|
import logging as log
|
7
|
-
import os
|
8
8
|
import time
|
9
|
-
from typing import Callable, Any
|
10
9
|
|
11
|
-
|
10
|
+
# Imports from environment
|
12
11
|
from cornflow_client import CornFlowApiError
|
13
12
|
from cornflow_client.constants import INSTANCE_SCHEMA, SOLUTION_SCHEMA
|
14
13
|
|
14
|
+
# Import internal modules
|
15
15
|
from cornflow.app import create_app
|
16
16
|
from cornflow.shared.const import (
|
17
17
|
EXEC_STATE_CORRECT,
|
@@ -19,13 +19,12 @@ from cornflow.shared.const import (
|
|
19
19
|
EXEC_STATE_RUNNING,
|
20
20
|
EXEC_STATE_QUEUED,
|
21
21
|
STATUS_HEALTHY,
|
22
|
-
REPORT_STATE,
|
23
22
|
)
|
24
|
-
from cornflow.tests.const import INSTANCE_PATH, CASE_PATH
|
25
|
-
from cornflow.tests.
|
23
|
+
from cornflow.tests.const import INSTANCE_PATH, CASE_PATH
|
24
|
+
from cornflow.tests.custom_liveServer import CustomTestCaseLive
|
26
25
|
|
27
26
|
|
28
|
-
def load_file(_file
|
27
|
+
def load_file(_file):
|
29
28
|
with open(_file) as f:
|
30
29
|
temp = json.load(f)
|
31
30
|
return temp
|
@@ -35,7 +34,6 @@ class TestCornflowClientBasic(CustomTestCaseLive):
|
|
35
34
|
def setUp(self, create_all=False):
|
36
35
|
super().setUp()
|
37
36
|
self.items_to_check = ["name", "description"]
|
38
|
-
log.info(f"Start test case name: {self.id()}")
|
39
37
|
|
40
38
|
def check_status_evolution(self, execution, end_state=EXEC_STATE_CORRECT):
|
41
39
|
statuses = [execution["state"]]
|
@@ -129,7 +127,7 @@ class TestCornflowClientBasic(CustomTestCaseLive):
|
|
129
127
|
return execution
|
130
128
|
|
131
129
|
def create_instance_and_execution(self):
|
132
|
-
one_instance = self.create_new_instance(
|
130
|
+
one_instance = self.create_new_instance("./cornflow/tests/data/test_mps.mps")
|
133
131
|
name = "test_execution_name_123"
|
134
132
|
description = "test_execution_description_123"
|
135
133
|
schema = "solve_model_dag"
|
@@ -142,31 +140,6 @@ class TestCornflowClientBasic(CustomTestCaseLive):
|
|
142
140
|
)
|
143
141
|
return self.create_new_execution(payload)
|
144
142
|
|
145
|
-
def create_instance_and_execution_report(
|
146
|
-
self,
|
147
|
-
schema="tsp",
|
148
|
-
solver="cpsat",
|
149
|
-
data=None,
|
150
|
-
timeLimit=10,
|
151
|
-
report_name="report",
|
152
|
-
):
|
153
|
-
name = "test_instance_1"
|
154
|
-
description = "description123"
|
155
|
-
if data is None:
|
156
|
-
data = load_file(INSTANCE_TSP)
|
157
|
-
payload = dict(data=data, name=name, description=description, schema=schema)
|
158
|
-
one_instance = self.create_new_instance_payload(payload)
|
159
|
-
payload = dict(
|
160
|
-
instance_id=one_instance["id"],
|
161
|
-
config=dict(
|
162
|
-
solver=solver, timeLimit=timeLimit, report=dict(name=report_name)
|
163
|
-
),
|
164
|
-
description="test_execution_description_123",
|
165
|
-
name="test_execution_123",
|
166
|
-
schema=schema,
|
167
|
-
)
|
168
|
-
return self.create_new_execution(payload)
|
169
|
-
|
170
143
|
def create_timer_instance_and_execution(self, seconds=5):
|
171
144
|
payload = dict(
|
172
145
|
data=dict(seconds=seconds),
|
@@ -184,47 +157,16 @@ class TestCornflowClientBasic(CustomTestCaseLive):
|
|
184
157
|
)
|
185
158
|
return self.create_new_execution(payload)
|
186
159
|
|
187
|
-
@staticmethod
|
188
|
-
def try_until_condition(
|
189
|
-
func: Callable,
|
190
|
-
condition: Callable[[Any], bool],
|
191
|
-
number_of_times: int = 10,
|
192
|
-
sleep_time: float = 10,
|
193
|
-
):
|
194
|
-
for i in range(number_of_times):
|
195
|
-
time.sleep(sleep_time)
|
196
|
-
result = func()
|
197
|
-
if condition(result):
|
198
|
-
return result
|
199
|
-
raise TimeoutError(
|
200
|
-
"Timed out after {} seconds".format(number_of_times * sleep_time)
|
201
|
-
)
|
202
|
-
|
203
|
-
@staticmethod
|
204
|
-
def wait_until_report_finishes(
|
205
|
-
client, execution_id: str, report_status=REPORT_STATE.CORRECT
|
206
|
-
):
|
207
|
-
def func():
|
208
|
-
my_reports = client.raw.get_results(execution_id).json()["reports"]
|
209
|
-
if len(my_reports) == 0:
|
210
|
-
return None
|
211
|
-
first = my_reports[0]
|
212
|
-
if first["state"] != report_status:
|
213
|
-
return None
|
214
|
-
return first
|
215
|
-
|
216
|
-
return func
|
217
|
-
|
218
160
|
|
219
161
|
class TestCornflowClientOpen(TestCornflowClientBasic):
|
220
162
|
# TODO: user management
|
221
163
|
# TODO: infeasible execution
|
222
164
|
|
223
165
|
def test_new_instance_file(self):
|
224
|
-
self.create_new_instance_file(
|
166
|
+
self.create_new_instance_file("./cornflow/tests/data/test_mps.mps")
|
225
167
|
|
226
168
|
def test_new_instance(self):
|
227
|
-
return self.create_new_instance(
|
169
|
+
return self.create_new_instance("./cornflow/tests/data/test_mps.mps")
|
228
170
|
|
229
171
|
# TODO: reactivate test with new version of cornflow client which allows to pass
|
230
172
|
# optional arguments for the headers of the request
|
@@ -247,49 +189,6 @@ class TestCornflowClientOpen(TestCornflowClientBasic):
|
|
247
189
|
def test_new_execution(self):
|
248
190
|
return self.create_instance_and_execution()
|
249
191
|
|
250
|
-
def test_new_execution_with_tsp_report(self):
|
251
|
-
return self.create_instance_and_execution_report()
|
252
|
-
|
253
|
-
#
|
254
|
-
def test_new_execution_with_tsp_report_wait(self):
|
255
|
-
execution = self.create_instance_and_execution_report()
|
256
|
-
func = self.wait_until_report_finishes(self.client, execution["id"])
|
257
|
-
reports_info = self.try_until_condition(func, lambda v: v is not None, 20, 5)
|
258
|
-
id_report = reports_info["id"]
|
259
|
-
my_name = "./my_report.html"
|
260
|
-
try:
|
261
|
-
os.remove(my_name)
|
262
|
-
except:
|
263
|
-
pass
|
264
|
-
self.client.raw.get_one_report(id_report, "./", my_name)
|
265
|
-
self.assertTrue(os.path.exists(my_name))
|
266
|
-
try:
|
267
|
-
os.remove(my_name)
|
268
|
-
except OSError:
|
269
|
-
pass
|
270
|
-
|
271
|
-
# read header of file? we can parse it with beatifulsoup
|
272
|
-
|
273
|
-
def test_new_execution_with_timer_report_wait(self):
|
274
|
-
payload = dict(
|
275
|
-
solver="default", schema="timer", data={"seconds": 1}, timeLimit=1
|
276
|
-
)
|
277
|
-
execution = self.create_instance_and_execution_report(**payload)
|
278
|
-
func = self.wait_until_report_finishes(self.client, execution["id"])
|
279
|
-
reports_info = self.try_until_condition(func, lambda v: v is not None, 20, 5)
|
280
|
-
id_report = reports_info["id"]
|
281
|
-
my_name = "./my_report.html"
|
282
|
-
try:
|
283
|
-
os.remove(my_name)
|
284
|
-
except:
|
285
|
-
pass
|
286
|
-
self.client.raw.get_one_report(id_report, "./", my_name)
|
287
|
-
self.assertTrue(os.path.exists(my_name))
|
288
|
-
try:
|
289
|
-
os.remove(my_name)
|
290
|
-
except OSError:
|
291
|
-
pass
|
292
|
-
|
293
192
|
def test_delete_execution(self):
|
294
193
|
execution = self.test_new_execution()
|
295
194
|
response = self.client.raw.get_api_for_id("execution/", execution["id"])
|
@@ -314,7 +213,7 @@ class TestCornflowClientOpen(TestCornflowClientBasic):
|
|
314
213
|
self.assertTrue("error" in response.json())
|
315
214
|
|
316
215
|
def test_new_execution_bad_dag_name(self):
|
317
|
-
one_instance = self.create_new_instance(
|
216
|
+
one_instance = self.create_new_instance("./cornflow/tests/data/test_mps.mps")
|
318
217
|
name = "test_execution_name_123"
|
319
218
|
description = "test_execution_description_123"
|
320
219
|
payload = dict(
|
@@ -328,7 +227,7 @@ class TestCornflowClientOpen(TestCornflowClientBasic):
|
|
328
227
|
self.assertRaises(CornFlowApiError, _bad_func)
|
329
228
|
|
330
229
|
def test_new_execution_with_schema(self):
|
331
|
-
one_instance = self.create_new_instance(
|
230
|
+
one_instance = self.create_new_instance("./cornflow/tests/data/test_mps.mps")
|
332
231
|
name = "test_execution_name_123"
|
333
232
|
description = "test_execution_description_123"
|
334
233
|
payload = dict(
|
@@ -410,6 +309,19 @@ class TestCornflowClientAdmin(TestCornflowClientBasic):
|
|
410
309
|
def setUp(self, create_all=False):
|
411
310
|
super().setUp()
|
412
311
|
|
312
|
+
# we create a service user:
|
313
|
+
self.create_service_user(
|
314
|
+
dict(username="airflow", pwd="Airflow_test_password1", email="af@cf.com")
|
315
|
+
)
|
316
|
+
|
317
|
+
self.create_service_user(
|
318
|
+
dict(
|
319
|
+
username="service_user@cornflow.com",
|
320
|
+
pwd="Serviceuser_1234",
|
321
|
+
email="service_user@cornflow.com",
|
322
|
+
)
|
323
|
+
)
|
324
|
+
|
413
325
|
# we create an admin user
|
414
326
|
# we guarantee that the admin is there for airflow
|
415
327
|
self.client.token = self.create_admin(
|
@@ -492,7 +404,7 @@ class TestCornflowClientAdmin(TestCornflowClientBasic):
|
|
492
404
|
self.assertIsNone(execution_data["data"])
|
493
405
|
|
494
406
|
def test_edit_one_execution(self):
|
495
|
-
one_instance = self.create_new_instance(
|
407
|
+
one_instance = self.create_new_instance("./cornflow/tests/data/test_mps.mps")
|
496
408
|
payload = dict(
|
497
409
|
name="bla",
|
498
410
|
config=dict(solver="CBC"),
|
@@ -543,7 +455,7 @@ class TestCornflowClientAdmin(TestCornflowClientBasic):
|
|
543
455
|
self.assertRaises(CornFlowApiError, _launch_too_soon_func)
|
544
456
|
|
545
457
|
def test_check_instance(self):
|
546
|
-
instance = self.create_new_instance(
|
458
|
+
instance = self.create_new_instance("./cornflow/tests/data/test_mps.mps")
|
547
459
|
data_check_execution = self.client.create_instance_data_check(instance["id"])
|
548
460
|
self.assertEqual(data_check_execution["instance_id"], instance["id"])
|
549
461
|
status = self.client.get_status(data_check_execution["id"])
|