cornflow 1.1.0a1__py3-none-any.whl → 1.1.1a1__py3-none-any.whl

cornflow/app.py

@@ -46,6 +46,9 @@ def create_app(env_name="development", dataconn=None):
     :return: the application that is going to be running :class:`Flask`
     :rtype: :class:`Flask`
     """
+    if os.getenv("FLASK_ENV", None) is not None:
+        env_name = os.getenv("FLASK_ENV")
+
     dictConfig(log_config(app_config[env_name].LOG_LEVEL))
 
     app = Flask(__name__)
@@ -74,6 +77,7 @@ def create_app(env_name="development", dataconn=None):
     api = Api(app)
     for res in resources:
         api.add_resource(res["resource"], res["urls"], endpoint=res["endpoint"])
+
     if app.config["ALARMS_ENDPOINTS"]:
         for res in alarms_resources:
             api.add_resource(res["resource"], res["urls"], endpoint=res["endpoint"])

cornflow/cli/utils.py

@@ -6,7 +6,7 @@ import warnings
 
 def get_app():
     env = os.getenv("FLASK_ENV", "development")
-    data_conn = os.getenv("DATABASE_URL", "sqlite:///cornflow.db")
+    data_conn = os.getenv("DATABASE_URL")
     if env == "production":
         warnings.filterwarnings("ignore")
     external = int(os.getenv("EXTERNAL_APP", 0))

cornflow/config.py

@@ -22,6 +22,14 @@ class DefaultConfig(object):
     SIGNUP_ACTIVATED = int(os.getenv("SIGNUP_ACTIVATED", 1))
     CORNFLOW_SERVICE_USER = os.getenv("CORNFLOW_SERVICE_USER", "service_user")
 
+    # file support for reports
+    FILE_BACKEND = os.getenv("FILE_BACKEND", "local")
+    UPLOAD_FOLDER = os.getenv(
+        "UPLOAD_FOLDER",
+        os.path.abspath(os.path.join(os.path.dirname(__file__), "./static")),
+    )
+    ALLOWED_EXTENSIONS = os.getenv("ALLOWED_EXTENSIONS", ["pdf", "html"])
+
     # Open deployment (all dags accessible to all users)
     OPEN_DEPLOYMENT = os.getenv("OPEN_DEPLOYMENT", 1)
 
@@ -84,7 +92,6 @@ class DefaultConfig(object):
 
 
 class Development(DefaultConfig):
-
     """ """
 
     ENV = "development"
@@ -95,7 +102,7 @@ class Testing(DefaultConfig):
 
     ENV = "testing"
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-    DEBUG = False
+    DEBUG = True
     TESTING = True
     PROPAGATE_EXCEPTIONS = True
     SECRET_TOKEN_KEY = "TESTINGSECRETKEY"
@@ -119,6 +126,7 @@ class Production(DefaultConfig):
     # needs to be on to avoid getting only 500 codes:
     # and https://medium.com/@johanesriandy/flask-error-handler-not-working-on-production-mode-3adca4c7385c
     PROPAGATE_EXCEPTIONS = True
+    UPLOAD_FOLDER = os.getenv("UPLOAD_FOLDER", "/usr/src/app/static")
 
 
 app_config = {"development": Development, "testing": Testing, "production": Production}

cornflow/endpoints/__init__.py

@@ -4,8 +4,8 @@ All references to endpoints should be imported from here
 The login resource gets created on app startup as it depends on configuration
 """
 from .action import ActionListEndpoint
+from .alarms import AlarmsEndpoint
 from .apiview import ApiViewListEndpoint
-
 from .case import (
     CaseEndpoint,
     CaseFromInstanceExecutionEndpoint,
@@ -15,7 +15,6 @@ from .case import (
     CaseToInstance,
     CaseCompare,
 )
-
 from .dag import (
     DAGDetailEndpoint,
     DAGEndpointManual,
@@ -24,7 +23,12 @@ from .dag import (
     DeployedDAGEndpoint,
     DeployedDagDetailEndpoint,
 )
-
+from .data_check import (
+    DataCheckExecutionEndpoint,
+    DataCheckInstanceEndpoint,
+    DataCheckCaseEndpoint,
+)
+from .example_data import ExampleDataListEndpoint, ExampleDataDetailEndpoint
 from .execution import (
     ExecutionEndpoint,
     ExecutionDetailsEndpoint,
@@ -34,35 +38,24 @@ from .execution import (
     ExecutionRelaunchEndpoint,
 )
 
+from .reports import ReportEndpoint, ReportDetailsEndpoint, ReportDetailsEditEndpoint
 from .health import HealthEndpoint
-
 from .instance import (
     InstanceEndpoint,
     InstanceDetailsEndpoint,
     InstanceFileEndpoint,
     InstanceDataEndpoint,
 )
-
-from .data_check import (
-    DataCheckExecutionEndpoint,
-    DataCheckInstanceEndpoint,
-    DataCheckCaseEndpoint,
-)
 from .licenses import LicensesEndpoint
+from .main_alarms import MainAlarmsEndpoint
 from .permission import PermissionsViewRoleEndpoint, PermissionsViewRoleDetailEndpoint
-
+from .reports import ReportEndpoint, ReportDetailsEndpoint
 from .roles import RolesListEndpoint, RoleDetailEndpoint
-
 from .schemas import SchemaDetailsEndpoint, SchemaEndpoint
+from .tables import TablesEndpoint, TablesDetailsEndpoint
 from .token import TokenEndpoint
-from .example_data import ExampleDataDetailsEndpoint
 from .user import UserEndpoint, UserDetailsEndpoint, ToggleUserAdmin, RecoverPassword
 from .user_role import UserRoleListEndpoint, UserRoleDetailEndpoint
-from .alarms import AlarmsEndpoint
-from .main_alarms import MainAlarmsEndpoint
-
-from .tables import TablesEndpoint, TablesDetailsEndpoint
-
 
 resources = [
     dict(resource=InstanceEndpoint, urls="/instance/", endpoint="instance"),
@@ -157,10 +150,15 @@ resources = [
         endpoint="schema-details",
     ),
     dict(
-        resource=ExampleDataDetailsEndpoint,
+        resource=ExampleDataListEndpoint,
         urls="/example/<string:dag_name>/",
         endpoint="example-data",
     ),
+    dict(
+        resource=ExampleDataDetailEndpoint,
+        urls="/example/<string:dag_name>/<string:example_name>/",
+        endpoint="example-data-detail",
+    ),
     dict(resource=HealthEndpoint, urls="/health/", endpoint="health"),
     dict(
         resource=CaseFromInstanceExecutionEndpoint,
@@ -221,6 +219,17 @@ resources = [
         urls="/table/<string:table_name>/<string:idx>/",
         endpoint="tables-detail",
     ),
+    dict(
+        resource=ReportDetailsEndpoint,
+        urls="/report/<int:idx>/",
+        endpoint="report-detail",
+    ),
+    dict(
+        resource=ReportDetailsEditEndpoint,
+        urls="/report/<int:idx>/edit/",
+        endpoint="report-detail-edit",
+    ),
+    dict(resource=ReportEndpoint, urls="/report/", endpoint="report"),
 ]
 
 

cornflow/endpoints/example_data.py

@@ -1,36 +1,78 @@
 """
 Endpoints to get the example data from a DAG
 """
+import json
 
-# Import from libraries
 from cornflow_client.airflow.api import Airflow
 from flask import current_app, request
 from flask_apispec import marshal_with, doc
-import json
 
-# Import from internal modules
 from cornflow.endpoints.meta_resource import BaseMetaResource
 from cornflow.models import PermissionsDAG
-from cornflow.schemas.example_data import ExampleData
+from cornflow.schemas.example_data import ExampleListData, ExampleDetailData
 from cornflow.shared.authentication import Auth, authenticate
 from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE
-from cornflow.shared.exceptions import AirflowError, NoPermission
+from cornflow.shared.exceptions import AirflowError, NoPermission, ObjectDoesNotExist
 
 
-class ExampleDataDetailsEndpoint(BaseMetaResource):
+class ExampleDataListEndpoint(BaseMetaResource):
     """
     Endpoint used to obtain schemas for one app
     """
 
     ROLES_WITH_ACCESS = [VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE]
 
-    @doc(description="Get example data from DAG", tags=["DAG"])
+    @doc(description="Get lsit of example data from DAG", tags=["DAG"])
     @authenticate(auth_class=Auth())
-    @marshal_with(ExampleData)
+    @marshal_with(ExampleListData(many=True))
     def get(self, dag_name):
         """
         API method to get example data for a given dag
 
+        :return: A dictionary with the names and descriptions of available data examples
+            and an integer with the HTTP status code
+        :rtype: Tuple(dict, integer)
+        """
+        user = Auth().get_user_from_header(request.headers)
+        permission = PermissionsDAG.check_if_has_permissions(
+            user_id=user.id, dag_id=dag_name
+        )
+
+        if permission:
+            af_client = Airflow.from_config(current_app.config)
+            if not af_client.is_alive():
+                current_app.logger.error(
+                    "Airflow not accessible when getting data {}".format(dag_name)
+                )
+                raise AirflowError(error="Airflow is not accessible")
+
+            # try airflow and see if dag_name exists
+            af_client.get_dag_info(dag_name)
+
+            current_app.logger.info("User gets example data from {}".format(dag_name))
+
+            variable_name = f"z_{dag_name}_examples"
+            response = af_client.get_one_variable(variable_name)
+
+            return json.loads(response["value"])
+        else:
+            err = "User does not have permission to access this dag."
+            raise NoPermission(
+                error=err,
+                status_code=403,
+                log_txt=f"Error while user {user} tries to get example data for dag {dag_name}. "
+                + err,
+            )
+
+
+class ExampleDataDetailEndpoint(BaseMetaResource):
+    @doc(description="Get example data from DAG", tags=["DAG"])
+    @authenticate(auth_class=Auth())
+    @marshal_with(ExampleDetailData)
+    def get(self, dag_name, example_name):
+        """
+        API method to get one example data for a given dag
+
         :return: A dictionary with a message and a integer with the HTTP status code
         :rtype: Tuple(dict, integer)
         """
@@ -54,15 +96,24 @@ class ExampleDataDetailsEndpoint(BaseMetaResource):
 
             variable_name = f"z_{dag_name}_examples"
             response = af_client.get_one_variable(variable_name)
-            result = dict()
-            result["examples"] = json.loads(response["value"])
-            result["name"] = response["key"]
 
-            return result
+            example = None
+            for item in json.loads(response["value"]):
+                if item["name"] == example_name:
+                    example = item
+                    break
+
+            if example is None:
+                raise ObjectDoesNotExist(
+                    error="The example does not exist", status_code=404
+                )
+
+            return example
         else:
             err = "User does not have permission to access this dag."
             raise NoPermission(
                 error=err,
                 status_code=403,
-                log_txt=f"Error while user {user} tries to get example data for dag {dag_name}. " + err
+                log_txt=f"Error while user {user} tries to get example data for dag {dag_name}. "
+                + err,
             )

cornflow/endpoints/execution.py

@@ -1,7 +1,7 @@
 """
 External endpoints to manage the executions: create new ones, list all of them, get one in particular
 or check the status of an ongoing one
-These endpoints hve different access url, but manage the same data entities
+These endpoints have different access url, but manage the same data entities
 """
 
 # Import from libraries

cornflow/endpoints/login.py

@@ -34,6 +34,7 @@ class LoginBaseEndpoint(BaseMetaResource):
     """
     Base endpoint to perform a login action from a user
     """
+
     def __init__(self):
         super().__init__()
         self.ldap_class = LDAPBase
@@ -63,6 +64,7 @@ class LoginBaseEndpoint(BaseMetaResource):
         try:
             token = self.auth_class.generate_token(user.id)
         except Exception as e:
+            current_app.logger.error(f"Error in generating user token: {str(e)}")
             raise InvalidUsage(f"Error in generating user token: {str(e)}", 400)
 
         response.update({"token": token, "id": user.id})
@@ -81,9 +83,11 @@ class LoginBaseEndpoint(BaseMetaResource):
         user = self.data_model.get_one_object(username=username)
 
         if not user:
+            current_app.logger.error(f"Error on login user does not exist")
             raise InvalidCredentials()
 
         if not user.check_hash(password):
+            current_app.logger.error(f"Error on login invalid credentials")
             raise InvalidCredentials()
 
         return user
@@ -102,7 +106,9 @@ class LoginBaseEndpoint(BaseMetaResource):
             raise InvalidCredentials()
         user = self.data_model.get_one_object(username=username)
         if not user:
-            current_app.logger.info(f"LDAP user {username} does not exist and is created")
+            current_app.logger.info(
+                f"LDAP user {username} does not exist and is created"
+            )
             email = ldap_obj.get_user_email(username)
             if not email:
                 email = ""
@@ -122,10 +128,14 @@ class LoginBaseEndpoint(BaseMetaResource):
 
         except IntegrityError as e:
             db.session.rollback()
-            current_app.logger.error(f"Integrity error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Integrity error on user role assignment on log in: {e}"
+            )
         except DBAPIError as e:
             db.session.rollback()
-            current_app.logger.error(f"Unknown error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Unknown error on user role assignment on log in: {e}"
+            )
 
         return user
 
@@ -163,7 +173,9 @@ class LoginBaseEndpoint(BaseMetaResource):
         user = self.data_model.get_one_object(username=username)
 
         if not user:
-            current_app.logger.info(f"OpenID user {username} does not exist and is created")
+            current_app.logger.info(
+                f"OpenID user {username} does not exist and is created"
+            )
 
             data = {"username": username, "email": username}
 
@@ -183,7 +195,11 @@ class LoginBaseEndpoint(BaseMetaResource):
 
 def check_last_password_change(user):
     if user.pwd_last_change:
-        if user.pwd_last_change + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"])) < datetime.utcnow():
+        if (
+            user.pwd_last_change
+            + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"]))
+            < datetime.utcnow()
+        ):
             return True
     return False
 
@@ -210,7 +226,11 @@ class LoginEndpoint(LoginBaseEndpoint):
         :rtype: Tuple(dict, integer)
         """
 
-        return self.log_in(**kwargs)
+        try:
+            return self.log_in(**kwargs)
+        except Exception as e:
+            current_app.logger.error(f"Final exception: {str(e)}")
+            raise e
 
 
 class LoginOpenAuthEndpoint(LoginBaseEndpoint):

cornflow/endpoints/reports.py

@@ -0,0 +1,283 @@
+"""
+External endpoints to manage the reports: create new ones, list all of them, get one in particular
+These endpoints have different access url, but manage the same data entities
+"""
+import os
+
+from flask import current_app, request, send_from_directory
+from flask_apispec import marshal_with, use_kwargs, doc
+from werkzeug.utils import secure_filename
+import uuid
+
+from cornflow.endpoints.meta_resource import BaseMetaResource
+from cornflow.models import ExecutionModel, ReportModel
+from cornflow.schemas.reports import (
+    ReportSchema,
+    ReportEditRequest,
+    QueryFiltersReports,
+    ReportRequest,
+)
+from cornflow.shared.authentication import Auth, authenticate
+from cornflow.shared.const import SERVICE_ROLE
+from cornflow.shared.exceptions import (
+    FileError,
+    ObjectDoesNotExist,
+    NoPermission,
+)
+
+
+class ReportEndpoint(BaseMetaResource):
+    """
+    Endpoint used to create a new report or get all the reports and their information back
+    """
+
+    ROLES_WITH_ACCESS = [SERVICE_ROLE]
+
+    def __init__(self):
+        super().__init__()
+        self.model = ReportModel
+        self.data_model = ReportModel
+        self.foreign_data = {"execution_id": ExecutionModel}
+
+    @doc(description="Get all reports", tags=["Reports"])
+    @authenticate(auth_class=Auth())
+    @marshal_with(ReportSchema(many=True))
+    @use_kwargs(QueryFiltersReports, location="query")
+    def get(self, **kwargs):
+        """
+        API method to get all the reports created by the user and its related info
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user
+
+        :return: A dictionary with a message (error if authentication failed or a list with all the reports
+          created by the authenticated user) and a integer with the HTTP status code
+        :rtype: Tuple(dict, integer)
+        """
+        reports = self.get_list(user=self.get_user(), **kwargs)
+        current_app.logger.info(f"User {self.get_user()} gets list of reports")
+        return reports
+
+    @doc(description="Create a report", tags=["Reports"])
+    @authenticate(auth_class=Auth())
+    @use_kwargs(ReportRequest, location="form")
+    @marshal_with(ReportSchema)
+    def post(self, **kwargs):
+        """
+        API method to create a new report linked to an existing execution
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user
+
+        :return: A dictionary with a message (error if authentication failed, error if data is not validated or
+          the reference_id for the newly created report if successful) and a integer with the HTTP status code
+        :rtype: Tuple(dict, integer)
+        """
+
+        execution = ExecutionModel.get_one_object(idx=kwargs["execution_id"])
+
+        if execution is None:
+            raise ObjectDoesNotExist("The execution does not exist")
+        if "file" not in request.files:
+            # we're creating an empty report.
+            # which is possible
+            report = ReportModel(get_report_info(kwargs, execution, None))
+
+            report.save()
+            return report, 201
+
+        file = request.files["file"]
+        report_name = new_file_name(file)
+
+        report = ReportModel(get_report_info(kwargs, execution, report_name))
+
+        report.save()
+
+        # We try to save the file, if an error is raised then we delete the record on the database
+        try:
+            write_file(file, execution.id, report_name)
+            return report, 201
+
+        except Exception as error:
+            report.delete()
+            current_app.logger.error(error)
+            raise FileError(error=str(error))
+
+
+class ReportDetailsEndpointBase(BaseMetaResource):
+    """
+    Endpoint used to get the information of a certain report. But not the data!
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.data_model = ReportModel
+        self.foreign_data = {"execution_id": ExecutionModel}
+
+
+class ReportDetailsEditEndpoint(ReportDetailsEndpointBase):
+
+    ROLES_WITH_ACCESS = [SERVICE_ROLE]
+
+    @doc(description="Edit a report", tags=["Reports"], inherit=False)
+    @authenticate(auth_class=Auth())
+    @use_kwargs(ReportEditRequest, location="form")
+    def put(self, idx, **data):
+        """
+        Edit an existing report
+
+        :param string idx: ID of the report.
+        :return: A dictionary with a message (error if authentication failed, or the report does not exist or
+          a message) and an integer with the HTTP status code.
+        :rtype: Tuple(dict, integer)
+        """
+        # TODO: forbid non-service users from running put
+        current_app.logger.info(f"User {self.get_user()} edits report {idx}")
+
+        report = self.get_detail(idx=idx)
+
+        if "file" not in request.files:
+            # we're creating an empty report.
+            # which is possible
+            report.update(data)
+            report.save()
+            return {"message": "Updated correctly"}, 200
+
+        # there's two cases,
+        # (1) the report already has a file
+        # (2) the report doesn't yet have a file
+        file = request.files["file"]
+        report_name = new_file_name(file)
+        old_name = report.file_url
+        # we update the report with the new content, including the new name
+        report.update(dict(**data, file_url=report_name))
+
+        # We try to save the file, if an error is raised then we delete the record on the database
+        try:
+            write_file(file, report.execution_id, report_name)
+            report.save()
+
+        except Exception as error:
+            # we do not save the report
+            current_app.logger.error(error)
+            raise FileError(error=str(error))
+
+        # if it saves correctly, we delete the old file, if exists
+        # if unsuccessful, we still return 201 but log the error
+        if old_name is not None:
+            try:
+                os.remove(get_report_path(report))
+            except OSError as error:
+                current_app.logger.error(error)
+        return {"message": "Updated correctly"}, 200
+
+
+class ReportDetailsEndpoint(ReportDetailsEndpointBase):
+    @doc(description="Get details of a report", tags=["Reports"], inherit=False)
+    @authenticate(auth_class=Auth())
+    @marshal_with(ReportSchema)
+    @BaseMetaResource.get_data_or_404
+    def get(self, idx):
+        """
+        API method to get a report created by the user and its related info.
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user.
+
+        :param str idx: ID of the report.
+        :return: A dictionary with a message (error if authentication failed, or the report does not exist or
+          the data of the report) and an integer with the HTTP status code.
+        :rtype: Tuple(dict, integer)
+        """
+        # TODO: are we able to download the name in the database and not as part of the file?
+        current_app.logger.info(f"User {self.get_user()} gets details of report {idx}")
+        report = self.get_detail(user=self.get_user(), idx=idx)
+
+        if report is None:
+            raise ObjectDoesNotExist
+
+        # if there's no file, we do not return it:
+        if report.file_url is None:
+            return report, 200
+
+        my_dir = get_report_dir(report.execution_id)
+        response = send_from_directory(my_dir, report.file_url)
+        response.headers["File-Description"] = report.description
+        response.headers["File-Name"] = report.file_url
+        return response
+
+    @doc(description="Delete a report", tags=["Reports"], inherit=False)
+    @authenticate(auth_class=Auth())
+    def delete(self, idx):
+        """
+        API method to delete a report created by the user and its related info.
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user.
+
+        :param string idx: ID of the report.
+        :return: A dictionary with a message (error if authentication failed, or the report does not exist or
+          a message) and an integer with the HTTP status code.
+        :rtype: Tuple(dict, integer)
+        """
+
+        # get report objet
+        report = self.get_detail(user_id=self.get_user_id(), idx=idx)
+
+        if report is None:
+            raise ObjectDoesNotExist
+
+        # delete file
+        os.remove(get_report_path(report))
+
+        return self.delete_detail(user_id=self.get_user_id(), idx=idx)
+
+
+def get_report_dir(execution_id):
+    return f"{current_app.config['UPLOAD_FOLDER']}/{execution_id}"
+
+
+def get_report_path(report):
+    try:
+        return f"{get_report_dir(report['execution_id'])}/{report['file_url']}"
+    except:
+        return f"{get_report_dir(report.execution_id)}/{report.file_url}"
+
+
+def new_file_name(file):
+
+    filename = secure_filename(file.filename)
+    filename_extension = filename.split(".")[-1]
+
+    if filename_extension not in current_app.config["ALLOWED_EXTENSIONS"]:
+        return {
+            "message": f"Invalid file extension. "
+            f"Valid extensions are: {current_app.config['ALLOWED_EXTENSIONS']}"
+        }, 400
+
+    report_name = f"{uuid.uuid4().hex}.{filename_extension}"
+
+    return report_name
+
+
+def write_file(file, execution_id, file_name):
+    my_directory = get_report_dir(execution_id)
+
+    # we create a directory for the execution
+    if not os.path.exists(my_directory):
+        current_app.logger.info(f"Creating directory {my_directory}")
+        os.mkdir(my_directory)
+
+    save_path = os.path.normpath(os.path.join(my_directory, file_name))
+
+    if "static" not in save_path or ".." in save_path:
+        raise NoPermission("Invalid file name")
+    file.save(save_path)
+
+
+def get_report_info(data, execution, file_url=None):
+    return {
+        "name": data["name"],
+        "file_url": file_url,
+        "execution_id": execution.id,
+        "user_id": execution.user_id,
+        "description": data.get("description", ""),
+        "state": data.get("state"),
+        "state_message": data.get("state_message"),
+    }