cornflow 1.1.0a2__tar.gz → 1.1.1a1__tar.gz

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/MANIFEST.in

@@ -3,4 +3,5 @@ include MANIFEST.in
 include README.rst
 include setup.py
 include cornflow/migrations/*
-include cornflow/migrations/versions/*
+include cornflow/migrations/versions/*
+include cornflow/static/*

{cornflow-1.1.0a2/cornflow.egg-info → cornflow-1.1.1a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: cornflow
-Version: 1.1.0a2
+Version: 1.1.1a1
 Summary: Cornflow is an open source multi-solver optimization server with a REST API built using flask.
 Home-page: https://github.com/baobabsoluciones/cornflow
 Author: baobab soluciones

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/app.py

@@ -46,6 +46,9 @@ def create_app(env_name="development", dataconn=None):
     :return: the application that is going to be running :class:`Flask`
     :rtype: :class:`Flask`
     """
+    if os.getenv("FLASK_ENV", None) is not None:
+        env_name = os.getenv("FLASK_ENV")
+
     dictConfig(log_config(app_config[env_name].LOG_LEVEL))
 
     app = Flask(__name__)
@@ -74,6 +77,7 @@ def create_app(env_name="development", dataconn=None):
     api = Api(app)
     for res in resources:
         api.add_resource(res["resource"], res["urls"], endpoint=res["endpoint"])
+
     if app.config["ALARMS_ENDPOINTS"]:
         for res in alarms_resources:
             api.add_resource(res["resource"], res["urls"], endpoint=res["endpoint"])

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/cli/utils.py

@@ -6,7 +6,7 @@ import warnings
 
 def get_app():
     env = os.getenv("FLASK_ENV", "development")
-    data_conn = os.getenv("DATABASE_URL", "sqlite:///cornflow.db")
+    data_conn = os.getenv("DATABASE_URL")
     if env == "production":
         warnings.filterwarnings("ignore")
     external = int(os.getenv("EXTERNAL_APP", 0))

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/config.py

@@ -22,6 +22,14 @@ class DefaultConfig(object):
     SIGNUP_ACTIVATED = int(os.getenv("SIGNUP_ACTIVATED", 1))
     CORNFLOW_SERVICE_USER = os.getenv("CORNFLOW_SERVICE_USER", "service_user")
 
+    # file support for reports
+    FILE_BACKEND = os.getenv("FILE_BACKEND", "local")
+    UPLOAD_FOLDER = os.getenv(
+        "UPLOAD_FOLDER",
+        os.path.abspath(os.path.join(os.path.dirname(__file__), "./static")),
+    )
+    ALLOWED_EXTENSIONS = os.getenv("ALLOWED_EXTENSIONS", ["pdf", "html"])
+
     # Open deployment (all dags accessible to all users)
     OPEN_DEPLOYMENT = os.getenv("OPEN_DEPLOYMENT", 1)
 
@@ -84,7 +92,6 @@ class DefaultConfig(object):
 
 
 class Development(DefaultConfig):
-
     """ """
 
     ENV = "development"
@@ -95,7 +102,7 @@ class Testing(DefaultConfig):
 
     ENV = "testing"
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-    DEBUG = False
+    DEBUG = True
     TESTING = True
     PROPAGATE_EXCEPTIONS = True
     SECRET_TOKEN_KEY = "TESTINGSECRETKEY"
@@ -119,6 +126,7 @@ class Production(DefaultConfig):
     # needs to be on to avoid getting only 500 codes:
     # and https://medium.com/@johanesriandy/flask-error-handler-not-working-on-production-mode-3adca4c7385c
     PROPAGATE_EXCEPTIONS = True
+    UPLOAD_FOLDER = os.getenv("UPLOAD_FOLDER", "/usr/src/app/static")
 
 
 app_config = {"development": Development, "testing": Testing, "production": Production}

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/endpoints/__init__.py

@@ -37,6 +37,8 @@ from .execution import (
     ExecutionLogEndpoint,
     ExecutionRelaunchEndpoint,
 )
+
+from .reports import ReportEndpoint, ReportDetailsEndpoint, ReportDetailsEditEndpoint
 from .health import HealthEndpoint
 from .instance import (
     InstanceEndpoint,
@@ -47,6 +49,7 @@ from .instance import (
 from .licenses import LicensesEndpoint
 from .main_alarms import MainAlarmsEndpoint
 from .permission import PermissionsViewRoleEndpoint, PermissionsViewRoleDetailEndpoint
+from .reports import ReportEndpoint, ReportDetailsEndpoint
 from .roles import RolesListEndpoint, RoleDetailEndpoint
 from .schemas import SchemaDetailsEndpoint, SchemaEndpoint
 from .tables import TablesEndpoint, TablesDetailsEndpoint
@@ -216,6 +219,17 @@ resources = [
         urls="/table/<string:table_name>/<string:idx>/",
         endpoint="tables-detail",
     ),
+    dict(
+        resource=ReportDetailsEndpoint,
+        urls="/report/<int:idx>/",
+        endpoint="report-detail",
+    ),
+    dict(
+        resource=ReportDetailsEditEndpoint,
+        urls="/report/<int:idx>/edit/",
+        endpoint="report-detail-edit",
+    ),
+    dict(resource=ReportEndpoint, urls="/report/", endpoint="report"),
 ]
 
 

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/endpoints/execution.py

@@ -1,7 +1,7 @@
 """
 External endpoints to manage the executions: create new ones, list all of them, get one in particular
 or check the status of an ongoing one
-These endpoints hve different access url, but manage the same data entities
+These endpoints have different access url, but manage the same data entities
 """
 
 # Import from libraries

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/endpoints/login.py

@@ -34,6 +34,7 @@ class LoginBaseEndpoint(BaseMetaResource):
     """
     Base endpoint to perform a login action from a user
     """
+
     def __init__(self):
         super().__init__()
         self.ldap_class = LDAPBase
@@ -63,6 +64,7 @@ class LoginBaseEndpoint(BaseMetaResource):
         try:
             token = self.auth_class.generate_token(user.id)
         except Exception as e:
+            current_app.logger.error(f"Error in generating user token: {str(e)}")
             raise InvalidUsage(f"Error in generating user token: {str(e)}", 400)
 
         response.update({"token": token, "id": user.id})
@@ -81,9 +83,11 @@ class LoginBaseEndpoint(BaseMetaResource):
         user = self.data_model.get_one_object(username=username)
 
         if not user:
+            current_app.logger.error(f"Error on login user does not exist")
             raise InvalidCredentials()
 
         if not user.check_hash(password):
+            current_app.logger.error(f"Error on login invalid credentials")
             raise InvalidCredentials()
 
         return user
@@ -102,7 +106,9 @@ class LoginBaseEndpoint(BaseMetaResource):
             raise InvalidCredentials()
         user = self.data_model.get_one_object(username=username)
         if not user:
-            current_app.logger.info(f"LDAP user {username} does not exist and is created")
+            current_app.logger.info(
+                f"LDAP user {username} does not exist and is created"
+            )
             email = ldap_obj.get_user_email(username)
             if not email:
                 email = ""
@@ -122,10 +128,14 @@ class LoginBaseEndpoint(BaseMetaResource):
 
         except IntegrityError as e:
             db.session.rollback()
-            current_app.logger.error(f"Integrity error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Integrity error on user role assignment on log in: {e}"
+            )
         except DBAPIError as e:
             db.session.rollback()
-            current_app.logger.error(f"Unknown error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Unknown error on user role assignment on log in: {e}"
+            )
 
         return user
 
@@ -163,7 +173,9 @@ class LoginBaseEndpoint(BaseMetaResource):
         user = self.data_model.get_one_object(username=username)
 
         if not user:
-            current_app.logger.info(f"OpenID user {username} does not exist and is created")
+            current_app.logger.info(
+                f"OpenID user {username} does not exist and is created"
+            )
 
             data = {"username": username, "email": username}
 
@@ -183,7 +195,11 @@ class LoginBaseEndpoint(BaseMetaResource):
 
 def check_last_password_change(user):
     if user.pwd_last_change:
-        if user.pwd_last_change + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"])) < datetime.utcnow():
+        if (
+            user.pwd_last_change
+            + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"]))
+            < datetime.utcnow()
+        ):
             return True
     return False
 
@@ -210,7 +226,11 @@ class LoginEndpoint(LoginBaseEndpoint):
         :rtype: Tuple(dict, integer)
         """
 
-        return self.log_in(**kwargs)
+        try:
+            return self.log_in(**kwargs)
+        except Exception as e:
+            current_app.logger.error(f"Final exception: {str(e)}")
+            raise e
 
 
 class LoginOpenAuthEndpoint(LoginBaseEndpoint):

cornflow-1.1.1a1/cornflow/endpoints/reports.py

@@ -0,0 +1,283 @@
+"""
+External endpoints to manage the reports: create new ones, list all of them, get one in particular
+These endpoints have different access url, but manage the same data entities
+"""
+import os
+
+from flask import current_app, request, send_from_directory
+from flask_apispec import marshal_with, use_kwargs, doc
+from werkzeug.utils import secure_filename
+import uuid
+
+from cornflow.endpoints.meta_resource import BaseMetaResource
+from cornflow.models import ExecutionModel, ReportModel
+from cornflow.schemas.reports import (
+    ReportSchema,
+    ReportEditRequest,
+    QueryFiltersReports,
+    ReportRequest,
+)
+from cornflow.shared.authentication import Auth, authenticate
+from cornflow.shared.const import SERVICE_ROLE
+from cornflow.shared.exceptions import (
+    FileError,
+    ObjectDoesNotExist,
+    NoPermission,
+)
+
+
+class ReportEndpoint(BaseMetaResource):
+    """
+    Endpoint used to create a new report or get all the reports and their information back
+    """
+
+    ROLES_WITH_ACCESS = [SERVICE_ROLE]
+
+    def __init__(self):
+        super().__init__()
+        self.model = ReportModel
+        self.data_model = ReportModel
+        self.foreign_data = {"execution_id": ExecutionModel}
+
+    @doc(description="Get all reports", tags=["Reports"])
+    @authenticate(auth_class=Auth())
+    @marshal_with(ReportSchema(many=True))
+    @use_kwargs(QueryFiltersReports, location="query")
+    def get(self, **kwargs):
+        """
+        API method to get all the reports created by the user and its related info
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user
+
+        :return: A dictionary with a message (error if authentication failed or a list with all the reports
+          created by the authenticated user) and a integer with the HTTP status code
+        :rtype: Tuple(dict, integer)
+        """
+        reports = self.get_list(user=self.get_user(), **kwargs)
+        current_app.logger.info(f"User {self.get_user()} gets list of reports")
+        return reports
+
+    @doc(description="Create a report", tags=["Reports"])
+    @authenticate(auth_class=Auth())
+    @use_kwargs(ReportRequest, location="form")
+    @marshal_with(ReportSchema)
+    def post(self, **kwargs):
+        """
+        API method to create a new report linked to an existing execution
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user
+
+        :return: A dictionary with a message (error if authentication failed, error if data is not validated or
+          the reference_id for the newly created report if successful) and a integer with the HTTP status code
+        :rtype: Tuple(dict, integer)
+        """
+
+        execution = ExecutionModel.get_one_object(idx=kwargs["execution_id"])
+
+        if execution is None:
+            raise ObjectDoesNotExist("The execution does not exist")
+        if "file" not in request.files:
+            # we're creating an empty report.
+            # which is possible
+            report = ReportModel(get_report_info(kwargs, execution, None))
+
+            report.save()
+            return report, 201
+
+        file = request.files["file"]
+        report_name = new_file_name(file)
+
+        report = ReportModel(get_report_info(kwargs, execution, report_name))
+
+        report.save()
+
+        # We try to save the file, if an error is raised then we delete the record on the database
+        try:
+            write_file(file, execution.id, report_name)
+            return report, 201
+
+        except Exception as error:
+            report.delete()
+            current_app.logger.error(error)
+            raise FileError(error=str(error))
+
+
+class ReportDetailsEndpointBase(BaseMetaResource):
+    """
+    Endpoint used to get the information of a certain report. But not the data!
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.data_model = ReportModel
+        self.foreign_data = {"execution_id": ExecutionModel}
+
+
+class ReportDetailsEditEndpoint(ReportDetailsEndpointBase):
+
+    ROLES_WITH_ACCESS = [SERVICE_ROLE]
+
+    @doc(description="Edit a report", tags=["Reports"], inherit=False)
+    @authenticate(auth_class=Auth())
+    @use_kwargs(ReportEditRequest, location="form")
+    def put(self, idx, **data):
+        """
+        Edit an existing report
+
+        :param string idx: ID of the report.
+        :return: A dictionary with a message (error if authentication failed, or the report does not exist or
+          a message) and an integer with the HTTP status code.
+        :rtype: Tuple(dict, integer)
+        """
+        # TODO: forbid non-service users from running put
+        current_app.logger.info(f"User {self.get_user()} edits report {idx}")
+
+        report = self.get_detail(idx=idx)
+
+        if "file" not in request.files:
+            # we're creating an empty report.
+            # which is possible
+            report.update(data)
+            report.save()
+            return {"message": "Updated correctly"}, 200
+
+        # there's two cases,
+        # (1) the report already has a file
+        # (2) the report doesn't yet have a file
+        file = request.files["file"]
+        report_name = new_file_name(file)
+        old_name = report.file_url
+        # we update the report with the new content, including the new name
+        report.update(dict(**data, file_url=report_name))
+
+        # We try to save the file, if an error is raised then we delete the record on the database
+        try:
+            write_file(file, report.execution_id, report_name)
+            report.save()
+
+        except Exception as error:
+            # we do not save the report
+            current_app.logger.error(error)
+            raise FileError(error=str(error))
+
+        # if it saves correctly, we delete the old file, if exists
+        # if unsuccessful, we still return 201 but log the error
+        if old_name is not None:
+            try:
+                os.remove(get_report_path(report))
+            except OSError as error:
+                current_app.logger.error(error)
+        return {"message": "Updated correctly"}, 200
+
+
+class ReportDetailsEndpoint(ReportDetailsEndpointBase):
+    @doc(description="Get details of a report", tags=["Reports"], inherit=False)
+    @authenticate(auth_class=Auth())
+    @marshal_with(ReportSchema)
+    @BaseMetaResource.get_data_or_404
+    def get(self, idx):
+        """
+        API method to get a report created by the user and its related info.
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user.
+
+        :param str idx: ID of the report.
+        :return: A dictionary with a message (error if authentication failed, or the report does not exist or
+          the data of the report) and an integer with the HTTP status code.
+        :rtype: Tuple(dict, integer)
+        """
+        # TODO: are we able to download the name in the database and not as part of the file?
+        current_app.logger.info(f"User {self.get_user()} gets details of report {idx}")
+        report = self.get_detail(user=self.get_user(), idx=idx)
+
+        if report is None:
+            raise ObjectDoesNotExist
+
+        # if there's no file, we do not return it:
+        if report.file_url is None:
+            return report, 200
+
+        my_dir = get_report_dir(report.execution_id)
+        response = send_from_directory(my_dir, report.file_url)
+        response.headers["File-Description"] = report.description
+        response.headers["File-Name"] = report.file_url
+        return response
+
+    @doc(description="Delete a report", tags=["Reports"], inherit=False)
+    @authenticate(auth_class=Auth())
+    def delete(self, idx):
+        """
+        API method to delete a report created by the user and its related info.
+        It requires authentication to be passed in the form of a token that has to be linked to
+        an existing session (login) made by a user.
+
+        :param string idx: ID of the report.
+        :return: A dictionary with a message (error if authentication failed, or the report does not exist or
+          a message) and an integer with the HTTP status code.
+        :rtype: Tuple(dict, integer)
+        """
+
+        # get report objet
+        report = self.get_detail(user_id=self.get_user_id(), idx=idx)
+
+        if report is None:
+            raise ObjectDoesNotExist
+
+        # delete file
+        os.remove(get_report_path(report))
+
+        return self.delete_detail(user_id=self.get_user_id(), idx=idx)
+
+
+def get_report_dir(execution_id):
+    return f"{current_app.config['UPLOAD_FOLDER']}/{execution_id}"
+
+
+def get_report_path(report):
+    try:
+        return f"{get_report_dir(report['execution_id'])}/{report['file_url']}"
+    except:
+        return f"{get_report_dir(report.execution_id)}/{report.file_url}"
+
+
+def new_file_name(file):
+
+    filename = secure_filename(file.filename)
+    filename_extension = filename.split(".")[-1]
+
+    if filename_extension not in current_app.config["ALLOWED_EXTENSIONS"]:
+        return {
+            "message": f"Invalid file extension. "
+            f"Valid extensions are: {current_app.config['ALLOWED_EXTENSIONS']}"
+        }, 400
+
+    report_name = f"{uuid.uuid4().hex}.{filename_extension}"
+
+    return report_name
+
+
+def write_file(file, execution_id, file_name):
+    my_directory = get_report_dir(execution_id)
+
+    # we create a directory for the execution
+    if not os.path.exists(my_directory):
+        current_app.logger.info(f"Creating directory {my_directory}")
+        os.mkdir(my_directory)
+
+    save_path = os.path.normpath(os.path.join(my_directory, file_name))
+
+    if "static" not in save_path or ".." in save_path:
+        raise NoPermission("Invalid file name")
+    file.save(save_path)
+
+
+def get_report_info(data, execution, file_url=None):
+    return {
+        "name": data["name"],
+        "file_url": file_url,
+        "execution_id": execution.id,
+        "user_id": execution.user_id,
+        "description": data.get("description", ""),
+        "state": data.get("state"),
+        "state_message": data.get("state_message"),
+    }

cornflow-1.1.1a1/cornflow/migrations/versions/83164be03c23_.py

@@ -0,0 +1,40 @@
+"""empty message
+
+Revision ID: 83164be03c23
+Revises: 96f00d0961d1
+Create Date: 2024-07-23 13:18:47.748324
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '83164be03c23'
+down_revision = '96f00d0961d1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('reports', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('state', sa.SmallInteger(), nullable=True))
+        batch_op.add_column(sa.Column('state_message', sa.TEXT(), nullable=True))
+        batch_op.alter_column('file_url',
+               existing_type=sa.VARCHAR(length=256),
+               nullable=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('reports', schema=None) as batch_op:
+        batch_op.alter_column('file_url',
+               existing_type=sa.VARCHAR(length=256),
+               nullable=False)
+        batch_op.drop_column('state_message')
+        batch_op.drop_column('state')
+
+    # ### end Alembic commands ###

cornflow-1.1.1a1/cornflow/migrations/versions/96f00d0961d1_reports_table.py

@@ -0,0 +1,50 @@
+"""
+Adds reports table to database
+
+Revision ID: 96f00d0961d1
+Revises: 991b98e24225
+Create Date: 2024-06-12 18:47:06.366487
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "96f00d0961d1"
+down_revision = "991b98e24225"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "reports",
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), nullable=False),
+        sa.Column("deleted_at", sa.DateTime(), nullable=True),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("execution_id", sa.String(length=256), nullable=False),
+        sa.Column("name", sa.String(length=256), nullable=False),
+        sa.Column("description", sa.TEXT(), nullable=True),
+        sa.Column("file_url", sa.String(length=256), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["execution_id"],
+            ["executions.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("reports")
+    # ### end Alembic commands ###

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/models/__init__.py

@@ -1,6 +1,7 @@
 """
 Initialization file for the models module
 """
+
 from .action import ActionModel
 from .alarms import AlarmsModel
 from .case import CaseModel
@@ -14,3 +15,4 @@ from .role import RoleModel
 from .user import UserModel
 from .user_role import UserRoleModel
 from .view import ViewModel
+from .reports import ReportModel

{cornflow-1.1.0a2 → cornflow-1.1.1a1}/cornflow/models/execution.py

@@ -64,6 +64,14 @@ class ExecutionModel(BaseDataModel):
         default=EXECUTION_STATE_MESSAGE_DICT[DEFAULT_EXECUTION_CODE],
         nullable=True,
     )
+    reports = db.relationship(
+        "ReportModel",
+        backref="executions",
+        lazy=True,
+        primaryjoin="and_(ExecutionModel.id==ReportModel.execution_id, "
+        "ReportModel.deleted_at==None)",
+        cascade="all,delete",
+    )
 
     def __init__(self, data):
         super().__init__(data)