cornflow 1.0.11a1__py3-none-any.whl → 1.1.0a2__py3-none-any.whl

cornflow/cli/service.py

@@ -14,6 +14,7 @@ from cornflow.commands import (
     register_deployed_dags_command,
     register_dag_permissions_command,
     update_schemas_command,
+    update_dag_registry_command,
 )
 from cornflow.shared.const import AUTH_DB, ADMIN_ROLE, SERVICE_ROLE
 from cornflow.shared import db
@@ -211,6 +212,9 @@ def init_cornflow_service():
             )
             register_dag_permissions_command(open_deployment, verbose=True)
             update_schemas_command(airflow_url, airflow_user, airflow_pwd, verbose=True)
+            update_dag_registry_command(
+                airflow_url, airflow_user, airflow_pwd, verbose=True
+            )
 
         os.system(
             f"/usr/local/bin/gunicorn -c python:cornflow.gunicorn "

cornflow/commands/__init__.py

@@ -6,7 +6,7 @@ from .permissions import (
     register_dag_permissions_command,
 )
 from .roles import register_roles_command
-from .schemas import update_schemas_command
+from .schemas import update_schemas_command, update_dag_registry_command
 from .users import (
     create_user_with_role,
     create_service_user_command,

cornflow/commands/schemas.py

@@ -27,3 +27,34 @@ def update_schemas_command(url, user, pwd, verbose: bool = False):
             current_app.logger.info("The DAGs schemas were not updated properly")
 
     return True
+
+
+def update_dag_registry_command(url, user, pwd, verbose: bool = False):
+    import time
+    from flask import current_app
+
+    from cornflow_client.airflow.api import Airflow
+
+    af_client = Airflow(url, user, pwd)
+    max_attempts = 20
+    attempts = 0
+    while not af_client.is_alive() and attempts < max_attempts:
+        attempts += 1
+        if verbose == 1:
+            current_app.logger.info(f"Airflow is not reachable (attempt {attempts})")
+        time.sleep(15)
+
+    if not af_client.is_alive():
+        if verbose == 1:
+            current_app.logger.info("Airflow is not reachable")
+        return False
+
+    response = af_client.update_dag_registry()
+    if response.status_code == 200:
+        if verbose:
+            current_app.logger.info("DAGs schemas updated on cornflow")
+    else:
+        if verbose:
+            current_app.logger.info("The DAGs schemas were not updated properly")
+
+    return True

cornflow/config.py

@@ -76,6 +76,12 @@ class DefaultConfig(object):
     # Alarms endpoints
     ALARMS_ENDPOINTS = os.getenv("CF_ALARMS_ENDPOINT", 0)
 
+    # Token duration in hours
+    TOKEN_DURATION = os.getenv("TOKEN_DURATION", 24)
+
+    # Password rotation time in days
+    PWD_ROTATION_TIME = os.getenv("PWD_ROTATION_TIME", 120)
+
 
 class Development(DefaultConfig):
 

cornflow/endpoints/__init__.py

@@ -4,8 +4,8 @@ All references to endpoints should be imported from here
 The login resource gets created on app startup as it depends on configuration
 """
 from .action import ActionListEndpoint
+from .alarms import AlarmsEndpoint
 from .apiview import ApiViewListEndpoint
-
 from .case import (
     CaseEndpoint,
     CaseFromInstanceExecutionEndpoint,
@@ -15,7 +15,6 @@ from .case import (
     CaseToInstance,
     CaseCompare,
 )
-
 from .dag import (
     DAGDetailEndpoint,
     DAGEndpointManual,
@@ -24,7 +23,12 @@ from .dag import (
     DeployedDAGEndpoint,
     DeployedDagDetailEndpoint,
 )
-
+from .data_check import (
+    DataCheckExecutionEndpoint,
+    DataCheckInstanceEndpoint,
+    DataCheckCaseEndpoint,
+)
+from .example_data import ExampleDataListEndpoint, ExampleDataDetailEndpoint
 from .execution import (
     ExecutionEndpoint,
     ExecutionDetailsEndpoint,
@@ -33,36 +37,22 @@ from .execution import (
     ExecutionLogEndpoint,
     ExecutionRelaunchEndpoint,
 )
-
 from .health import HealthEndpoint
-
 from .instance import (
     InstanceEndpoint,
     InstanceDetailsEndpoint,
     InstanceFileEndpoint,
     InstanceDataEndpoint,
 )
-
-from .data_check import (
-    DataCheckExecutionEndpoint,
-    DataCheckInstanceEndpoint,
-    DataCheckCaseEndpoint,
-)
 from .licenses import LicensesEndpoint
+from .main_alarms import MainAlarmsEndpoint
 from .permission import PermissionsViewRoleEndpoint, PermissionsViewRoleDetailEndpoint
-
 from .roles import RolesListEndpoint, RoleDetailEndpoint
-
 from .schemas import SchemaDetailsEndpoint, SchemaEndpoint
+from .tables import TablesEndpoint, TablesDetailsEndpoint
 from .token import TokenEndpoint
-from .example_data import ExampleDataDetailsEndpoint
 from .user import UserEndpoint, UserDetailsEndpoint, ToggleUserAdmin, RecoverPassword
 from .user_role import UserRoleListEndpoint, UserRoleDetailEndpoint
-from .alarms import AlarmsEndpoint
-from .main_alarms import MainAlarmsEndpoint
-
-from .tables import TablesEndpoint, TablesDetailsEndpoint
-
 
 resources = [
     dict(resource=InstanceEndpoint, urls="/instance/", endpoint="instance"),
@@ -157,10 +147,15 @@ resources = [
         endpoint="schema-details",
     ),
     dict(
-        resource=ExampleDataDetailsEndpoint,
+        resource=ExampleDataListEndpoint,
         urls="/example/<string:dag_name>/",
         endpoint="example-data",
     ),
+    dict(
+        resource=ExampleDataDetailEndpoint,
+        urls="/example/<string:dag_name>/<string:example_name>/",
+        endpoint="example-data-detail",
+    ),
     dict(resource=HealthEndpoint, urls="/health/", endpoint="health"),
     dict(
         resource=CaseFromInstanceExecutionEndpoint,

cornflow/endpoints/example_data.py

@@ -1,36 +1,78 @@
 """
 Endpoints to get the example data from a DAG
 """
+import json
 
-# Import from libraries
 from cornflow_client.airflow.api import Airflow
 from flask import current_app, request
 from flask_apispec import marshal_with, doc
-import json
 
-# Import from internal modules
 from cornflow.endpoints.meta_resource import BaseMetaResource
 from cornflow.models import PermissionsDAG
-from cornflow.schemas.example_data import ExampleData
+from cornflow.schemas.example_data import ExampleListData, ExampleDetailData
 from cornflow.shared.authentication import Auth, authenticate
 from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE
-from cornflow.shared.exceptions import AirflowError, NoPermission
+from cornflow.shared.exceptions import AirflowError, NoPermission, ObjectDoesNotExist
 
 
-class ExampleDataDetailsEndpoint(BaseMetaResource):
+class ExampleDataListEndpoint(BaseMetaResource):
     """
     Endpoint used to obtain schemas for one app
     """
 
     ROLES_WITH_ACCESS = [VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE]
 
-    @doc(description="Get example data from DAG", tags=["DAG"])
+    @doc(description="Get lsit of example data from DAG", tags=["DAG"])
     @authenticate(auth_class=Auth())
-    @marshal_with(ExampleData)
+    @marshal_with(ExampleListData(many=True))
     def get(self, dag_name):
         """
         API method to get example data for a given dag
 
+        :return: A dictionary with the names and descriptions of available data examples
+            and an integer with the HTTP status code
+        :rtype: Tuple(dict, integer)
+        """
+        user = Auth().get_user_from_header(request.headers)
+        permission = PermissionsDAG.check_if_has_permissions(
+            user_id=user.id, dag_id=dag_name
+        )
+
+        if permission:
+            af_client = Airflow.from_config(current_app.config)
+            if not af_client.is_alive():
+                current_app.logger.error(
+                    "Airflow not accessible when getting data {}".format(dag_name)
+                )
+                raise AirflowError(error="Airflow is not accessible")
+
+            # try airflow and see if dag_name exists
+            af_client.get_dag_info(dag_name)
+
+            current_app.logger.info("User gets example data from {}".format(dag_name))
+
+            variable_name = f"z_{dag_name}_examples"
+            response = af_client.get_one_variable(variable_name)
+
+            return json.loads(response["value"])
+        else:
+            err = "User does not have permission to access this dag."
+            raise NoPermission(
+                error=err,
+                status_code=403,
+                log_txt=f"Error while user {user} tries to get example data for dag {dag_name}. "
+                + err,
+            )
+
+
+class ExampleDataDetailEndpoint(BaseMetaResource):
+    @doc(description="Get example data from DAG", tags=["DAG"])
+    @authenticate(auth_class=Auth())
+    @marshal_with(ExampleDetailData)
+    def get(self, dag_name, example_name):
+        """
+        API method to get one example data for a given dag
+
         :return: A dictionary with a message and a integer with the HTTP status code
         :rtype: Tuple(dict, integer)
         """
@@ -54,15 +96,24 @@ class ExampleDataDetailsEndpoint(BaseMetaResource):
 
             variable_name = f"z_{dag_name}_examples"
             response = af_client.get_one_variable(variable_name)
-            result = dict()
-            result["examples"] = json.loads(response["value"])
-            result["name"] = response["key"]
 
-            return result
+            example = None
+            for item in json.loads(response["value"]):
+                if item["name"] == example_name:
+                    example = item
+                    break
+
+            if example is None:
+                raise ObjectDoesNotExist(
+                    error="The example does not exist", status_code=404
+                )
+
+            return example
         else:
             err = "User does not have permission to access this dag."
             raise NoPermission(
                 error=err,
                 status_code=403,
-                log_txt=f"Error while user {user} tries to get example data for dag {dag_name}. " + err
+                log_txt=f"Error while user {user} tries to get example data for dag {dag_name}. "
+                + err,
             )

cornflow/endpoints/execution.py

@@ -24,6 +24,7 @@ from cornflow.schemas.execution import (
     ExecutionEditRequest,
     QueryFiltersExecution,
     ReLaunchExecutionRequest,
+    ExecutionDetailsWithIndicatorsAndLogResponse
 )
 from cornflow.shared.authentication import Auth, authenticate
 from cornflow.shared.compress import compressed
@@ -58,7 +59,7 @@ class ExecutionEndpoint(BaseMetaResource):
 
     @doc(description="Get all executions", tags=["Executions"])
     @authenticate(auth_class=Auth())
-    @marshal_with(ExecutionDetailsEndpointWithIndicatorsResponse(many=True))
+    @marshal_with(ExecutionDetailsWithIndicatorsAndLogResponse(many=True))
     @use_kwargs(QueryFiltersExecution, location="query")
     def get(self, **kwargs):
         """

cornflow/endpoints/login.py

@@ -6,10 +6,11 @@ External endpoint for the user to login to the cornflow webserver
 from flask import current_app
 from flask_apispec import use_kwargs, doc
 from sqlalchemy.exc import IntegrityError, DBAPIError
+from datetime import datetime, timedelta
 
 # Import from internal modules
 from cornflow.endpoints.meta_resource import BaseMetaResource
-from cornflow.models import PermissionsDAG, UserModel, UserRoleModel
+from cornflow.models import UserModel, UserRoleModel
 from cornflow.schemas.user import LoginEndpointRequest, LoginOpenAuthRequest
 from cornflow.shared import db
 from cornflow.shared.authentication import Auth, LDAPBase
@@ -47,9 +48,11 @@ class LoginBaseEndpoint(BaseMetaResource):
         :rtype: dict
         """
         auth_type = current_app.config["AUTH_TYPE"]
+        response = {}
 
         if auth_type == AUTH_DB:
             user = self.auth_db_authenticate(**kwargs)
+            response.update({"change_password": check_last_password_change(user)})
         elif auth_type == AUTH_LDAP:
             user = self.auth_ldap_authenticate(**kwargs)
         elif auth_type == AUTH_OID:
@@ -62,7 +65,9 @@ class LoginBaseEndpoint(BaseMetaResource):
         except Exception as e:
             raise InvalidUsage(f"Error in generating user token: {str(e)}", 400)
 
-        return {"token": token, "id": user.id}, 200
+        response.update({"token": token, "id": user.id})
+
+        return response, 200
 
     def auth_db_authenticate(self, username, password):
         """
@@ -176,6 +181,13 @@ class LoginBaseEndpoint(BaseMetaResource):
         return user
 
 
+def check_last_password_change(user):
+    if user.pwd_last_change:
+        if user.pwd_last_change + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"])) < datetime.utcnow():
+            return True
+    return False
+
+
 class LoginEndpoint(LoginBaseEndpoint):
     """
     Endpoint used to do the login to the cornflow webserver
@@ -198,11 +210,7 @@ class LoginEndpoint(LoginBaseEndpoint):
         :rtype: Tuple(dict, integer)
         """
 
-        content, status = self.log_in(**kwargs)
-        if int(current_app.config["OPEN_DEPLOYMENT"]) == 1:
-            PermissionsDAG.delete_all_permissions_from_user(content["id"])
-            PermissionsDAG.add_all_permissions_to_user(content["id"])
-        return content, status
+        return self.log_in(**kwargs)
 
 
 class LoginOpenAuthEndpoint(LoginBaseEndpoint):
@@ -218,9 +226,4 @@ class LoginOpenAuthEndpoint(LoginBaseEndpoint):
     @use_kwargs(LoginOpenAuthRequest, location="json")
     def post(self, **kwargs):
         """ """
-
-        content, status = self.log_in(**kwargs)
-        if int(current_app.config["OPEN_DEPLOYMENT"]) == 1:
-            PermissionsDAG.delete_all_permissions_from_user(content["id"])
-            PermissionsDAG.add_all_permissions_to_user(content["id"])
-        return content, status
+        return self.log_in(**kwargs)

cornflow/endpoints/user.py

@@ -170,7 +170,7 @@ class UserDetailsEndpoint(BaseMetaResource):
                 f"To edit a user, go to the OID provider.",
             )
 
-        if data.get("password"):
+        if data.get("password") is not None:
             check, msg = check_password_pattern(data.get("password"))
             if not check:
                 raise InvalidCredentials(
@@ -179,7 +179,7 @@ class UserDetailsEndpoint(BaseMetaResource):
                     f"The new password is not valid.",
                 )
 
-        if data.get("email"):
+        if data.get("email") is not None:
             check, msg = check_email_pattern(data.get("email"))
             if not check:
                 raise InvalidCredentials(

cornflow/migrations/versions/991b98e24225_.py

@@ -0,0 +1,33 @@
+"""
+Added pwd_last_change column to users table
+
+Revision ID: 991b98e24225
+Revises: ebdd955fcc5e
+Create Date: 2024-01-31 19:17:18.009264
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "991b98e24225"
+down_revision = "ebdd955fcc5e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("users", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("pwd_last_change", sa.DateTime(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("users", schema=None) as batch_op:
+        batch_op.drop_column("pwd_last_change")
+
+    # ### end Alembic commands ###

cornflow/models/user.py

@@ -4,6 +4,7 @@ This file contains the UserModel
 # Imports from external libraries
 import random
 import string
+from datetime import datetime
 
 # Imports from internal modules
 from cornflow.models.meta_models import TraceAttributesModel
@@ -51,6 +52,7 @@ class UserModel(TraceAttributesModel):
     last_name = db.Column(db.String(128), nullable=True)
     username = db.Column(db.String(128), nullable=False, unique=True)
     password = db.Column(db.String(128), nullable=True)
+    pwd_last_change = db.Column(db.DateTime, nullable=True)
     email = db.Column(db.String(128), nullable=False, unique=True)
 
     user_roles = db.relationship(
@@ -93,6 +95,7 @@ class UserModel(TraceAttributesModel):
         self.first_name = data.get("first_name")
         self.last_name = data.get("last_name")
         self.username = data.get("username")
+        self.pwd_last_change = datetime.utcnow()
         # TODO: handle better None passwords that can be found when using ldap
         check_pass, msg = check_password_pattern(data.get("password"))
         if check_pass:
@@ -123,6 +126,7 @@ class UserModel(TraceAttributesModel):
         if new_password:
             new_password = self.__generate_hash(new_password)
             data["password"] = new_password
+            data["pwd_last_change"] = datetime.utcnow()
         super().update(data)
 
     def comes_from_external_provider(self):

cornflow/schemas/example_data.py

@@ -1,6 +1,11 @@
 from marshmallow import fields, Schema
 
 
-class ExampleData(Schema):
+class ExampleListData(Schema):
     name = fields.Str(required=True)
-    examples = fields.Raw(required=True)
+    description = fields.Str(required=False)
+
+
+class ExampleDetailData(ExampleListData):
+    instance = fields.Raw(required=True)
+    solution = fields.Raw(required=False)

cornflow/schemas/execution.py

@@ -4,7 +4,7 @@ from marshmallow import fields, Schema, validate
 # Imports from internal modules
 from cornflow.shared.const import MIN_EXECUTION_STATUS_CODE, MAX_EXECUTION_STATUS_CODE
 from .common import QueryFilters, BaseDataEndpointResponse
-from .solution_log import LogSchema
+from .solution_log import LogSchema, BasicLogSchema
 
 
 class QueryFiltersExecution(QueryFilters):
@@ -114,6 +114,12 @@ class ExecutionDetailsEndpointWithIndicatorsResponse(ExecutionDetailsEndpointRes
     indicators = fields.Method("get_indicators")
 
 
+class ExecutionDetailsWithIndicatorsAndLogResponse(
+    ExecutionDetailsEndpointWithIndicatorsResponse
+):
+    log = fields.Nested(BasicLogSchema, attribute="log_json")
+
+
 class ExecutionStatusEndpointResponse(Schema):
     id = fields.Str()
     state = fields.Int()
@@ -129,6 +135,7 @@ class ExecutionStatusEndpointUpdate(Schema):
 class ExecutionDataEndpointResponse(ExecutionDetailsEndpointResponse):
     data = fields.Raw()
     checks = fields.Raw()
+    log = fields.Nested(BasicLogSchema, attribute="log_json")
 
 
 class ExecutionLogEndpointResponse(ExecutionDetailsEndpointWithIndicatorsResponse):

cornflow/schemas/solution_log.py

@@ -1,4 +1,4 @@
-from marshmallow import fields, Schema
+from marshmallow import fields, Schema, EXCLUDE
 
 options = dict(required=True, allow_none=True)
 log_options = dict(required=False, allow_none=True)
@@ -49,10 +49,18 @@ class FirstSolution(Schema):
     CutsBestBound = fields.Float(**options)
 
 
-class LogSchema(Schema):
+class BasicLogSchema(Schema):
+    status = fields.Str(**log_options)
+    status_code = fields.Int(**log_options)
+    sol_code = fields.Int(**log_options)
+
+
+class LogSchema(BasicLogSchema):
+    class Meta:
+        unknown = EXCLUDE
+
     version = fields.Str(**log_options)
     solver = fields.Str(**log_options)
-    status = fields.Str(**log_options)
     best_bound = fields.Float(**log_options)
     best_solution = fields.Float(**log_options)
     gap = fields.Float(**log_options)
@@ -63,8 +71,6 @@ class LogSchema(Schema):
     presolve = fields.Nested(PresolveSchema, **log_options)
     first_relaxed = fields.Float(**log_options)
     first_solution = fields.Nested(FirstSolution, **log_options)
-    status_code = fields.Int(**log_options)
-    sol_code = fields.Int(**log_options)
     nodes = fields.Int(**log_options)
     progress = fields.Nested(ProgressSchema, required=False)
     cut_info = fields.Raw(**log_options)

cornflow/schemas/user.py

@@ -25,6 +25,7 @@ class UserEndpointResponse(Schema):
     last_name = fields.Str()
     email = fields.Str()
     created_at = fields.Str()
+    pwd_last_change = fields.Str()
 
 
 class UserDetailsEndpointResponse(Schema):
@@ -33,6 +34,7 @@ class UserDetailsEndpointResponse(Schema):
     last_name = fields.Str()
     username = fields.Str()
     email = fields.Str()
+    pwd_last_change = fields.Str()
 
 
 class TokenEndpointResponse(Schema):
@@ -49,6 +51,7 @@ class UserEditRequest(Schema):
     last_name = fields.Str(required=False)
     email = fields.Str(required=False)
     password = fields.Str(required=False)
+    pwd_last_change = fields.DateTime(required=False)
 
 
 class LoginEndpointRequest(Schema):

cornflow/shared/authentication/auth.py

@@ -103,7 +103,7 @@ class Auth:
             )
 
         payload = {
-            "exp": datetime.utcnow() + timedelta(days=1),
+            "exp": datetime.utcnow() + timedelta(hours=float(current_app.config["TOKEN_DURATION"])),
             "iat": datetime.utcnow(),
             "sub": user_id,
         }

cornflow/shared/licenses.py

@@ -1,63 +1,21 @@
-import pkg_resources
+import importlib.metadata as metadata
 
 
-def get_license_txt(pkg):
-    if pkg.has_metadata("LICENSE"):
-        lic = pkg.get_metadata("LICENSE")
-    else:
-        lic = "(license detail not found)"
-    return lic
-
-
-def get_info(name, lines):
+def get_info(name, pkg):
     """
-    Search information in a list of lines.
+    Search information in a package metadata.
     The expected format of the line is "name: info"
-    This function search the name and return the info.
+    This function searches for the name and returns the info.
 
-    :param name: name to be search at the beginning of a line.
-    :param lines: list of strings.
+    :param name: name to be searched.
+    :param pkg: a dictionary representing the package metadata.
     :return: the info part of the line for the given name.
     """
-    sep = name + ": "
-    for line in lines:
-        if line.startswith(sep):
-            return line.split(sep, maxsplit=1)[1]
+    if name in pkg:
+        return pkg[name]
     return f"({name} not found)"
 
 
-def get_main_info(pkg):
-    """
-    Get information from libraries.
-
-    :param pkg: a package object from pkg_resources.working_set
-    :return: a dict with library, license, version, author, description and home page.
-    """
-    lines1 = []
-    lines2 = []
-    # Find info in metadata
-    if pkg.has_metadata("METADATA"):
-        lines1 = pkg.get_metadata_lines("METADATA")
-    # find info in PKG-INFO
-    if pkg.has_metadata("PKG-INFO"):
-        lines2 = pkg.get_metadata_lines("PKG-INFO")
-    # Transform lines into list
-    lines = [l for l in lines1] + [l for l in lines2]
-
-    # Manage case where license is UNKNOWN
-    lic = get_info("License", lines)
-    if lic == "UNKNOWN":
-        lic = get_info("Classifier: License :", lines)
-    return {
-        "library": get_info("Name", lines),
-        "license": lic,
-        "version": get_info("Version", lines),
-        "author": get_info("Author", lines),
-        "description": get_info("Summary", lines),
-        "home page": get_info("Home-page", lines),
-    }
-
-
 def get_licenses_summary():
     """
     Get a list of dicts with licenses and library information.
@@ -65,12 +23,17 @@ def get_licenses_summary():
     :return: a list of dicts with library, license, version, author, description, home page and license text.
     """
     license_list = []
-    # TODO: pkg_resources.working_set is deprecated, find a better way to get the list of packages
-    for pkg in sorted(pkg_resources.working_set, key=lambda x: str(x).lower()):
+    for pkg in sorted(metadata.distributions(), key=lambda x: x.metadata['Name'].lower()):
+        pkg_metadata = dict(pkg.metadata.items())
         license_list += [
             {
-                **get_main_info(pkg),
-                "license_text": get_license_txt(pkg),
+                "library": get_info("Name", pkg_metadata),
+                "license": get_info("License", pkg_metadata),
+                "version": get_info("Version", pkg_metadata),
+                "author": get_info("Author", pkg_metadata),
+                "description": get_info("Summary", pkg_metadata),
+                "home page": get_info("Home-page", pkg_metadata),
             }
         ]
+
     return license_list