cornflow 1.0.11a1__py3-none-any.whl → 1.1.0a1__py3-none-any.whl

cornflow/cli/service.py

@@ -14,6 +14,7 @@ from cornflow.commands import (
     register_deployed_dags_command,
     register_dag_permissions_command,
     update_schemas_command,
+    update_dag_registry_command,
 )
 from cornflow.shared.const import AUTH_DB, ADMIN_ROLE, SERVICE_ROLE
 from cornflow.shared import db
@@ -211,6 +212,9 @@ def init_cornflow_service():
             )
             register_dag_permissions_command(open_deployment, verbose=True)
             update_schemas_command(airflow_url, airflow_user, airflow_pwd, verbose=True)
+            update_dag_registry_command(
+                airflow_url, airflow_user, airflow_pwd, verbose=True
+            )
 
         os.system(
             f"/usr/local/bin/gunicorn -c python:cornflow.gunicorn "

cornflow/commands/__init__.py

@@ -6,7 +6,7 @@ from .permissions import (
     register_dag_permissions_command,
 )
 from .roles import register_roles_command
-from .schemas import update_schemas_command
+from .schemas import update_schemas_command, update_dag_registry_command
 from .users import (
     create_user_with_role,
     create_service_user_command,

cornflow/commands/schemas.py

@@ -27,3 +27,34 @@ def update_schemas_command(url, user, pwd, verbose: bool = False):
             current_app.logger.info("The DAGs schemas were not updated properly")
 
     return True
+
+
+def update_dag_registry_command(url, user, pwd, verbose: bool = False):
+    import time
+    from flask import current_app
+
+    from cornflow_client.airflow.api import Airflow
+
+    af_client = Airflow(url, user, pwd)
+    max_attempts = 20
+    attempts = 0
+    while not af_client.is_alive() and attempts < max_attempts:
+        attempts += 1
+        if verbose == 1:
+            current_app.logger.info(f"Airflow is not reachable (attempt {attempts})")
+        time.sleep(15)
+
+    if not af_client.is_alive():
+        if verbose == 1:
+            current_app.logger.info("Airflow is not reachable")
+        return False
+
+    response = af_client.update_dag_registry()
+    if response.status_code == 200:
+        if verbose:
+            current_app.logger.info("DAGs schemas updated on cornflow")
+    else:
+        if verbose:
+            current_app.logger.info("The DAGs schemas were not updated properly")
+
+    return True

cornflow/config.py

@@ -76,6 +76,12 @@ class DefaultConfig(object):
     # Alarms endpoints
     ALARMS_ENDPOINTS = os.getenv("CF_ALARMS_ENDPOINT", 0)
 
+    # Token duration in hours
+    TOKEN_DURATION = os.getenv("TOKEN_DURATION", 24)
+
+    # Password rotation time in days
+    PWD_ROTATION_TIME = os.getenv("PWD_ROTATION_TIME", 120)
+
 
 class Development(DefaultConfig):
 

cornflow/endpoints/execution.py

@@ -24,6 +24,7 @@ from cornflow.schemas.execution import (
     ExecutionEditRequest,
     QueryFiltersExecution,
     ReLaunchExecutionRequest,
+    ExecutionDetailsWithIndicatorsAndLogResponse
 )
 from cornflow.shared.authentication import Auth, authenticate
 from cornflow.shared.compress import compressed
@@ -58,7 +59,7 @@ class ExecutionEndpoint(BaseMetaResource):
 
     @doc(description="Get all executions", tags=["Executions"])
     @authenticate(auth_class=Auth())
-    @marshal_with(ExecutionDetailsEndpointWithIndicatorsResponse(many=True))
+    @marshal_with(ExecutionDetailsWithIndicatorsAndLogResponse(many=True))
     @use_kwargs(QueryFiltersExecution, location="query")
     def get(self, **kwargs):
         """

cornflow/endpoints/login.py

@@ -6,10 +6,11 @@ External endpoint for the user to login to the cornflow webserver
 from flask import current_app
 from flask_apispec import use_kwargs, doc
 from sqlalchemy.exc import IntegrityError, DBAPIError
+from datetime import datetime, timedelta
 
 # Import from internal modules
 from cornflow.endpoints.meta_resource import BaseMetaResource
-from cornflow.models import PermissionsDAG, UserModel, UserRoleModel
+from cornflow.models import UserModel, UserRoleModel
 from cornflow.schemas.user import LoginEndpointRequest, LoginOpenAuthRequest
 from cornflow.shared import db
 from cornflow.shared.authentication import Auth, LDAPBase
@@ -47,9 +48,11 @@ class LoginBaseEndpoint(BaseMetaResource):
         :rtype: dict
         """
         auth_type = current_app.config["AUTH_TYPE"]
+        response = {}
 
         if auth_type == AUTH_DB:
             user = self.auth_db_authenticate(**kwargs)
+            response.update({"change_password": check_last_password_change(user)})
         elif auth_type == AUTH_LDAP:
             user = self.auth_ldap_authenticate(**kwargs)
         elif auth_type == AUTH_OID:
@@ -62,7 +65,9 @@ class LoginBaseEndpoint(BaseMetaResource):
         except Exception as e:
             raise InvalidUsage(f"Error in generating user token: {str(e)}", 400)
 
-        return {"token": token, "id": user.id}, 200
+        response.update({"token": token, "id": user.id})
+
+        return response, 200
 
     def auth_db_authenticate(self, username, password):
         """
@@ -176,6 +181,13 @@ class LoginBaseEndpoint(BaseMetaResource):
         return user
 
 
+def check_last_password_change(user):
+    if user.pwd_last_change:
+        if user.pwd_last_change + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"])) < datetime.utcnow():
+            return True
+    return False
+
+
 class LoginEndpoint(LoginBaseEndpoint):
     """
     Endpoint used to do the login to the cornflow webserver
@@ -198,11 +210,7 @@ class LoginEndpoint(LoginBaseEndpoint):
         :rtype: Tuple(dict, integer)
         """
 
-        content, status = self.log_in(**kwargs)
-        if int(current_app.config["OPEN_DEPLOYMENT"]) == 1:
-            PermissionsDAG.delete_all_permissions_from_user(content["id"])
-            PermissionsDAG.add_all_permissions_to_user(content["id"])
-        return content, status
+        return self.log_in(**kwargs)
 
 
 class LoginOpenAuthEndpoint(LoginBaseEndpoint):
@@ -218,9 +226,4 @@ class LoginOpenAuthEndpoint(LoginBaseEndpoint):
     @use_kwargs(LoginOpenAuthRequest, location="json")
     def post(self, **kwargs):
         """ """
-
-        content, status = self.log_in(**kwargs)
-        if int(current_app.config["OPEN_DEPLOYMENT"]) == 1:
-            PermissionsDAG.delete_all_permissions_from_user(content["id"])
-            PermissionsDAG.add_all_permissions_to_user(content["id"])
-        return content, status
+        return self.log_in(**kwargs)

cornflow/endpoints/user.py

@@ -170,7 +170,7 @@ class UserDetailsEndpoint(BaseMetaResource):
                 f"To edit a user, go to the OID provider.",
             )
 
-        if data.get("password"):
+        if data.get("password") is not None:
             check, msg = check_password_pattern(data.get("password"))
             if not check:
                 raise InvalidCredentials(
@@ -179,7 +179,7 @@ class UserDetailsEndpoint(BaseMetaResource):
                     f"The new password is not valid.",
                 )
 
-        if data.get("email"):
+        if data.get("email") is not None:
             check, msg = check_email_pattern(data.get("email"))
             if not check:
                 raise InvalidCredentials(

cornflow/migrations/versions/991b98e24225_.py

@@ -0,0 +1,33 @@
+"""
+Added pwd_last_change column to users table
+
+Revision ID: 991b98e24225
+Revises: ebdd955fcc5e
+Create Date: 2024-01-31 19:17:18.009264
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "991b98e24225"
+down_revision = "ebdd955fcc5e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("users", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("pwd_last_change", sa.DateTime(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("users", schema=None) as batch_op:
+        batch_op.drop_column("pwd_last_change")
+
+    # ### end Alembic commands ###

cornflow/models/user.py

@@ -4,6 +4,7 @@ This file contains the UserModel
 # Imports from external libraries
 import random
 import string
+from datetime import datetime
 
 # Imports from internal modules
 from cornflow.models.meta_models import TraceAttributesModel
@@ -51,6 +52,7 @@ class UserModel(TraceAttributesModel):
     last_name = db.Column(db.String(128), nullable=True)
     username = db.Column(db.String(128), nullable=False, unique=True)
     password = db.Column(db.String(128), nullable=True)
+    pwd_last_change = db.Column(db.DateTime, nullable=True)
     email = db.Column(db.String(128), nullable=False, unique=True)
 
     user_roles = db.relationship(
@@ -93,6 +95,7 @@ class UserModel(TraceAttributesModel):
         self.first_name = data.get("first_name")
         self.last_name = data.get("last_name")
         self.username = data.get("username")
+        self.pwd_last_change = datetime.utcnow()
         # TODO: handle better None passwords that can be found when using ldap
         check_pass, msg = check_password_pattern(data.get("password"))
         if check_pass:
@@ -123,6 +126,7 @@ class UserModel(TraceAttributesModel):
         if new_password:
             new_password = self.__generate_hash(new_password)
             data["password"] = new_password
+            data["pwd_last_change"] = datetime.utcnow()
         super().update(data)
 
     def comes_from_external_provider(self):

cornflow/schemas/execution.py

@@ -4,7 +4,7 @@ from marshmallow import fields, Schema, validate
 # Imports from internal modules
 from cornflow.shared.const import MIN_EXECUTION_STATUS_CODE, MAX_EXECUTION_STATUS_CODE
 from .common import QueryFilters, BaseDataEndpointResponse
-from .solution_log import LogSchema
+from .solution_log import LogSchema, BasicLogSchema
 
 
 class QueryFiltersExecution(QueryFilters):
@@ -114,6 +114,12 @@ class ExecutionDetailsEndpointWithIndicatorsResponse(ExecutionDetailsEndpointRes
     indicators = fields.Method("get_indicators")
 
 
+class ExecutionDetailsWithIndicatorsAndLogResponse(
+    ExecutionDetailsEndpointWithIndicatorsResponse
+):
+    log = fields.Nested(BasicLogSchema, attribute="log_json")
+
+
 class ExecutionStatusEndpointResponse(Schema):
     id = fields.Str()
     state = fields.Int()
@@ -129,6 +135,7 @@ class ExecutionStatusEndpointUpdate(Schema):
 class ExecutionDataEndpointResponse(ExecutionDetailsEndpointResponse):
     data = fields.Raw()
     checks = fields.Raw()
+    log = fields.Nested(BasicLogSchema, attribute="log_json")
 
 
 class ExecutionLogEndpointResponse(ExecutionDetailsEndpointWithIndicatorsResponse):

cornflow/schemas/solution_log.py

@@ -1,4 +1,4 @@
-from marshmallow import fields, Schema
+from marshmallow import fields, Schema, EXCLUDE
 
 options = dict(required=True, allow_none=True)
 log_options = dict(required=False, allow_none=True)
@@ -49,10 +49,18 @@ class FirstSolution(Schema):
     CutsBestBound = fields.Float(**options)
 
 
-class LogSchema(Schema):
+class BasicLogSchema(Schema):
+    status = fields.Str(**log_options)
+    status_code = fields.Int(**log_options)
+    sol_code = fields.Int(**log_options)
+
+
+class LogSchema(BasicLogSchema):
+    class Meta:
+        unknown = EXCLUDE
+
     version = fields.Str(**log_options)
     solver = fields.Str(**log_options)
-    status = fields.Str(**log_options)
     best_bound = fields.Float(**log_options)
     best_solution = fields.Float(**log_options)
     gap = fields.Float(**log_options)
@@ -63,8 +71,6 @@ class LogSchema(Schema):
     presolve = fields.Nested(PresolveSchema, **log_options)
     first_relaxed = fields.Float(**log_options)
     first_solution = fields.Nested(FirstSolution, **log_options)
-    status_code = fields.Int(**log_options)
-    sol_code = fields.Int(**log_options)
     nodes = fields.Int(**log_options)
     progress = fields.Nested(ProgressSchema, required=False)
     cut_info = fields.Raw(**log_options)

cornflow/schemas/user.py

@@ -25,6 +25,7 @@ class UserEndpointResponse(Schema):
     last_name = fields.Str()
     email = fields.Str()
     created_at = fields.Str()
+    pwd_last_change = fields.Str()
 
 
 class UserDetailsEndpointResponse(Schema):
@@ -33,6 +34,7 @@ class UserDetailsEndpointResponse(Schema):
     last_name = fields.Str()
     username = fields.Str()
     email = fields.Str()
+    pwd_last_change = fields.Str()
 
 
 class TokenEndpointResponse(Schema):
@@ -49,6 +51,7 @@ class UserEditRequest(Schema):
     last_name = fields.Str(required=False)
     email = fields.Str(required=False)
     password = fields.Str(required=False)
+    pwd_last_change = fields.DateTime(required=False)
 
 
 class LoginEndpointRequest(Schema):

cornflow/shared/authentication/auth.py

@@ -103,7 +103,7 @@ class Auth:
             )
 
         payload = {
-            "exp": datetime.utcnow() + timedelta(days=1),
+            "exp": datetime.utcnow() + timedelta(hours=float(current_app.config["TOKEN_DURATION"])),
             "iat": datetime.utcnow(),
             "sub": user_id,
         }

cornflow/shared/licenses.py

@@ -1,63 +1,21 @@
-import pkg_resources
+import importlib.metadata as metadata
 
 
-def get_license_txt(pkg):
-    if pkg.has_metadata("LICENSE"):
-        lic = pkg.get_metadata("LICENSE")
-    else:
-        lic = "(license detail not found)"
-    return lic
-
-
-def get_info(name, lines):
+def get_info(name, pkg):
     """
-    Search information in a list of lines.
+    Search information in a package metadata.
     The expected format of the line is "name: info"
-    This function search the name and return the info.
+    This function searches for the name and returns the info.
 
-    :param name: name to be search at the beginning of a line.
-    :param lines: list of strings.
+    :param name: name to be searched.
+    :param pkg: a dictionary representing the package metadata.
     :return: the info part of the line for the given name.
     """
-    sep = name + ": "
-    for line in lines:
-        if line.startswith(sep):
-            return line.split(sep, maxsplit=1)[1]
+    if name in pkg:
+        return pkg[name]
     return f"({name} not found)"
 
 
-def get_main_info(pkg):
-    """
-    Get information from libraries.
-
-    :param pkg: a package object from pkg_resources.working_set
-    :return: a dict with library, license, version, author, description and home page.
-    """
-    lines1 = []
-    lines2 = []
-    # Find info in metadata
-    if pkg.has_metadata("METADATA"):
-        lines1 = pkg.get_metadata_lines("METADATA")
-    # find info in PKG-INFO
-    if pkg.has_metadata("PKG-INFO"):
-        lines2 = pkg.get_metadata_lines("PKG-INFO")
-    # Transform lines into list
-    lines = [l for l in lines1] + [l for l in lines2]
-
-    # Manage case where license is UNKNOWN
-    lic = get_info("License", lines)
-    if lic == "UNKNOWN":
-        lic = get_info("Classifier: License :", lines)
-    return {
-        "library": get_info("Name", lines),
-        "license": lic,
-        "version": get_info("Version", lines),
-        "author": get_info("Author", lines),
-        "description": get_info("Summary", lines),
-        "home page": get_info("Home-page", lines),
-    }
-
-
 def get_licenses_summary():
     """
     Get a list of dicts with licenses and library information.
@@ -65,12 +23,17 @@ def get_licenses_summary():
     :return: a list of dicts with library, license, version, author, description, home page and license text.
     """
     license_list = []
-    # TODO: pkg_resources.working_set is deprecated, find a better way to get the list of packages
-    for pkg in sorted(pkg_resources.working_set, key=lambda x: str(x).lower()):
+    for pkg in sorted(metadata.distributions(), key=lambda x: x.metadata['Name'].lower()):
+        pkg_metadata = dict(pkg.metadata.items())
         license_list += [
             {
-                **get_main_info(pkg),
-                "license_text": get_license_txt(pkg),
+                "library": get_info("Name", pkg_metadata),
+                "license": get_info("License", pkg_metadata),
+                "version": get_info("Version", pkg_metadata),
+                "author": get_info("Author", pkg_metadata),
+                "description": get_info("Summary", pkg_metadata),
+                "home page": get_info("Home-page", pkg_metadata),
             }
         ]
+
     return license_list

cornflow/tests/custom_test_case.py

@@ -5,6 +5,9 @@ This file contains the different custom test classes used to generalize the unit
 # Import from libraries
 import logging as log
 from datetime import datetime, timedelta
+
+from typing import List
+
 from flask import current_app
 from flask_testing import TestCase
 import json
@@ -27,7 +30,6 @@ from cornflow.tests.const import (
     TOKEN_URL,
 )
 
-
 try:
     date_from_str = datetime.fromisoformat
 except:
@@ -172,7 +174,9 @@ class CustomTestCase(TestCase):
                 self.assertEqual(getattr(row, key), payload[key])
         return row.id
 
-    def get_rows(self, url, data, token=None, check_data=True):
+    def get_rows(
+        self, url, data, token=None, check_data=True, keys_to_check: List[str] = None
+    ):
         token = token or self.token
 
         codes = [
@@ -187,6 +191,8 @@ class CustomTestCase(TestCase):
         if check_data:
             for i in range(len(data)):
                 self.assertEqual(rows_data[i]["id"], codes[i])
+                if keys_to_check:
+                    self.assertCountEqual(list(rows_data[i].keys()), keys_to_check)
                 for key in self.get_keys_to_check(data[i]):
                     self.assertIn(key, rows_data[i])
                     if key in data[i]:
@@ -199,7 +205,13 @@ class CustomTestCase(TestCase):
         return payload.keys()
 
     def get_one_row(
-        self, url, payload, expected_status=200, check_payload=True, token=None
+        self,
+        url,
+        payload,
+        expected_status=200,
+        check_payload=True,
+        token=None,
+        keys_to_check: List[str] = None,
     ):
         token = token or self.token
 
@@ -210,6 +222,8 @@ class CustomTestCase(TestCase):
         self.assertEqual(expected_status, row.status_code)
         if not check_payload:
             return row.json
+        if keys_to_check:
+            self.assertCountEqual(list(row.json.keys()), keys_to_check)
         self.assertEqual(row.json["id"], payload["id"])
         for key in self.get_keys_to_check(payload):
             self.assertIn(key, row.json)

cornflow/tests/integration/test_cornflowclient.py

@@ -35,6 +35,24 @@ class TestCornflowClientBasic(CustomTestCaseLive):
         super().setUp()
         self.items_to_check = ["name", "description"]
 
+    def check_status_evolution(self, execution, end_state=EXEC_STATE_CORRECT):
+        statuses = [execution["state"]]
+        while end_state not in statuses and len(statuses) < 100:
+            time.sleep(1)
+            status = self.client.get_status(execution["id"])
+            statuses.append(status["state"])
+
+        self.assertIn(EXEC_STATE_QUEUED, statuses)
+        self.assertIn(EXEC_STATE_RUNNING, statuses)
+        self.assertIn(end_state, statuses)
+
+        queued_idx = statuses.index(EXEC_STATE_QUEUED)
+        running_idx = statuses.index(EXEC_STATE_RUNNING)
+        end_state_idx = statuses.index(end_state)
+
+        self.assertLess(queued_idx, running_idx)
+        self.assertLess(running_idx, end_state_idx)
+
     def create_new_instance_file(self, mps_file):
         name = "test_instance1"
         description = "description123"
@@ -141,7 +159,6 @@ class TestCornflowClientBasic(CustomTestCaseLive):
 
 
 class TestCornflowClientOpen(TestCornflowClientBasic):
-
     # TODO: user management
     # TODO: infeasible execution
 
@@ -242,7 +259,6 @@ class TestCornflowClientOpen(TestCornflowClientBasic):
         self.client.create_instance(**payload)
 
     def test_new_instance_with_schema_good(self):
-
         payload = load_file(INSTANCE_PATH)
         payload["schema"] = "solve_model_dag"
         self.create_new_instance_payload(payload)
@@ -335,23 +351,13 @@ class TestCornflowClientAdmin(TestCornflowClientBasic):
 
     def test_status_solving(self):
         execution = self.create_instance_and_execution()
-        time.sleep(10)
-        status = self.client.get_status(execution["id"])
-        self.assertEqual(status["state"], EXEC_STATE_CORRECT)
+        self.check_status_evolution(execution, EXEC_STATE_CORRECT)
 
     def test_status_solving_timer(self):
         execution = self.create_timer_instance_and_execution(10)
-        status = self.client.get_status(execution["id"])
-        self.assertEqual(status["state"], EXEC_STATE_QUEUED)
-        time.sleep(5)
-        status = self.client.get_status(execution["id"])
-        self.assertEqual(status["state"], EXEC_STATE_RUNNING)
-        time.sleep(12)
-        status = self.client.get_status(execution["id"])
-        self.assertEqual(status["state"], EXEC_STATE_CORRECT)
+        self.check_status_evolution(execution, EXEC_STATE_CORRECT)
 
     def test_manual_execution(self):
-
         instance_payload = load_file(INSTANCE_PATH)
         one_instance = self.create_new_instance_payload(instance_payload)
         name = "test_execution_name_123"