cornflow 1.1.2__tar.gz → 1.1.5a1__tar.gz

{cornflow-1.1.2 → cornflow-1.1.5a1}/MANIFEST.in

@@ -3,4 +3,5 @@ include MANIFEST.in
 include README.rst
 include setup.py
 include cornflow/migrations/*
-include cornflow/migrations/versions/*
+include cornflow/migrations/versions/*
+include requirements.txt

cornflow-1.1.5a1/PKG-INFO

@@ -0,0 +1,264 @@
+Metadata-Version: 2.2
+Name: cornflow
+Version: 1.1.5a1
+Summary: Cornflow is an open source multi-solver optimization server with a REST API built using flask.
+Home-page: https://github.com/baobabsoluciones/cornflow
+Author: baobab soluciones
+Author-email: cornflow@baobabsoluciones.es
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Requires-Python: >=3.9
+Requires-Dist: alembic==1.9.2
+Requires-Dist: apispec<=6.3.0
+Requires-Dist: click<=8.1.7
+Requires-Dist: cornflow-client>=1.1.0
+Requires-Dist: cryptography<=42.0.5
+Requires-Dist: disposable-email-domains>=0.0.86
+Requires-Dist: Flask==2.3.2
+Requires-Dist: flask-apispec<=0.11.4
+Requires-Dist: Flask-Bcrypt<=1.0.1
+Requires-Dist: Flask-Compress<=1.14
+Requires-Dist: flask-cors>=4.0.2
+Requires-Dist: flask-inflate<=0.3
+Requires-Dist: Flask-Migrate<=4.0.5
+Requires-Dist: Flask-RESTful<=0.3.10
+Requires-Dist: Flask-SQLAlchemy==2.5.1
+Requires-Dist: gevent==23.9.1
+Requires-Dist: greenlet<=2.0.2; python_version < "3.11"
+Requires-Dist: greenlet==3.0.3; python_version >= "3.11"
+Requires-Dist: gunicorn<=22.0.0
+Requires-Dist: jsonpatch<=1.33
+Requires-Dist: ldap3<=2.9.1
+Requires-Dist: marshmallow<=3.20.2
+Requires-Dist: PuLP<=2.9.0
+Requires-Dist: psycopg2<=2.9.9
+Requires-Dist: PyJWT<=2.8.0
+Requires-Dist: pytups>=0.86.2
+Requires-Dist: requests<=2.32.3
+Requires-Dist: SQLAlchemy==1.3.21
+Requires-Dist: webargs<=8.3.0
+Requires-Dist: Werkzeug==3.0.6
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+Cornflow
+=========
+
+.. image:: https://github.com/baobabsoluciones/cornflow/workflows/build/badge.svg?style=svg
+    :target: https://github.com/baobabsoluciones/cornflow/actions
+
+.. image:: https://github.com/baobabsoluciones/cornflow/workflows/docs/badge.svg?style=svg
+    :target: https://github.com/baobabsoluciones/cornflow/actions
+
+.. image:: https://github.com/baobabsoluciones/cornflow/workflows/integration/badge.svg?style=svg
+    :target: https://github.com/baobabsoluciones/cornflow/actions
+
+.. image:: https://img.shields.io/pypi/v/cornflow-client.svg?style=svg
+   :target: https://pypi.python.org/pypi/cornflow-client
+
+.. image:: https://img.shields.io/pypi/pyversions/cornflow-client.svg?style=svg
+   :target: https://pypi.python.org/pypi/cornflow-client
+
+.. image:: https://img.shields.io/badge/License-Apache2.0-blue
+
+Cornflow is an open source multi-solver optimization server with a REST API built using `flask <https://flask.palletsprojects.com>`_, `airflow <https://airflow.apache.org/>`_ and `pulp <https://coin-or.github.io/pulp/>`_.
+
+While most deployment servers are based on the solving technique (MIP, CP, NLP, etc.), Cornflow focuses on the optimization problems themselves. However, it does not impose any constraint on the type of problem and solution method to use.
+
+With Cornflow you can deploy a Traveling Salesman Problem solver next to a Knapsack solver or a Nurse Rostering Problem solver. As long as you describe the input and output data, you can upload any solution method for any problem and then use it with any data you want.
+
+Cornflow helps you formalize your problem by proposing development guidelines. It also provides a range of functionalities around your deployed solution method, namely:
+
+* storage of users, instances, solutions and solution logs.
+* deployment and maintenance of models, solvers and algorithms.
+* scheduling of executions in remote machines.
+* management of said executions: start, monitor, interrupt.
+* centralizing of commercial licenses.
+* scenario storage and comparison.
+* user management, roles and groups.
+
+
+.. contents:: **Table of Contents**
+
+Installation instructions
+-------------------------------
+
+Cornflow is tested with Ubuntu 20.04, python >= 3.8 and git.
+
+Download the Cornflow project and install requirements::
+
+    python3 -m venv venv
+    venv/bin/pip3 install cornflow
+
+initialize the sqlite database::
+
+    source venv/bin/activate
+    export FLASK_APP=cornflow.app
+    export DATABASE_URL=sqlite:///cornflow.db
+    flask db upgrade
+    flask access_init
+    flask create_service_user  -u airflow -e airflow_test@admin.com -p airflow_test_password
+    flask create_admin_user  -u cornflow -e cornflow_admin@admin.com -p cornflow_admin_password
+
+
+activate the virtual environment and run Cornflow::
+
+    source venv/bin/activate
+    export FLASK_APP=cornflow.app
+    export SECRET_KEY=THISNEEDSTOBECHANGED
+    export DATABASE_URL=sqlite:///cornflow.db
+    export AIRFLOW_URL=http://127.0.0.1:8080/
+    export AIRFLOW_USER=airflow_user
+    export AIRFLOW_PWD=airflow_pwd
+    flask run
+
+**Cornflow needs a running installation of Airflow to operate and more configuration**. Check `the installation docs <https://baobabsoluciones.github.io/cornflow/main/install.html>`_ for more details on installing airflow, configuring the application and initializing the database.
+
+Using cornflow to solve a PuLP model
+---------------------------------------
+
+We're going to test the cornflow server by using the `cornflow-client` and the `pulp` python package::
+
+    pip install cornflow-client pulp
+
+Initialize the api client::
+
+    from cornflow_client import CornFlow
+    email = 'some_email@gmail.com'
+    pwd = 'Some_password1'
+    username = 'some_name'
+    client = CornFlow(url="http://127.0.0.1:5000")
+
+Create a user::
+
+    config = dict(username=username, email=email, pwd=pwd)
+    client.sign_up(**config)
+
+Log in::
+
+    client.login(username=username, pwd=pwd)
+
+Prepare an instance::
+
+    import pulp
+    prob = pulp.LpProblem("test_export_dict_MIP", pulp.LpMinimize)
+    x = pulp.LpVariable("x", 0, 4)
+    y = pulp.LpVariable("y", -1, 1)
+    z = pulp.LpVariable("z", 0, None, pulp.LpInteger)
+    prob += x + 4 * y + 9 * z, "obj"
+    prob += x + y <= 5, "c1"
+    prob += x + z >= 10, "c2"
+    prob += -y + z == 7.5, "c3"
+    data = prob.to_dict()
+    insName = 'test_export_dict_MIP'
+    description = 'very small example'
+
+Send instance::
+
+    instance = client.create_instance(data, name=insName, description=description, schema="solve_model_dag",)
+
+Solve an instance::
+
+    config = dict(
+        solver = "PULP_CBC_CMD",
+        timeLimit = 10
+    )
+    execution = client.create_execution(
+        instance['id'], config, name='execution1', description='execution of a very small instance',
+        schema="solve_model_dag",
+    )
+
+Check the status of an execution::
+
+    status = client.get_status(execution["id"])
+    print(status['state'])
+    # 1 means "finished correctly"
+
+Retrieve a solution::
+
+    results = client.get_solution(execution['id'])
+    print(results['data'])
+    # returns a json with the solved pulp object
+    _vars, prob = pulp.LpProblem.from_dict(results['data'])
+
+Retrieve the log of the solver::
+
+    log = client.get_log(execution['id'])
+    print(log['log'])
+    # json format of the solver log
+
+Using cornflow to deploy a solution method
+---------------------------------------------
+
+To deploy a cornflow solution method, the following tasks need to be accomplished:
+
+#. Create an Application for the new problem
+#. Do a PR to a compatible repo linked to a server instance (e.g., like `this one <https://github.com/baobabsoluciones/cornflow>`_).
+
+For more details on each part, check the `deployment guide <https://baobabsoluciones.github.io/cornflow/guides/deploy_solver.html>`_.
+
+Using cornflow to solve a problem
+-------------------------------------------
+
+For this example we only need the cornflow_client package. We will test the graph-coloring demo defined `here <https://github.com/baobabsoluciones/cornflow-dags-public/tree/main/DAG/graph_coloring>`_. We will use the test server to solve it.
+
+Initialize the api client::
+
+    from cornflow_client import CornFlow
+    email = 'readme@gmail.com'
+    pwd = 'some_password'
+    username = 'some_name'
+    client = CornFlow(url="https://devsm.cornflow.baobabsoluciones.app/")
+    client.login(username=username, pwd=pwd)
+
+solve a graph coloring problem and get the solution::
+
+    data = dict(pairs=[dict(n1=0, n2=1), dict(n1=1, n2=2), dict(n1=1, n2=3)])
+    instance = client.create_instance(data, name='gc_4_1', description='very small gc problem', schema="graph_coloring")
+    config = dict()
+    execution = client.create_execution(
+        instance['id'], config, name='gc_4_1_exec', description='execution of very small gc problem',
+        schema="graph_coloring",
+    )
+    status = client.get_status(execution["id"])
+    print(status['state'])
+    solution = client.get_solution(execution["id"])
+    print(solution['data']['assignment'])
+
+
+Running tests and coverage
+------------------------------
+
+Then you have to run the following commands::
+
+    export FLASK_ENV=testing
+
+Finally you can run all the tests with the following command::
+
+    python -m unittest discover -s cornflow.tests
+
+If you want to only run the unit tests (without a local airflow webserver)::
+
+    python -m unittest discover -s cornflow.tests.unit
+
+If you want to only run the integration test with a local airflow webserver::
+
+    python -m unittest discover -s cornflow.tests.integration
+
+After if you want to check the coverage report you need to run::
+
+    coverage run  --source=./cornflow/ -m unittest discover -s=./cornflow/tests/
+    coverage report -m
+
+or to get the html reports::
+
+    coverage html
+

{cornflow-1.1.2 → cornflow-1.1.5a1}/cornflow/app.py

@@ -1,6 +1,7 @@
 """
 Main file with the creation of the app logic
 """
+
 # Full imports
 import os
 import click
@@ -13,6 +14,8 @@ from flask_cors import CORS
 from flask_migrate import Migrate
 from flask_restful import Api
 from logging.config import dictConfig
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from werkzeug.exceptions import NotFound
 
 # Module imports
 from cornflow.commands import (
@@ -112,6 +115,11 @@ def create_app(env_name="development", dataconn=None):
     app.cli.add_command(register_deployed_dags)
     app.cli.add_command(register_dag_permissions)
 
+    if app.config["APPLICATION_ROOT"] != "/" and app.config["EXTERNAL_APP"] == 0:
+        app.wsgi_app = DispatcherMiddleware(
+            NotFound(), {app.config["APPLICATION_ROOT"]: app.wsgi_app}
+        )
+
     return app
 
 

{cornflow-1.1.2 → cornflow-1.1.5a1}/cornflow/config.py

@@ -5,6 +5,12 @@ from apispec.ext.marshmallow import MarshmallowPlugin
 
 
 class DefaultConfig(object):
+    """
+    Default configuration class
+    """
+
+    APPLICATION_ROOT = os.getenv("APPLICATION_ROOT", "/")
+    EXTERNAL_APP = int(os.getenv("EXTERNAL_APP", 0))
     SERVICE_NAME = os.getenv("SERVICE_NAME", "Cornflow")
     SECRET_TOKEN_KEY = os.getenv("SECRET_KEY")
     SECRET_BI_KEY = os.getenv("SECRET_BI_KEY")
@@ -22,6 +28,11 @@ class DefaultConfig(object):
     SIGNUP_ACTIVATED = int(os.getenv("SIGNUP_ACTIVATED", 1))
     CORNFLOW_SERVICE_USER = os.getenv("CORNFLOW_SERVICE_USER", "service_user")
 
+    # If service user is allow to log with username and password
+    SERVICE_USER_ALLOW_PASSWORD_LOGIN = int(
+        os.getenv("SERVICE_USER_ALLOW_PASSWORD_LOGIN", 1)
+    )
+
     # Open deployment (all dags accessible to all users)
     OPEN_DEPLOYMENT = os.getenv("OPEN_DEPLOYMENT", 1)
 
@@ -84,14 +95,17 @@ class DefaultConfig(object):
 
 
 class Development(DefaultConfig):
-
-    """ """
+    """
+    Configuration class for development
+    """
 
     ENV = "development"
 
 
 class Testing(DefaultConfig):
-    """ """
+    """
+    Configuration class for testing
+    """
 
     ENV = "testing"
     SQLALCHEMY_TRACK_MODIFICATIONS = False
@@ -109,8 +123,26 @@ class Testing(DefaultConfig):
     LOG_LEVEL = int(os.getenv("LOG_LEVEL", 10))
 
 
+class TestingOpenAuth(Testing):
+    """
+    Configuration class for testing some edge cases with Open Auth login
+    """
+
+    AUTH_TYPE = 0
+
+
+class TestingApplicationRoot(Testing):
+    """
+    Configuration class for testing with application root
+    """
+
+    APPLICATION_ROOT = "/test"
+
+
 class Production(DefaultConfig):
-    """ """
+    """
+    Configuration class for production
+    """
 
     ENV = "production"
     SQLALCHEMY_TRACK_MODIFICATIONS = False
@@ -121,4 +153,10 @@ class Production(DefaultConfig):
     PROPAGATE_EXCEPTIONS = True
 
 
-app_config = {"development": Development, "testing": Testing, "production": Production}
+app_config = {
+    "development": Development,
+    "testing": Testing,
+    "production": Production,
+    "testing-oauth": TestingOpenAuth,
+    "testing-root": TestingApplicationRoot,
+}

{cornflow-1.1.2 → cornflow-1.1.5a1}/cornflow/endpoints/login.py

@@ -34,9 +34,11 @@ class LoginBaseEndpoint(BaseMetaResource):
     """
     Base endpoint to perform a login action from a user
     """
+
     def __init__(self):
         super().__init__()
         self.ldap_class = LDAPBase
+        self.user_role_association = UserRoleModel
 
     def log_in(self, **kwargs):
         """
@@ -102,7 +104,9 @@ class LoginBaseEndpoint(BaseMetaResource):
             raise InvalidCredentials()
         user = self.data_model.get_one_object(username=username)
         if not user:
-            current_app.logger.info(f"LDAP user {username} does not exist and is created")
+            current_app.logger.info(
+                f"LDAP user {username} does not exist and is created"
+            )
             email = ldap_obj.get_user_email(username)
             if not email:
                 email = ""
@@ -122,68 +126,115 @@ class LoginBaseEndpoint(BaseMetaResource):
 
         except IntegrityError as e:
             db.session.rollback()
-            current_app.logger.error(f"Integrity error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Integrity error on user role assignment on log in: {e}"
+            )
         except DBAPIError as e:
             db.session.rollback()
-            current_app.logger.error(f"Unknown error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Unknown error on user role assignment on log in: {e}"
+            )
 
         return user
 
-    def auth_oid_authenticate(self, token):
+    def auth_oid_authenticate(
+        self, token: str = None, username: str = None, password: str = None
+    ):
         """
-        Method  in charge of performing the log in with the token issued by an Open ID provider
+        Method  in charge of performing the log in with the token issued by an Open ID provider.
+        It has an exception and thus accepts username and password for service users if needed.
 
         :param str token: the token that the user has obtained from the Open ID provider
+        :param str username: the username of the user to log in
+        :param str password: the password of the user to log in
         :return: the user object or it raises an error if it has not been possible to log in
         :rtype: :class:`UserModel`
         """
-        oid_provider = int(current_app.config["OID_PROVIDER"])
 
-        client_id = current_app.config["OID_CLIENT_ID"]
-        tenant_id = current_app.config["OID_TENANT_ID"]
-        issuer = current_app.config["OID_ISSUER"]
+        if token:
 
-        if client_id is None or tenant_id is None or issuer is None:
-            raise ConfigurationError("The OID provider configuration is not valid")
+            oid_provider = int(current_app.config["OID_PROVIDER"])
 
-        if oid_provider == OID_AZURE:
-            decoded_token = self.auth_class().validate_oid_token(
-                token, client_id, tenant_id, issuer, oid_provider
-            )
+            client_id = current_app.config["OID_CLIENT_ID"]
+            tenant_id = current_app.config["OID_TENANT_ID"]
+            issuer = current_app.config["OID_ISSUER"]
 
-        elif oid_provider == OID_GOOGLE:
-            raise EndpointNotImplemented("The selected OID provider is not implemented")
-        elif oid_provider == OID_NONE:
-            raise EndpointNotImplemented("The OID provider configuration is not valid")
-        else:
-            raise EndpointNotImplemented("The OID provider configuration is not valid")
+            if client_id is None or tenant_id is None or issuer is None:
+                raise ConfigurationError("The OID provider configuration is not valid")
 
-        username = decoded_token["preferred_username"]
+            if oid_provider == OID_AZURE:
+                decoded_token = self.auth_class().validate_oid_token(
+                    token, client_id, tenant_id, issuer, oid_provider
+                )
 
-        user = self.data_model.get_one_object(username=username)
+            elif oid_provider == OID_GOOGLE:
+                raise EndpointNotImplemented(
+                    "The selected OID provider is not implemented"
+                )
+            elif oid_provider == OID_NONE:
+                raise EndpointNotImplemented(
+                    "The OID provider configuration is not valid"
+                )
+            else:
+                raise EndpointNotImplemented(
+                    "The OID provider configuration is not valid"
+                )
 
-        if not user:
-            current_app.logger.info(f"OpenID user {username} does not exist and is created")
+            username = decoded_token["preferred_username"]
+            email = decoded_token.get("email", f"{username}@test.org")
+            first_name = decoded_token.get("given_name", "")
+            last_name = decoded_token.get("family_name", "")
 
-            data = {"username": username, "email": username}
+            user = self.data_model.get_one_object(username=username)
 
-            user = self.data_model(data=data)
-            user.save()
+            if not user:
+                current_app.logger.info(
+                    f"OpenID user {username} does not exist and is created"
+                )
 
-            self.user_role_association(user.id)
+                data = {
+                    "username": username,
+                    "email": email,
+                    "first_name": first_name,
+                    "last_name": last_name,
+                }
 
-            user_role = self.user_role_association(
-                {"user_id": user.id, "role_id": int(current_app.config["DEFAULT_ROLE"])}
-            )
+                user = self.data_model(data=data)
+                user.save()
 
-            user_role.save()
+                user_role = self.user_role_association(
+                    {
+                        "user_id": user.id,
+                        "role_id": int(current_app.config["DEFAULT_ROLE"]),
+                    }
+                )
 
-        return user
+                user_role.save()
+
+            return user
+        elif (
+            username
+            and password
+            and current_app.config["SERVICE_USER_ALLOW_PASSWORD_LOGIN"] == 1
+        ):
+
+            user = self.auth_db_authenticate(username, password)
+
+            if user.is_service_user():
+                return user
+            else:
+                raise InvalidUsage("Invalid request")
+        else:
+            raise InvalidUsage("Invalid request")
 
 
 def check_last_password_change(user):
     if user.pwd_last_change:
-        if user.pwd_last_change + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"])) < datetime.utcnow():
+        if (
+            user.pwd_last_change
+            + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"]))
+            < datetime.utcnow()
+        ):
             return True
     return False
 

{cornflow-1.1.2 → cornflow-1.1.5a1}/cornflow/schemas/user.py

@@ -1,12 +1,14 @@
 """
 This file contains the schemas used for the users defined in the application
 """
-from marshmallow import fields, Schema
+
+from marshmallow import fields, Schema, validates_schema, ValidationError
 from .instance import InstanceSchema
 
 
 class UserSchema(Schema):
     """ """
+
     id = fields.Int(dump_only=True)
     first_name = fields.Str()
     last_name = fields.Str()
@@ -66,9 +68,23 @@ class LoginEndpointRequest(Schema):
 class LoginOpenAuthRequest(Schema):
     """
     This is the schema used by the login endpoint with Open ID protocol
+    Validates that either a token is provided, or both username and password are present
     """
 
-    token = fields.Str(required=True)
+    token = fields.Str(required=False)
+    username = fields.Str(required=False)
+    password = fields.Str(required=False)
+
+    @validates_schema
+    def validate_fields(self, data, **kwargs):
+        if data.get("token") is None:
+            if not data.get("username") or not data.get("password"):
+                raise ValidationError(
+                    "A token needs to be provided when using Open ID authentication"
+                )
+        else:
+            if data.get("username") or data.get("password"):
+                raise ValidationError("The login needs to be done with a token only")
 
 
 class SignupRequest(Schema):

{cornflow-1.1.2 → cornflow-1.1.5a1}/cornflow/shared/authentication/auth.py

@@ -14,6 +14,8 @@ from datetime import datetime, timedelta
 from flask import request, g, current_app, Request
 from functools import wraps
 from typing import Union, Tuple
+
+from jwt import DecodeError
 from werkzeug.datastructures import Headers
 
 # Imports from internal modules
@@ -103,7 +105,8 @@ class Auth:
             )
 
         payload = {
-            "exp": datetime.utcnow() + timedelta(hours=float(current_app.config["TOKEN_DURATION"])),
+            "exp": datetime.utcnow()
+            + timedelta(hours=float(current_app.config["TOKEN_DURATION"])),
             "iat": datetime.utcnow(),
             "sub": user_id,
         }
@@ -314,7 +317,10 @@ class Auth:
         :return: the key identifier
         :rtype: str
         """
-        headers = jwt.get_unverified_header(token)
+        try:
+            headers = jwt.get_unverified_header(token)
+        except DecodeError as err:
+            raise InvalidCredentials("Token is not valid")
         if not headers:
             raise InvalidCredentials("Token is missing the headers")
         try:
@@ -346,9 +352,9 @@ class Auth:
         try:
             response = requests.get(discovery_url)
             response.raise_for_status()
-        except requests.exceptions.HTTPError as error:
+        except requests.exceptions.HTTPError:
             raise CommunicationError(
-                f"Error getting issuer discovery meta from {discovery_url}", error
+                f"Error getting issuer discovery meta from {discovery_url}"
             )
         return response.json()