cornflow 1.2.3a5__py3-none-any.whl → 1.3.0rc1__py3-none-any.whl

cornflow/app.py

@@ -4,6 +4,8 @@ Main file with the creation of the app logic
 
 # Full imports
 import os
+from logging.config import dictConfig
+
 import click
 
 # Partial imports
@@ -13,9 +15,8 @@ from flask_apispec.extension import FlaskApiSpec
 from flask_cors import CORS
 from flask_migrate import Migrate
 from flask_restful import Api
-from logging.config import dictConfig
-from werkzeug.middleware.dispatcher import DispatcherMiddleware
 from werkzeug.exceptions import NotFound
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
 
 # Module imports
 from cornflow.commands import (
@@ -36,7 +37,14 @@ from cornflow.endpoints.login import LoginEndpoint, LoginOpenAuthEndpoint
 from cornflow.endpoints.signup import SignUpEndpoint
 from cornflow.shared import db, bcrypt
 from cornflow.shared.compress import init_compress
-from cornflow.shared.const import AUTH_DB, AUTH_LDAP, AUTH_OID
+from cornflow.shared.const import (
+    AUTH_DB,
+    AUTH_LDAP,
+    AUTH_OID,
+    CONDITIONAL_ENDPOINTS,
+    SIGNUP_WITH_AUTH,
+    SIGNUP_WITH_NO_AUTH,
+)
 from cornflow.shared.exceptions import initialize_errorhandlers, ConfigurationError
 from cornflow.shared.log_config import log_config
 
@@ -95,13 +103,21 @@ def create_app(env_name="development", dataconn=None):
 
     if auth_type == AUTH_DB:
         signup_activated = int(app.config["SIGNUP_ACTIVATED"])
-        if signup_activated == 1:
-            api.add_resource(SignUpEndpoint, "/signup/", endpoint="signup")
-        api.add_resource(LoginEndpoint, "/login/", endpoint="login")
+        if signup_activated in [SIGNUP_WITH_AUTH, SIGNUP_WITH_NO_AUTH]:
+            api.add_resource(
+                SignUpEndpoint, CONDITIONAL_ENDPOINTS["signup"], endpoint="signup"
+            )
+        api.add_resource(
+            LoginEndpoint, CONDITIONAL_ENDPOINTS["login"], endpoint="login"
+        )
     elif auth_type == AUTH_LDAP:
-        api.add_resource(LoginEndpoint, "/login/", endpoint="login")
+        api.add_resource(
+            LoginEndpoint, CONDITIONAL_ENDPOINTS["login"], endpoint="login"
+        )
     elif auth_type == AUTH_OID:
-        api.add_resource(LoginOpenAuthEndpoint, "/login/", endpoint="login")
+        api.add_resource(
+            LoginOpenAuthEndpoint, CONDITIONAL_ENDPOINTS["login"], endpoint="login"
+        )
     else:
         raise ConfigurationError(
             error="Invalid authentication type",

cornflow/cli/service.py

@@ -36,6 +36,12 @@ from cornflow.shared.const import (
     ADMIN_ROLE,
     SERVICE_ROLE,
     PLANNER_ROLE,
+    DATABRICKS_BACKEND,
+    AIRFLOW_BACKEND,
+    SIGNUP_WITH_AUTH,
+    OID_PROVIDER_AWS,
+    OID_PROVIDER_AZURE,
+    OID_OTHER,
 )
 from cornflow.shared import db
 from cryptography.fernet import Fernet
@@ -82,13 +88,14 @@ def init_cornflow_service():
                 config["cornflow_service_email"],
                 config["cornflow_service_pwd"],
             )
-            _sync_with_airflow(
-                config["airflow_url"],
-                config["airflow_user"],
-                config["airflow_pwd"],
-                config["open_deployment"],
-                external_app=False,
-            )
+            if config["cornflow_backend"] == AIRFLOW_BACKEND:
+                _sync_with_airflow(
+                    config["airflow_url"],
+                    config["airflow_user"],
+                    config["airflow_pwd"],
+                    config["open_deployment"],
+                    external_app=False,
+                )
         _start_application(external_application, environment)
 
     elif external_application == 1:
@@ -114,14 +121,14 @@ def init_cornflow_service():
                 config["cornflow_service_email"],
                 config["cornflow_service_pwd"],
             )
-
-            _sync_with_airflow(
-                config["airflow_url"],
-                config["airflow_user"],
-                config["airflow_pwd"],
-                config["open_deployment"],
-                external_app=True,
-            )
+            if config["cornflow_backend"] == AIRFLOW_BACKEND:
+                _sync_with_airflow(
+                    config["airflow_url"],
+                    config["airflow_user"],
+                    config["airflow_pwd"],
+                    config["open_deployment"],
+                    external_app=True,
+                )
         _start_application(external_application, environment, external_app_module)
 
     else:
@@ -135,14 +142,36 @@ def _setup_environment_variables():
     environment = os.getenv("FLASK_ENV", "development")
     os.environ["FLASK_ENV"] = environment
 
-    # Airflow details
-    airflow_user = os.getenv("AIRFLOW_USER", "admin")
-    airflow_pwd = os.getenv("AIRFLOW_PWD", "admin")
-    airflow_url = os.getenv("AIRFLOW_URL", "http://webserver:8080")
-    os.environ["AIRFLOW_USER"] = airflow_user
-    os.environ["AIRFLOW_PWD"] = airflow_pwd
-    os.environ["AIRFLOW_URL"] = airflow_url
-
+    ###################################
+    # Global defaults and back-compat #
+    ###################################
+    # cornflow backend selection
+    cornflow_backend = os.getenv("CORNFLOW_BACKEND", str(AIRFLOW_BACKEND))
+    os.environ["CORNFLOW_BACKEND"] = cornflow_backend
+    cornflow_backend = int(cornflow_backend)
+    # Airflow global default conn
+    if cornflow_backend == AIRFLOW_BACKEND:
+        airflow_user = os.getenv("AIRFLOW_USER", "admin")
+        airflow_pwd = os.getenv("AIRFLOW_PWD", "admin")
+        airflow_url = os.getenv("AIRFLOW_URL", "http://webserver:8080")
+        os.environ["AIRFLOW_USER"] = airflow_user
+        os.environ["AIRFLOW_PWD"] = airflow_pwd
+        os.environ["AIRFLOW_URL"] = airflow_url
+    elif cornflow_backend == DATABRICKS_BACKEND:
+        databricks_url = os.getenv("DATABRICKS_HOST")
+        databricks_auth_secret = os.getenv("DATABRICKS_CLIENT_SECRET")
+        databricks_token_endpoint = os.getenv("DATABRICKS_TOKEN_ENDPOINT")
+        databricks_ep_clusters = os.getenv("DATABRICKS_EP_CLUSTERS")
+        databricks_client_id = os.getenv("DATABRICKS_CLIENT_ID")
+        databricks_health_path = os.getenv("DATABRICKS_HEALTH_PATH")
+        os.environ["DATABRICKS_HEALTH_PATH"] = databricks_health_path
+        os.environ["DATABRICKS_HOST"] = databricks_url
+        os.environ["DATABRICKS_CLIENT_SECRET"] = databricks_auth_secret
+        os.environ["DATABRICKS_TOKEN_ENDPOINT"] = databricks_token_endpoint
+        os.environ["DATABRICKS_EP_CLUSTERS"] = databricks_ep_clusters
+        os.environ["DATABRICKS_CLIENT_ID"] = databricks_client_id
+    else:
+        raise Exception("Selected backend not among valid options")
     # Cornflow app config
     os.environ.setdefault("cornflow_url", "http://cornflow:5000")
     os.environ["FLASK_APP"] = "cornflow.app"
@@ -155,6 +184,10 @@ def _setup_environment_variables():
 
     # Platform auth config and service users
     auth = int(os.getenv("AUTH_TYPE", AUTH_DB))
+    if auth == AUTH_OID:
+        oid_provider = int(os.getenv("OID_PROVIDER", OID_PROVIDER_AWS))
+        os.environ["OID_PROVIDER"] = str(oid_provider)
+
     cornflow_admin_user = os.getenv("CORNFLOW_ADMIN_USER", "cornflow_admin")
     cornflow_admin_email = os.getenv(
         "CORNFLOW_ADMIN_EMAIL", "cornflow_admin@cornflow.com"
@@ -171,7 +204,7 @@ def _setup_environment_variables():
     os.environ["CORNFLOW_LOGGING"] = cornflow_logging
     open_deployment = os.getenv("OPEN_DEPLOYMENT", 1)
     os.environ["OPEN_DEPLOYMENT"] = str(open_deployment)
-    signup_activated = os.getenv("SIGNUP_ACTIVATED", 1)
+    signup_activated = os.getenv("SIGNUP_ACTIVATED", SIGNUP_WITH_AUTH)
     os.environ["SIGNUP_ACTIVATED"] = str(signup_activated)
     user_access_all_objects = os.getenv("USER_ACCESS_ALL_OBJECTS", 0)
     os.environ["USER_ACCESS_ALL_OBJECTS"] = str(user_access_all_objects)
@@ -191,25 +224,42 @@ def _setup_environment_variables():
 
     external_application = int(os.getenv("EXTERNAL_APP", 0))
     external_app_module = os.getenv("EXTERNAL_APP_MODULE")
+    base_dict = {
+            "environment": environment,
+            "auth": auth,
+            "cornflow_db_conn": cornflow_db_conn,
+            "cornflow_admin_user": cornflow_admin_user,
+            "cornflow_admin_email": cornflow_admin_email,
+            "cornflow_admin_pwd": cornflow_admin_pwd,
+            "cornflow_service_user": cornflow_service_user,
+            "cornflow_service_email": cornflow_service_email,
+            "cornflow_service_pwd": cornflow_service_pwd,
+            "cornflow_logging": cornflow_logging,
+            "open_deployment": open_deployment,
+            "external_application": external_application,
+            "external_app_module": external_app_module,
+            "cornflow_backend": cornflow_backend,
+        }
+    if cornflow_backend == AIRFLOW_BACKEND:
+        base_dict["airflow_user"] = airflow_user
+        base_dict["airflow_pwd"] = airflow_pwd
+        base_dict["airflow_url"] = airflow_url
+    
+    elif cornflow_backend == DATABRICKS_BACKEND:
+        base_dict["databricks_url"] = databricks_url
+        base_dict["databricks_auth_secret"] = databricks_auth_secret
+        base_dict["databricks_token_endpoint"] = databricks_token_endpoint
+        base_dict["databricks_ep_clusters"] = databricks_ep_clusters
+        base_dict["databricks_client_id"] = databricks_client_id
+        base_dict["databricks_health_path"] = databricks_health_path
+        
+    else:
+        raise Exception("Selected backend not among valid options")
+
+    if auth == AUTH_OID:
+        base_dict["oid_provider"] = oid_provider
 
-    return {
-        "environment": environment,
-        "auth": auth,
-        "airflow_user": airflow_user,
-        "airflow_pwd": airflow_pwd,
-        "airflow_url": airflow_url,
-        "cornflow_db_conn": cornflow_db_conn,
-        "cornflow_admin_user": cornflow_admin_user,
-        "cornflow_admin_email": cornflow_admin_email,
-        "cornflow_admin_pwd": cornflow_admin_pwd,
-        "cornflow_service_user": cornflow_service_user,
-        "cornflow_service_email": cornflow_service_email,
-        "cornflow_service_pwd": cornflow_service_pwd,
-        "cornflow_logging": cornflow_logging,
-        "open_deployment": open_deployment,
-        "external_application": external_application,
-        "external_app_module": external_app_module,
-    }
+    return base_dict
 
 
 def _configure_logging(cornflow_logging):
@@ -377,4 +427,4 @@ def _register_key():
         os.system(add_key)
         return True
     else:
-        return False
+        return False

cornflow/commands/auxiliar.py

@@ -3,7 +3,7 @@ from importlib import import_module
 
 from flask import current_app
 
-from cornflow.endpoints import resources, alarms_resources
+from cornflow.endpoints import alarms_resources, get_resources
 from cornflow.models import RoleModel
 from cornflow.shared.const import (
     EXTRA_PERMISSION_ASSIGNATION,
@@ -17,6 +17,9 @@ def get_all_external(external_app):
     Get all resources, extra permissions, and custom roles actions.
     external_app: If provided, it will get the resources and extra permissions for the external app.
     """
+    # We get base and conditional resources
+    resources = get_resources()
+
     if external_app is None:
         resources_to_register = resources
         extra_permissions = EXTRA_PERMISSION_ASSIGNATION

cornflow/commands/dag.py

@@ -10,7 +10,7 @@ def register_deployed_dags_command(
 
     # Internal modules imports
     from cornflow_client.airflow.api import Airflow
-    from cornflow.models import DeployedDAG
+    from cornflow.models import DeployedWorkflow
     from cornflow.shared import db
     from cornflow.shared.const import AIRFLOW_NOT_REACHABLE_MSG
 
@@ -28,7 +28,7 @@ def register_deployed_dags_command(
             current_app.logger.info(f"{AIRFLOW_NOT_REACHABLE_MSG}")
         return False
 
-    dags_registered = [dag.id for dag in DeployedDAG.get_all_objects()]
+    dags_registered = [dag.id for dag in DeployedWorkflow.get_all_objects()]
 
     response = af_client.get_model_dags()
     dag_list = response.json()["dags"]
@@ -40,7 +40,7 @@ def register_deployed_dags_command(
     }
 
     processed_dags = [
-        DeployedDAG(
+        DeployedWorkflow(
             {
                 "id": dag["dag_id"],
                 "description": dag["description"],
@@ -76,15 +76,15 @@ def register_deployed_dags_command(
 
 
 def register_deployed_dags_command_test(dags: list = None, verbose: bool = False):
-    from cornflow.models import DeployedDAG
+    from cornflow.models import DeployedWorkflow
     from flask import current_app
     from cornflow_client import get_pulp_jsonschema, get_empty_schema
 
     if dags is None:
-        dags = ["solve_model_dag", "gc", "timer"]
+        dags = ["solve_model_dag", "gc", "timer", "979073949072767"]
 
     deployed_dag = [
-        DeployedDAG(
+        DeployedWorkflow(
             {
                 "id": "solve_model_dag",
                 "description": None,
@@ -96,7 +96,7 @@ def register_deployed_dags_command_test(dags: list = None, verbose: bool = False
             }
         )
     ] + [
-        DeployedDAG(
+        DeployedWorkflow(
             {
                 "id": dag,
                 "description": None,

cornflow/commands/permissions.py

@@ -8,6 +8,7 @@ from cornflow.commands.auxiliar import (
 from cornflow.models import ViewModel, PermissionViewRoleModel
 from cornflow.shared import db
 from cornflow.shared.const import ALL_DEFAULT_ROLES, GET_ACTION
+import click
 from cornflow.shared.const import (
     BASE_PERMISSION_ASSIGNATION,
 )
@@ -53,12 +54,14 @@ def register_base_permissions_command(external_app: str = None, verbose: bool =
     save_and_delete_permissions(permissions_to_register, permissions_to_delete)
 
     if len(permissions_to_register) > 0:
-        current_app.logger.info(f"Permissions registered: {permissions_to_register}")
+        current_app.logger.info(
+            f"Permissions registered: {len(permissions_to_register)}"
+        )
     else:
         current_app.logger.info("No new permissions to register")
 
     if len(permissions_to_delete) > 0:
-        current_app.logger.info(f"Permissions deleted: {permissions_to_delete}")
+        current_app.logger.info(f"Permissions deleted: {len(permissions_to_delete)}")
     else:
         current_app.logger.info("No permissions to delete")
 
@@ -233,7 +236,7 @@ def register_dag_permissions_command(
     from flask import current_app
     from sqlalchemy.exc import DBAPIError, IntegrityError
 
-    from cornflow.models import DeployedDAG, PermissionsDAG, UserModel
+    from cornflow.models import DeployedWorkflow, PermissionsDAG, UserModel
     from cornflow.shared import db
 
     if open_deployment is None:
@@ -251,9 +254,10 @@ def register_dag_permissions_command(
         current_app.logger.error(f"Unknown error on database commit: {e}")
 
     all_users = UserModel.get_all_users().all()
-    all_dags = DeployedDAG.get_all_objects().all()
+    all_dags = DeployedWorkflow.get_all_objects().all()
 
     if open_deployment == 1:
+
         permissions = [
             PermissionsDAG({"dag_id": dag.id, "user_id": user.id})
             for user in all_users
@@ -295,9 +299,11 @@ def register_dag_permissions_command(
             )
 
     if verbose:
+        click.echo(f"Workflow permissions registered")
         if len(permissions) > 1:
-            current_app.logger.info(f"DAG permissions registered: {permissions}")
+            current_app.logger.info(
+                f"Workflow permissions registered: {len(permissions)}"
+            )
         else:
-            current_app.logger.info("No new DAG permissions")
+            current_app.logger.info("No new Workflow permissions")
 
-    pass

cornflow/commands/views.py

@@ -7,6 +7,8 @@ from sqlalchemy.exc import DBAPIError, IntegrityError
 
 from cornflow.endpoints import resources, alarms_resources
 
+from cornflow.endpoints import alarms_resources, get_resources
+
 # Imports from internal libraries
 from cornflow.models import ViewModel
 from cornflow.shared import db
@@ -182,6 +184,7 @@ def get_database_view():
 
 
 def get_resources_to_register(external_app):
+    resources = get_resources()
     if external_app is None:
         resources_to_register = resources
         if current_app.config["ALARMS_ENDPOINTS"]:

cornflow/config.py

@@ -1,7 +1,8 @@
 import os
-from .shared.const import AUTH_DB, PLANNER_ROLE, AUTH_OID
+from .shared.const import AUTH_DB, PLANNER_ROLE, AUTH_OID, SIGNUP_WITH_AUTH, SIGNUP_WITH_NO_AUTH
 from apispec import APISpec
 from apispec.ext.marshmallow import MarshmallowPlugin
+from cornflow.shared.const import AIRFLOW_BACKEND, DATABRICKS_BACKEND
 
 
 class DefaultConfig(object):
@@ -15,9 +16,7 @@ class DefaultConfig(object):
     SECRET_TOKEN_KEY = os.getenv("SECRET_KEY")
     SECRET_BI_KEY = os.getenv("SECRET_BI_KEY")
     SQLALCHEMY_DATABASE_URI = os.getenv("DATABASE_URL", "sqlite:///cornflow.db")
-    AIRFLOW_URL = os.getenv("AIRFLOW_URL")
-    AIRFLOW_USER = os.getenv("AIRFLOW_USER")
-    AIRFLOW_PWD = os.getenv("AIRFLOW_PWD")
+
     AUTH_TYPE = int(os.getenv("AUTH_TYPE", AUTH_DB))
     DEFAULT_ROLE = int(os.getenv("DEFAULT_ROLE", PLANNER_ROLE))
     CORS_ORIGINS = os.getenv("CORS_ORIGINS", "*")
@@ -25,9 +24,24 @@ class DefaultConfig(object):
     DEBUG = True
     TESTING = True
     LOG_LEVEL = int(os.getenv("LOG_LEVEL", 20))
-    SIGNUP_ACTIVATED = int(os.getenv("SIGNUP_ACTIVATED", 1))
+    SIGNUP_ACTIVATED = int(os.getenv("SIGNUP_ACTIVATED", SIGNUP_WITH_AUTH))
     CORNFLOW_SERVICE_USER = os.getenv("CORNFLOW_SERVICE_USER", "service_user")
 
+    # To change the tasks backend used by cornflow to solve the optimization models
+    CORNFLOW_BACKEND = int(os.getenv("CORNFLOW_BACKEND", AIRFLOW_BACKEND))
+
+    # AIRFLOW config
+    AIRFLOW_URL = os.getenv("AIRFLOW_URL")
+    AIRFLOW_USER = os.getenv("AIRFLOW_USER")
+    AIRFLOW_PWD = os.getenv("AIRFLOW_PWD")
+
+    # DATABRICKS config
+    DATABRICKS_URL = os.getenv("DATABRICKS_HOST")
+    DATABRICKS_AUTH_SECRET = os.getenv("DATABRICKS_CLIENT_SECRET")
+    DATABRICKS_TOKEN_ENDPOINT = os.getenv("DATABRICKS_TOKEN_ENDPOINT")
+    DATABRICKS_EP_CLUSTERS = os.getenv("DATABRICKS_EP_CLUSTERS")
+    DATABRICKS_CLIENT_ID = os.getenv("DATABRICKS_CLIENT_ID")
+    DATABRICKS_HEALTH_PATH = os.getenv("DATABRICKS_HEALTH_PATH", "default path")
     # If service user is allowed to log with username and password
     SERVICE_USER_ALLOW_PASSWORD_LOGIN = int(
         os.getenv("SERVICE_USER_ALLOW_PASSWORD_LOGIN", 1)
@@ -119,12 +133,17 @@ class Testing(DefaultConfig):
     AIRFLOW_PWD = os.getenv("AIRFLOW_PWD", "admin")
     OPEN_DEPLOYMENT = 1
     LOG_LEVEL = int(os.getenv("LOG_LEVEL", 10))
+    SIGNUP_ACTIVATED = SIGNUP_WITH_NO_AUTH
+
+class TestingDatabricks(Testing):
+    CORNFLOW_BACKEND = DATABRICKS_BACKEND
 
 
 class TestingOpenAuth(Testing):
     """
     Configuration class for testing some edge cases with Open Auth login
     """
+
     AUTH_TYPE = AUTH_OID
     OID_PROVIDER = "https://test-provider.example.com"
     OID_EXPECTED_AUDIENCE = "test-audience-id"
@@ -157,5 +176,6 @@ app_config = {
     "testing": Testing,
     "production": Production,
     "testing-oauth": TestingOpenAuth,
-    "testing-root": TestingApplicationRoot
+    "testing-root": TestingApplicationRoot,
+    "testing-databricks": TestingDatabricks,
 }

cornflow/endpoints/__init__.py

@@ -4,6 +4,8 @@ All references to endpoints should be imported from here
 The login resource gets created on app startup as it depends on configuration
 """
 
+from flask import current_app
+from cornflow.shared.const import CONDITIONAL_ENDPOINTS
 from .action import ActionListEndpoint
 from .alarms import AlarmsEndpoint, AlarmDetailEndpoint
 from .apiview import ApiViewListEndpoint
@@ -237,3 +239,28 @@ alarms_resources = [
         endpoint="main-alarms",
     ),
 ]
+
+
+def get_resources():
+    """
+    Get the resources based on the configuration
+
+    :return: The resources based on the configuration
+    :rtype: list
+    """
+    base_resources = resources.copy()
+    registered_resources = current_app.view_functions.keys()
+    for resource in registered_resources:
+        if resource in CONDITIONAL_ENDPOINTS.keys():
+            # Check if the resource already exists
+            if resource not in [
+                present_resource["endpoint"] for present_resource in base_resources
+            ]:
+                base_resources.append(
+                    dict(
+                        resource=current_app.view_functions[resource].view_class,
+                        urls=CONDITIONAL_ENDPOINTS[resource],
+                        endpoint=resource,
+                    )
+                )
+    return base_resources