cornflow 2.0.0a12__py3-none-any.whl → 2.0.0a14__py3-none-any.whl

cornflow/app.py

@@ -51,6 +51,8 @@ def create_app(env_name="development", dataconn=None):
     """
     dictConfig(log_config(app_config[env_name].LOG_LEVEL))
 
+    # Note: Explicit CSRF protection is not configured as the application uses
+    # JWT for authentication via headers, mitigating standard CSRF vulnerabilities.
     app = Flask(__name__)
     app.json.sort_keys = False
     app.logger.setLevel(app_config[env_name].LOG_LEVEL)
@@ -103,7 +105,7 @@ def create_app(env_name="development", dataconn=None):
     else:
         raise ConfigurationError(
             error="Invalid authentication type",
-            log_txt="Error while configuring authentication. The authentication type is not valid."
+            log_txt="Error while configuring authentication. The authentication type is not valid.",
         )
 
     initialize_errorhandlers(app)

cornflow/cli/__init__.py

@@ -17,6 +17,10 @@ from cornflow.cli.views import views
 
 @click.group(name="cornflow", help="Commands in the cornflow cli")
 def cli():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 

cornflow/cli/actions.py

@@ -7,6 +7,10 @@ from .arguments import verbose
 
 @click.group(name="actions", help="Commands to manage the actions")
 def actions():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 

cornflow/cli/config.py

@@ -7,6 +7,10 @@ from .arguments import path
 
 @click.group(name="config", help="Commands to manage the configuration variables")
 def config():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 

cornflow/cli/migrations.py

@@ -3,6 +3,7 @@ import os.path
 
 import click
 from cornflow.shared import db
+from cornflow.shared.const import MIGRATIONS_DEFAULT_PATH
 from flask_migrate import Migrate, migrate, upgrade, downgrade, init
 
 from .utils import get_app
@@ -10,6 +11,10 @@ from .utils import get_app
 
 @click.group(name="migrations", help="Commands to manage the migrations")
 def migrations():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 
@@ -18,12 +23,12 @@ def migrate_migrations():
     app = get_app()
     external = int(os.getenv("EXTERNAL_APP", 0))
     if external == 0:
-        path = "./cornflow/migrations"
+        path = MIGRATIONS_DEFAULT_PATH
     else:
         path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
 
     with app.app_context():
-        migration_client = Migrate(app=app, db=db, directory=path)
+        Migrate(app=app, db=db, directory=path)
         migrate()
 
 
@@ -35,12 +40,12 @@ def upgrade_migrations(revision="head"):
     app = get_app()
     external = int(os.getenv("EXTERNAL_APP", 0))
     if external == 0:
-        path = "./cornflow/migrations"
+        path = MIGRATIONS_DEFAULT_PATH
     else:
         path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
 
     with app.app_context():
-        migration_client = Migrate(app=app, db=db, directory=path)
+        Migrate(app=app, db=db, directory=path)
         upgrade(revision=revision)
 
 
@@ -52,12 +57,12 @@ def downgrade_migrations(revision="-1"):
     app = get_app()
     external = int(os.getenv("EXTERNAL_APP", 0))
     if external == 0:
-        path = "./cornflow/migrations"
+        path = MIGRATIONS_DEFAULT_PATH
     else:
         path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
 
     with app.app_context():
-        migration_client = Migrate(app=app, db=db, directory=path)
+        Migrate(app=app, db=db, directory=path)
         downgrade(revision=revision)
 
 
@@ -69,10 +74,10 @@ def init_migrations():
     app = get_app()
     external = int(os.getenv("EXTERNAL_APP", 0))
     if external == 0:
-        path = "./cornflow/migrations"
+        path = MIGRATIONS_DEFAULT_PATH
     else:
         path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
 
     with app.app_context():
-        migration_client = Migrate(app=app, db=db, directory=path)
+        Migrate(app=app, db=db, directory=path)
         init()

cornflow/cli/permissions.py

@@ -11,6 +11,10 @@ from .utils import get_app
 
 @click.group(name="permissions", help="Commands to manage the permissions")
 def permissions():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 

cornflow/cli/roles.py

@@ -6,6 +6,10 @@ from .utils import get_app
 
 @click.group(name="roles", help="Commands to manage the roles")
 def roles():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 

cornflow/cli/schemas.py

@@ -1,6 +1,7 @@
 """
 File that implements the generate from schema cli command
 """
+
 import click
 from .tools.api_generator import APIGenerator
 from .tools.schema_generator import SchemaGenerator
@@ -20,6 +21,10 @@ METHOD_OPTIONS = [
 
 @click.group(name="schemas", help="Commands to manage the schemas")
 def schemas():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 

cornflow/cli/service.py

@@ -31,16 +31,94 @@ from cornflow.shared import db
 from cryptography.fernet import Fernet
 from flask_migrate import Migrate, upgrade
 
+MAIN_WD = "/usr/src/app"
+
 
 @click.group(name="service", help="Commands to run the cornflow service")
 def service():
+    """
+    This method is empty but it serves as the building block
+    for the rest of the commands
+    """
     pass
 
 
 @service.command(name="init", help="Initialize the service")
 def init_cornflow_service():
     click.echo("Starting the service")
-    os.chdir("/usr/src/app")
+    os.chdir(MAIN_WD)
+
+    config = _setup_environment_variables()
+    _configure_logging(config["cornflow_logging"])
+
+    external_application = config["external_application"]
+    environment = config["environment"]
+    cornflow_db_conn = config["cornflow_db_conn"]
+    external_app_module = config["external_app_module"]
+
+    app = None  # Initialize app to None
+
+    if external_application == 0:
+        click.echo("Initializing standard Cornflow application")
+        app = create_app(environment, cornflow_db_conn)
+        with app.app_context():
+            _initialize_database(app)
+            _create_initial_users(
+                config["auth"],
+                config["cornflow_admin_user"],
+                config["cornflow_admin_email"],
+                config["cornflow_admin_pwd"],
+                config["cornflow_service_user"],
+                config["cornflow_service_email"],
+                config["cornflow_service_pwd"],
+            )
+            _sync_with_airflow(
+                config["airflow_url"],
+                config["airflow_user"],
+                config["airflow_pwd"],
+                config["open_deployment"],
+                external_app=False,
+            )
+        _start_application(external_application, environment)
+
+    elif external_application == 1:
+        click.echo(f"Initializing Cornflow with external app: {external_app_module}")
+        if not external_app_module:
+            sys.exit("FATAL: EXTERNAL_APP is 1 but EXTERNAL_APP_MODULE is not set.")
+
+        _setup_external_app()
+        from importlib import import_module
+
+        external_app_lib = import_module(external_app_module)
+        app = external_app_lib.create_wsgi_app(environment, cornflow_db_conn)
+
+        with app.app_context():
+            _initialize_database(app, external_app_module)
+            _create_initial_users(
+                config["auth"],
+                config["cornflow_admin_user"],
+                config["cornflow_admin_email"],
+                config["cornflow_admin_pwd"],
+                config["cornflow_service_user"],
+                config["cornflow_service_email"],
+                config["cornflow_service_pwd"],
+            )
+            _sync_with_airflow(
+                config["airflow_url"],
+                config["airflow_user"],
+                config["airflow_pwd"],
+                config["open_deployment"],
+                external_app=True,
+            )
+        _start_application(external_application, environment, external_app_module)
+
+    else:
+        # This case should ideally be caught earlier or handled differently
+        sys.exit(f"FATAL: Invalid EXTERNAL_APP value: {external_application}")
+
+
+def _setup_environment_variables():
+    """Reads environment variables, sets defaults, and returns config values."""
     environment = os.getenv("FLASK_ENV", "development")
     os.environ["FLASK_ENV"] = environment
 
@@ -72,7 +150,8 @@ def init_cornflow_service():
         os.environ["DATABRICKS_CLIENT_ID"] = databricks_client_id
     else:
         raise Exception("Selected backend not among valid options")
-
+    # Cornflow app config
+    os.environ.setdefault("cornflow_url", "http://cornflow:5000")
     os.environ["FLASK_APP"] = "cornflow.app"
     os.environ["SECRET_KEY"] = os.getenv("FERNET_KEY", Fernet.generate_key().decode())
 
@@ -94,10 +173,9 @@ def init_cornflow_service():
     )
     cornflow_service_pwd = os.getenv("CORNFLOW_SERVICE_PWD", "Service_user1234")
 
-    # Cornflow logging and storage config
+    # Cornflow logging and deployment config
     cornflow_logging = os.getenv("CORNFLOW_LOGGING", "console")
     os.environ["CORNFLOW_LOGGING"] = cornflow_logging
-
     open_deployment = os.getenv("OPEN_DEPLOYMENT", 1)
     os.environ["OPEN_DEPLOYMENT"] = str(open_deployment)
     signup_activated = os.getenv("SIGNUP_ACTIVATED", 1)
@@ -108,176 +186,211 @@ def init_cornflow_service():
     os.environ["DEFAULT_ROLE"] = str(default_role)
 
     # Check LDAP parameters for active directory and show message
-    if os.getenv("AUTH_TYPE") == AUTH_LDAP:
-        print(
-            "WARNING: Cornflow will be deployed with LDAP Authorization. Please review your ldap auth configuration."
+    if auth == AUTH_LDAP:
+        click.echo(
+            "WARNING: Cornflow will be deployed with LDAP Authorization. "
+            "Please review your ldap auth configuration."
         )
 
     # check database param from docker env
-    if os.getenv("DATABASE_URL") is None:
+    if cornflow_db_conn is None:
         sys.exit("FATAL: you need to provide a postgres database for Cornflow")
 
-    # set logrotate config file
+    external_application = int(os.getenv("EXTERNAL_APP", 0))
+    external_app_module = os.getenv("EXTERNAL_APP_MODULE")
+    if cornflow_backend == AIRFLOW_BACKEND:
+        return {
+            "environment": environment,
+            "auth": auth,
+            "airflow_user": airflow_user,
+            "airflow_pwd": airflow_pwd,
+            "airflow_url": airflow_url,
+            "cornflow_db_conn": cornflow_db_conn,
+            "cornflow_admin_user": cornflow_admin_user,
+            "cornflow_admin_email": cornflow_admin_email,
+            "cornflow_admin_pwd": cornflow_admin_pwd,
+            "cornflow_service_user": cornflow_service_user,
+            "cornflow_service_email": cornflow_service_email,
+            "cornflow_service_pwd": cornflow_service_pwd,
+            "cornflow_logging": cornflow_logging,
+            "open_deployment": open_deployment,
+            "external_application": external_application,
+            "external_app_module": external_app_module,
+        }
+    elif cornflow_backend == DATABRICKS_BACKEND:
+        return {
+            "environment": environment,
+            "auth": auth,
+            "databricks_url": databricks_url,
+            "databricks_auth_secret": databricks_auth_secret,
+            "databricks_token_endpoint": databricks_token_endpoint,
+            "databricks_ep_clusters": databricks_ep_clusters,
+            "databricks_client_id": databricks_client_id,
+            "cornflow_db_conn": cornflow_db_conn,
+            "cornflow_admin_user": cornflow_admin_user,
+            "cornflow_admin_email": cornflow_admin_email,
+            "cornflow_admin_pwd": cornflow_admin_pwd,
+            "cornflow_service_user": cornflow_service_user,
+            "cornflow_service_email": cornflow_service_email,
+            "cornflow_service_pwd": cornflow_service_pwd,
+            "cornflow_logging": cornflow_logging,
+            "open_deployment": open_deployment,
+            "external_application": external_application,
+            "external_app_module": external_app_module,
+        }
+    else:
+        raise Exception("Selected backend not among valid options")
+
+
+def _configure_logging(cornflow_logging):
+    """Configures log rotation if logging to file."""
     if cornflow_logging == "file":
         try:
-            conf = "/usr/src/app/log/*.log {\n\
-            rotate 30\n \
-            daily\n\
-            compress\n\
-            size 20M\n\
-            postrotate\n\
-             kill -HUP \$(cat /usr/src/app/gunicorn.pid)\n \
-            endscript}"
+            conf = f"""/usr/src/app/log/*.log {{
+            rotate 30
+            daily
+            compress
+            size 20M
+            postrotate
+             kill -HUP $(cat {MAIN_WD}/gunicorn.pid)
+            endscript}}"""
             logrotate = subprocess.run(
-                f"cat > /etc/logrotate.d/cornflow <<EOF\n {conf} \nEOF", shell=True
+                f"cat > /etc/logrotate.d/cornflow <<EOF\n {conf} \nEOF",
+                shell=True,
+                capture_output=True,
+                text=True,
             )
-            out_logrotate = logrotate.stdout
-            click.echo(out_logrotate)
-
-        except error:
-            click.echo(error)
-
-    external_application = int(os.getenv("EXTERNAL_APP", 0))
-    if external_application == 0:
-        os.environ["GUNICORN_WORKING_DIR"] = "/usr/src/app"
-    elif external_application == 1:
-        os.environ["GUNICORN_WORKING_DIR"] = "/usr/src/app"
-    else:
-        raise Exception("No external application found")
-
-    if external_application == 0:
-        click.echo("Starting cornflow")
-        app = create_app(environment, cornflow_db_conn)
-        with app.app_context():
-            path = f"{os.path.dirname(cornflow.__file__)}/migrations"
-            Migrate(app=app, db=db, directory=path)
-            upgrade()
-            access_init_command(verbose=False)
-            if auth == AUTH_DB or auth == AUTH_OID:
-                # create cornflow admin user
-                create_user_with_role(
-                    cornflow_admin_user,
-                    cornflow_admin_email,
-                    cornflow_admin_pwd,
-                    "admin",
-                    ADMIN_ROLE,
-                    verbose=True,
-                )
-                # create cornflow service user
-                create_user_with_role(
-                    cornflow_service_user,
-                    cornflow_service_email,
-                    cornflow_service_pwd,
-                    "serviceuser",
-                    SERVICE_ROLE,
-                    verbose=True,
-                )
-
-            if cornflow_backend == AIRFLOW_BACKEND:
-                register_deployed_dags_command(
-                    airflow_url, airflow_user, airflow_pwd, verbose=True
-                )
-                register_dag_permissions_command(open_deployment, verbose=True)
-                update_schemas_command(
-                    airflow_url, airflow_user, airflow_pwd, verbose=True
-                )
+            if logrotate.returncode != 0:
+                error(f"Error configuring logrotate: {logrotate.stderr}")
             else:
-                register_dag_permissions_command(open_deployment, verbose=True)
-
-        # execute gunicorn application
-        os.system(
-            "/usr/local/bin/gunicorn -c python:cornflow.gunicorn \"cornflow.app:create_app('$FLASK_ENV')\""
+                print(logrotate.stdout)
+        except Exception as e:
+            error(f"Exception during logrotate configuration: {e}")
+
+
+def _initialize_database(app, external_app_module=None):
+    """Initializes the database and runs migrations."""
+    with app.app_context():
+        if external_app_module:
+            from importlib import import_module
+
+            external_app_lib = import_module(external_app_module)
+            migrations_path = f"{os.path.dirname(external_app_lib.__file__)}/migrations"
+        else:
+            migrations_path = f"{os.path.dirname(cornflow.__file__)}/migrations"
+
+        Migrate(app=app, db=db, directory=migrations_path)
+        upgrade()
+        access_init_command(verbose=False)
+
+
+def _create_initial_users(
+    auth,
+    admin_user,
+    admin_email,
+    admin_pwd,
+    service_user,
+    service_email,
+    service_pwd,
+):
+    """Creates the initial admin and service users if using DB or OID auth."""
+    if auth == AUTH_DB or auth == AUTH_OID:
+        # create cornflow admin user
+        create_user_with_role(
+            admin_user,
+            admin_email,
+            admin_pwd,
+            "admin",
+            ADMIN_ROLE,
+            verbose=True,
+        )
+        # create cornflow service user
+        create_user_with_role(
+            service_user,
+            service_email,
+            service_pwd,
+            "serviceuser",
+            SERVICE_ROLE,
+            verbose=True,
         )
 
-    elif external_application == 1:
-        click.echo(f"Starting cornflow + {os.getenv('EXTERNAL_APP_MODULE')}")
-        os.chdir("/usr/src/app")
-
-        if register_key():
-            prefix = "CUSTOM_SSH_"
-            env_variables = {}
-            for key, value in os.environ.items():
-                if key.startswith(prefix):
-                    env_variables[key] = value
 
-            for _, value in env_variables.items():
-                register_ssh_host(value)
+def _sync_with_airflow(
+    airflow_url, airflow_user, airflow_pwd, open_deployment, external_app=False
+):
+    """Syncs DAGs, permissions, and schemas with Airflow."""
+    register_deployed_dags_command(airflow_url, airflow_user, airflow_pwd, verbose=True)
+    register_dag_permissions_command(open_deployment, verbose=True)
+    update_schemas_command(airflow_url, airflow_user, airflow_pwd, verbose=True)
+    if external_app:
+        update_dag_registry_command(
+            airflow_url, airflow_user, airflow_pwd, verbose=True
+        )
 
-        os.system("$(command -v pip) install --user -r requirements.txt")
-        time.sleep(5)
-        sys.path.append("/usr/src/app")
 
-        from importlib import import_module
+def _setup_external_app():
+    """Performs setup steps specific to external applications."""
+    os.chdir(MAIN_WD)
+    if _register_key():
+        prefix = "CUSTOM_SSH_"
+        env_variables = {
+            key: value for key, value in os.environ.items() if key.startswith(prefix)
+        }
+        for _, value in env_variables.items():
+            _register_ssh_host(value)
+
+    # Install requirements for the external app
+    pip_install_cmd = "$(command -v pip) install --user -r requirements.txt"
+    click.echo(f"Running: {pip_install_cmd}")
+    result = subprocess.run(pip_install_cmd, shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        error(f"Error installing requirements: {result.stderr}")
+    else:
+        print(result.stdout)
+    time.sleep(5)  # Consider if this sleep is truly necessary
+    sys.path.append(MAIN_WD)
 
-        external_app = import_module(os.getenv("EXTERNAL_APP_MODULE"))
-        app = external_app.create_wsgi_app(environment, cornflow_db_conn)
-        with app.app_context():
-            path = f"{os.path.dirname(external_app.__file__)}/migrations"
-            migrate = Migrate(app=app, db=db, directory=path)
-            upgrade()
-            access_init_command(verbose=False)
-            if auth == AUTH_DB or auth == AUTH_OID:
-                # create cornflow admin user
-                create_user_with_role(
-                    cornflow_admin_user,
-                    cornflow_admin_email,
-                    cornflow_admin_pwd,
-                    "admin",
-                    ADMIN_ROLE,
-                    verbose=True,
-                )
-                # create cornflow service user
-                create_user_with_role(
-                    cornflow_service_user,
-                    cornflow_service_email,
-                    cornflow_service_pwd,
-                    "serviceuser",
-                    SERVICE_ROLE,
-                    verbose=True,
-                )
-
-            click.echo(f"Selected backend is: {cornflow_backend}")
-            if cornflow_backend == AIRFLOW_BACKEND:
-                register_deployed_dags_command(
-                    airflow_url, airflow_user, airflow_pwd, verbose=True
-                )
-
-                register_dag_permissions_command(open_deployment, verbose=True)
-                update_schemas_command(
-                    airflow_url, airflow_user, airflow_pwd, verbose=True
-                )
-                update_dag_registry_command(
-                    airflow_url, airflow_user, airflow_pwd, verbose=True
-                )
-            elif cornflow_backend == DATABRICKS_BACKEND:
-                register_dag_permissions_command(open_deployment, verbose=True)
-            else:
-                raise Exception("Selected backend not among valid options")
 
-        os.system(
-            f"/usr/local/bin/gunicorn -c python:cornflow.gunicorn "
-            f"\"$EXTERNAL_APP_MODULE:create_wsgi_app('$FLASK_ENV')\""
+def _start_application(external_application, environment, external_app_module=None):
+    """Starts the Gunicorn server."""
+    if external_application == 0:
+        os.environ["GUNICORN_WORKING_DIR"] = MAIN_WD
+        gunicorn_cmd = (
+            "/usr/local/bin/gunicorn -c python:cornflow.gunicorn "
+            f"\"cornflow.app:create_app('{environment}')\""
+        )
+    elif external_application == 1:
+        os.environ["GUNICORN_WORKING_DIR"] = MAIN_WD
+        if not external_app_module:
+            raise ValueError(
+                "EXTERNAL_APP_MODULE must be set for external applications"
+            )
+        gunicorn_cmd = (
+            "/usr/local/bin/gunicorn -c python:cornflow.gunicorn "
+            f"\"{external_app_module}:create_wsgi_app('{environment}')\""
         )
-
     else:
-        raise Exception("No external application found")
+        raise ValueError(f"Invalid EXTERNAL_APP value: {external_application}")
 
+    click.echo(f"Starting application with Gunicorn: {gunicorn_cmd}")
+    os.system(gunicorn_cmd)
 
-def register_ssh_host(host):
+
+def _register_ssh_host(host):
     if host is not None:
-        add_host = f"ssh-keyscan {host} >> /usr/src/app/.ssh/known_hosts"
-        config_ssh_host = f"echo Host {host} >> /usr/src/app/.ssh/config"
-        config_ssh_key = 'echo "   IdentityFile /usr/src/app/.ssh/id_rsa" >> /usr/src/app/.ssh/config'
+        add_host = f"ssh-keyscan {host} >> {MAIN_WD}/.ssh/known_hosts"
+        config_ssh_host = f"echo Host {host} >> {MAIN_WD}/.ssh/config"
+        config_ssh_key = (
+            'echo "   IdentityFile {MAIN_WD}/.ssh/id_rsa" >> {MAIN_WD}/.ssh/config'
+        )
         os.system(add_host)
         os.system(config_ssh_host)
         os.system(config_ssh_key)
 
 
-def register_key():
-    if os.path.isfile("/usr/src/app/.ssh/id_rsa"):
-        add_key = (
-            "chmod 0600 /usr/src/app/.ssh/id_rsa && ssh-add /usr/src/app/.ssh/id_rsa"
-        )
+def _register_key():
+    if os.path.isfile(f"{MAIN_WD}/.ssh/id_rsa"):
+        add_key = f"chmod 0600 {MAIN_WD}/.ssh/id_rsa && ssh-add {MAIN_WD}/.ssh/id_rsa"
         os.system(add_key)
         return True
     else: