mage-ai 0.9.44__py3-none-any.whl → 0.9.46__py3-none-any.whl

mage_ai/orchestration/pipeline_scheduler.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 import traceback
 from datetime import datetime, timedelta
+from itertools import groupby
 from typing import Any, Dict, List, Set, Tuple
 
 import pytz
@@ -30,7 +31,7 @@ from mage_ai.data_preparation.models.triggers import (
 )
 from mage_ai.data_preparation.repo_manager import get_repo_config
 from mage_ai.data_preparation.sync.git_sync import get_sync_config
-from mage_ai.orchestration.concurrency import ConcurrencyConfig
+from mage_ai.orchestration.concurrency import ConcurrencyConfig, OnLimitReached
 from mage_ai.orchestration.db import db_connection, safe_db_query
 from mage_ai.orchestration.db.models.schedules import (
     Backfill,
@@ -1347,9 +1348,22 @@ def check_sla():
 
 def schedule_all():
     """
+    This method manages the scheduling and execution of pipeline runs based on specified
+    concurrency and pipeline scheduling rules.
+
     1. Check whether any new pipeline runs need to be scheduled.
-    2. Run git sync if "sync_on_pipeline_run" is enabled.
-    3. In active pipeline runs, check whether any block runs need to be scheduled.
+    2. Group active pipeline runs by pipeline.
+    3. Run git sync if "sync_on_pipeline_run" is enabled.
+    4. For each pipeline, check whether or not any pipeline runs need to be scheduled for
+       the active pipeline schedules by performing the following steps:
+        1. Loop over pipeline schedules and acquire locks.
+        2. Determine whether to schedule pipeline runs based on pipeline schedule trigger interval.
+        3. Enforce per trigger pipeline run limit and create or cancel pipeline runs.
+        4. Start pipeline runs and handle per pipeline pipeline run limit.
+    5. In active pipeline runs, check whether any block runs need to be scheduled.
+
+    The current limit checks can potentially run into race conditions with api or event triggered
+    schedules, so that needs to be addressed at some point.
     """
     db_connection.session.expire_all()
 
@@ -1397,106 +1411,153 @@ def schedule_all():
             pr, _ = tup
             previous_pipeline_run_by_pipeline_schedule_id[pr.pipeline_schedule_id] = pr
 
-    pipeline_by_uuid = dict()
-    concurrency_config_by_uuid = dict()
-
     git_sync_result = None
     sync_config = get_sync_config()
-    for pipeline_schedule in active_pipeline_schedules:
-        lock_key = f'pipeline_schedule_{pipeline_schedule.id}'
-        if not lock.try_acquire_lock(lock_key):
-            continue
-
-        previous_runtimes = []
-        if pipeline_schedule.id in active_pipeline_schedule_ids_with_landing_time_enabled:
-            previous_pipeline_run = previous_pipeline_run_by_pipeline_schedule_id.get(
-                pipeline_schedule.id,
-            )
-            if previous_pipeline_run:
-                previous_runtimes = pipeline_schedule.runtime_history(
-                    pipeline_run=previous_pipeline_run,
-                )
-
-        # Decide whether to schedule any pipeline runs
-        should_schedule = pipeline_schedule.should_schedule(previous_runtimes=previous_runtimes)
-        initial_pipeline_runs = [
-            r for r in pipeline_schedule.pipeline_runs
-            if r.status == PipelineRun.PipelineRunStatus.INITIAL
-        ]
-
-        if not should_schedule and not initial_pipeline_runs:
-            lock.release_lock(lock_key)
-            continue
 
-        pipeline_uuid = pipeline_schedule.pipeline_uuid
-        if pipeline_uuid not in pipeline_by_uuid:
-            pipeline = pipeline_by_uuid.setdefault(pipeline_uuid, Pipeline.get(pipeline_uuid))
-        else:
-            pipeline = pipeline_by_uuid[pipeline_uuid]
+    active_pipeline_uuids = list(set([s.pipeline_uuid for s in active_pipeline_schedules]))
+    pipeline_runs_by_pipeline = PipelineRun.active_runs_for_pipelines_grouped(active_pipeline_uuids)
+
+    pipeline_schedules_by_pipeline = {
+        key: list(runs)
+        for key, runs in groupby(active_pipeline_schedules, key=lambda x: x.pipeline_uuid)
+    }
+
+    # Iterate through pipeline schedules by pipeline to handle pipeline run limits for
+    # each pipeline.
+    for pipeline_uuid, active_pipeline_schedules in pipeline_schedules_by_pipeline.items():
+        pipeline = Pipeline.get(pipeline_uuid)
+        concurrency_config = ConcurrencyConfig.load(config=pipeline.concurrency_config)
+
+        pipeline_runs_to_start = []
+        pipeline_runs_excluded_by_limit = []
+        for pipeline_schedule in active_pipeline_schedules:
+            lock_key = f'pipeline_schedule_{pipeline_schedule.id}'
+            if not lock.try_acquire_lock(lock_key):
+                continue
 
-        if pipeline_uuid not in concurrency_config_by_uuid:
-            concurrency_config = concurrency_config_by_uuid.setdefault(
-                pipeline_uuid,
-                ConcurrencyConfig.load(config=pipeline.concurrency_config),
-            )
-        else:
-            concurrency_config = concurrency_config_by_uuid[pipeline_uuid]
+            try:
+                previous_runtimes = []
+                if pipeline_schedule.id in active_pipeline_schedule_ids_with_landing_time_enabled:
+                    previous_pipeline_run = previous_pipeline_run_by_pipeline_schedule_id.get(
+                        pipeline_schedule.id,
+                    )
+                    if previous_pipeline_run:
+                        previous_runtimes = pipeline_schedule.runtime_history(
+                            pipeline_run=previous_pipeline_run,
+                        )
 
-        running_pipeline_runs = [
-            r for r in pipeline_schedule.pipeline_runs
-            if r.status == PipelineRun.PipelineRunStatus.RUNNING
-        ]
+                # Decide whether to schedule any pipeline runs
+                should_schedule = pipeline_schedule.should_schedule(
+                    previous_runtimes=previous_runtimes
+                )
+                initial_pipeline_runs = [
+                    r for r in pipeline_schedule.pipeline_runs
+                    if r.status == PipelineRun.PipelineRunStatus.INITIAL
+                ]
 
-        if should_schedule and \
-                pipeline_schedule.id not in backfills_by_pipeline_schedule_id:
-            # Perform git sync if "sync_on_pipeline_run" is enabled and no other git sync has been
-            # run for this scheduler loop.
-            if not git_sync_result and sync_config and sync_config.sync_on_pipeline_run:
-                git_sync_result = run_git_sync(lock=lock, sync_config=sync_config)
+                if not should_schedule and not initial_pipeline_runs:
+                    lock.release_lock(lock_key)
+                    continue
 
-            payload = dict(
-                execution_date=pipeline_schedule.current_execution_date(),
-                pipeline_schedule_id=pipeline_schedule.id,
-                pipeline_uuid=pipeline_uuid,
-                variables=pipeline_schedule.variables,
-            )
+                running_pipeline_runs = [
+                    r for r in pipeline_schedule.pipeline_runs
+                    if r.status == PipelineRun.PipelineRunStatus.RUNNING
+                ]
 
-            if len(previous_runtimes) >= 1:
-                payload['metrics'] = dict(previous_runtimes=previous_runtimes)
+                if should_schedule and \
+                        pipeline_schedule.id not in backfills_by_pipeline_schedule_id:
+                    # Perform git sync if "sync_on_pipeline_run" is enabled and no other git sync
+                    # has been run for this scheduler loop.
+                    if not git_sync_result and sync_config and sync_config.sync_on_pipeline_run:
+                        git_sync_result = run_git_sync(lock=lock, sync_config=sync_config)
+
+                    payload = dict(
+                        execution_date=pipeline_schedule.current_execution_date(),
+                        pipeline_schedule_id=pipeline_schedule.id,
+                        pipeline_uuid=pipeline_uuid,
+                        variables=pipeline_schedule.variables,
+                    )
 
-            if PipelineType.INTEGRATION == pipeline.type:
-                payload['create_block_runs'] = False
+                    if len(previous_runtimes) >= 1:
+                        payload['metrics'] = dict(previous_runtimes=previous_runtimes)
+
+                    if (
+                        pipeline_schedule.get_settings().skip_if_previous_running
+                        and (initial_pipeline_runs or running_pipeline_runs)
+                    ):
+                        # Cancel the pipeline run if previous pipeline runs haven't completed and
+                        # skip_if_previous_running is enabled
+                        from mage_ai.orchestration.triggers.utils import (
+                            create_and_cancel_pipeline_run,
+                        )
 
-            if pipeline_schedule.get_settings().skip_if_previous_running and \
-                    (initial_pipeline_runs or running_pipeline_runs):
-                # Cancel the current pipeline run if previous pipeline runs haven't completed
-                payload['create_block_runs'] = False
-                pipeline_run = PipelineRun.create(**payload)
-                pipeline_run.update(status=PipelineRun.PipelineRunStatus.CANCELLED)
-            else:
-                pipeline_run = PipelineRun.create(**payload)
-                # Log Git sync status for new pipeline runs if a git sync result exists
-                if git_sync_result:
-                    pipeline_scheduler = PipelineScheduler(pipeline_run)
-                    log_git_sync(
-                        git_sync_result,
-                        pipeline_scheduler.logger,
-                        pipeline_scheduler.build_tags(),
+                        pipeline_run = create_and_cancel_pipeline_run(
+                            pipeline,
+                            pipeline_schedule,
+                            payload,
+                            message='Pipeline run limit reached... skipping this run',
+                        )
+                    else:
+                        payload['create_block_runs'] = False
+                        pipeline_run = PipelineRun.create(**payload)
+                        # Log Git sync status for new pipeline runs if a git sync result exists
+                        if git_sync_result:
+                            pipeline_scheduler = PipelineScheduler(pipeline_run)
+                            log_git_sync(
+                                git_sync_result,
+                                pipeline_scheduler.logger,
+                                pipeline_scheduler.build_tags(),
+                            )
+                        initial_pipeline_runs.append(pipeline_run)
+
+                # Enforce pipeline concurrency limit
+                pipeline_run_quota = None
+                if concurrency_config.pipeline_run_limit is not None:
+                    pipeline_run_quota = concurrency_config.pipeline_run_limit - \
+                        len(running_pipeline_runs)
+
+                if pipeline_run_quota is None:
+                    pipeline_run_quota = len(initial_pipeline_runs)
+
+                if pipeline_run_quota > 0:
+                    initial_pipeline_runs.sort(key=lambda x: x.execution_date)
+                    pipeline_runs_to_start.extend(initial_pipeline_runs[:pipeline_run_quota])
+                    pipeline_runs_excluded_by_limit.extend(
+                        initial_pipeline_runs[pipeline_run_quota:]
                     )
-                initial_pipeline_runs.append(pipeline_run)
-
-        # Enforce pipeline concurrency limit
-        pipeline_run_quota = len(initial_pipeline_runs)
-        if concurrency_config.pipeline_run_limit:
-            pipeline_run_quota = concurrency_config.pipeline_run_limit - \
-                len(running_pipeline_runs)
+            finally:
+                lock.release_lock(lock_key)
 
-        if pipeline_run_quota > 0:
-            initial_pipeline_runs.sort(key=lambda x: x.execution_date)
-            for r in initial_pipeline_runs[:pipeline_run_quota]:
-                PipelineScheduler(r).start(should_schedule=False)
+        pipeline_run_limit = concurrency_config.pipeline_run_limit_all_triggers
+        if pipeline_run_limit is not None:
+            pipeline_quota = pipeline_run_limit - len(
+                pipeline_runs_by_pipeline.get(pipeline_uuid, [])
+            )
+        else:
+            pipeline_quota = None
+
+        quota_filtered_runs = pipeline_runs_to_start
+        if pipeline_quota is not None:
+            pipeline_quota = pipeline_quota if pipeline_quota > 0 else 0
+            pipeline_runs_to_start.sort(key=lambda x: x.execution_date)
+            quota_filtered_runs = pipeline_runs_to_start[:pipeline_quota]
+            pipeline_runs_excluded_by_limit.extend(
+                pipeline_runs_to_start[pipeline_quota:]
+            )
 
-        lock.release_lock(lock_key)
+        for r in quota_filtered_runs:
+            PipelineScheduler(r).start()
+
+        # If on_pipeline_run_limit_reached is set as SKIP, cancel the pipeline runs that
+        # were not scheduled due to pipeline run limits.
+        if concurrency_config.on_pipeline_run_limit_reached == OnLimitReached.SKIP:
+            for r in pipeline_runs_excluded_by_limit:
+                pipeline_scheduler = PipelineScheduler(r)
+                pipeline_scheduler.logger.warning(
+                    'Pipeline run limit reached... skipping this run',
+                    **pipeline_scheduler.build_tags(),
+                )
+                r.update(status=PipelineRun.PipelineRunStatus.CANCELLED)
 
     # Schedule active pipeline runs
     active_pipeline_runs = PipelineRun.active_runs_for_pipelines(
@@ -1517,6 +1578,17 @@ def schedule_all():
 
 
 def schedule_with_event(event: Dict = None):
+    """
+    This method manages the scheduling and execution of pipeline runs for event triggered
+    schedules. The logic is relatively similar to the `schedule_all()` method.
+
+    1. Evaluate event matchers and get active pipeline schedules for each matched event matcher.
+    2. Group matched pipeline schedules by pipeline.
+    3. Create a new pipeline run for each matched pipeline schedule.
+
+    Args:
+        event (Dict): the trigger event
+    """
     if event is None:
         event = dict()
     logger.info(f'Schedule with event {event}')
@@ -1531,12 +1603,7 @@ def schedule_with_event(event: Dict = None):
             logger.info(f'Event not matched with {e}')
 
     if len(matched_pipeline_schedules) > 0:
-        sync_config = get_sync_config()
-
-        git_sync_result = None
-        if sync_config and sync_config.sync_on_pipeline_run:
-            git_sync_result = run_git_sync(lock=lock, sync_config=sync_config)
-
+        from mage_ai.orchestration.triggers.utils import create_and_start_pipeline_run
         for p in matched_pipeline_schedules:
             payload = dict(
                 execution_date=datetime.now(tz=pytz.UTC),
@@ -1544,17 +1611,13 @@ def schedule_with_event(event: Dict = None):
                 pipeline_uuid=p.pipeline_uuid,
                 variables=merge_dict(p.variables or dict(), dict(event=event)),
             )
-            pipeline_run = PipelineRun.create(**payload)
-            pipeline_scheduler = PipelineScheduler(pipeline_run)
-
-            log_git_sync(
-                git_sync_result,
-                pipeline_scheduler.logger,
-                pipeline_scheduler.build_tags(),
+            create_and_start_pipeline_run(
+                p.pipeline,
+                p,
+                payload,
+                should_schedule=False,
             )
 
-            pipeline_scheduler.start(should_schedule=True)
-
 
 def sync_schedules(pipeline_uuids: List[str]):
     trigger_configs = []

mage_ai/orchestration/queue/process_queue.py

@@ -116,7 +116,15 @@ class ProcessQueue(Queue):
             if job_client_id != self.client_id and self.redis_client.get(job_client_id):
                 return True
         job = self.job_dict.get(job_id)
-        return job is not None and (job == JobStatus.QUEUED or isinstance(job, int))
+        return (
+            job is not None and
+            (
+                # In queue
+                (job == JobStatus.QUEUED and not self.queue.empty()) or
+                # Running
+                isinstance(job, int)
+            )
+        )
 
     def kill_job(self, job_id: str):
         """

mage_ai/orchestration/triggers/api.py

@@ -22,18 +22,21 @@ def trigger_pipeline(
     error_on_failure: bool = False,
     poll_interval: float = DEFAULT_POLL_INTERVAL,
     poll_timeout: Optional[float] = None,
+    schedule_name: str = None,
     verbose: bool = True,
+    _should_schedule: bool = False,  # For internal use only (e.g. running hooks from notebook).
 ) -> PipelineRun:
     if variables is None:
         variables = {}
     pipeline = Pipeline.get(pipeline_uuid)
 
-    pipeline_schedule = __fetch_or_create_pipeline_schedule(pipeline)
+    pipeline_schedule = __fetch_or_create_pipeline_schedule(pipeline, schedule_name=schedule_name)
 
     pipeline_run = create_and_start_pipeline_run(
         pipeline,
         pipeline_schedule,
         dict(variables=variables),
+        should_schedule=_should_schedule,
     )
 
     if check_status:
@@ -48,9 +51,14 @@ def trigger_pipeline(
     return pipeline_run
 
 
-def __fetch_or_create_pipeline_schedule(pipeline: Pipeline) -> PipelineSchedule:
+def __fetch_or_create_pipeline_schedule(
+    pipeline: Pipeline,
+    schedule_name: str = None,
+) -> PipelineSchedule:
+    if schedule_name is None:
+        schedule_name = TRIGGER_NAME_FOR_TRIGGER_CREATED_FROM_CODE
+
     pipeline_uuid = pipeline.uuid
-    schedule_name = TRIGGER_NAME_FOR_TRIGGER_CREATED_FROM_CODE
     schedule_type = ScheduleType.API
 
     pipeline_schedule = PipelineSchedule.repo_query.filter(

mage_ai/orchestration/triggers/constants.py

@@ -1,3 +1,4 @@
 DEFAULT_POLL_INTERVAL = 60
 TRIGGER_NAME_FOR_GLOBAL_DATA_PRODUCT = 'GLOBAL_DATA_PRODUCT_TRIGGER'
+TRIGGER_NAME_FOR_GLOBAL_HOOK = 'GLOBAL_HOOK_TRIGGER'
 TRIGGER_NAME_FOR_TRIGGER_CREATED_FROM_CODE = 'API_TRIGGER_CREATED_FROM_CODE'

mage_ai/orchestration/triggers/utils.py

@@ -54,6 +54,29 @@ def check_pipeline_run_status(
     return pipeline_run
 
 
+def create_and_cancel_pipeline_run(
+    pipeline: Pipeline,
+    pipeline_schedule: PipelineSchedule,
+    payload: Dict,
+    message: str = None,
+) -> PipelineRun:
+    from mage_ai.orchestration.pipeline_scheduler import PipelineScheduler
+
+    payload_copy = payload.copy()
+    configured_payload, _ = configure_pipeline_run_payload(
+        pipeline_schedule,
+        pipeline.type,
+        payload_copy,
+    )
+    configured_payload['create_block_runs'] = False
+    pipeline_run = PipelineRun.create(**configured_payload)
+    if message:
+        pipeline_scheduler = PipelineScheduler(pipeline_run)
+        pipeline_scheduler.logger.warning(message, **pipeline_scheduler.build_tags())
+    pipeline_run.update(status=PipelineRun.PipelineRunStatus.CANCELLED)
+    return pipeline_run
+
+
 @safe_db_query
 def create_and_start_pipeline_run(
     pipeline: Pipeline,

mage_ai/server/active_kernel.py

@@ -3,8 +3,13 @@ from typing import Dict
 from jupyter_client import KernelClient, KernelManager
 from jupyter_client.kernelspec import NoSuchKernel
 
+from mage_ai.data_preparation.models.project import Project
+from mage_ai.data_preparation.models.project.constants import FeatureUUID
 from mage_ai.server.kernels import DEFAULT_KERNEL_NAME, KernelName, kernel_managers
 from mage_ai.server.logger import Logger
+from mage_ai.services.spark.constants import ComputeServiceUUID
+from mage_ai.services.spark.utils import get_compute_service
+from mage_ai.services.ssh.aws.emr.utils import cluster_info_from_tunnel
 
 logger = Logger().new_server_logger(__name__)
 
@@ -68,10 +73,38 @@ def switch_active_kernel(
             from mage_ai.cluster_manager.aws.emr_cluster_manager import (
                 emr_cluster_manager,
             )
-            emr_cluster_manager.set_active_cluster(
-                auto_selection=True,
-                emr_config=emr_config,
-            )
+
+            should_set_active = True
+            auto_creation = True
+            cluster_id = None
+            project = Project()
+
+            if project.is_feature_enabled(FeatureUUID.COMPUTE_MANAGEMENT):
+                if ComputeServiceUUID.AWS_EMR == get_compute_service(
+                    emr_config=emr_config,
+                    kernel_name=kernel_name,
+                ):
+                    auto_creation = False
+                    should_set_active = False
+
+                    cluster_info = cluster_info_from_tunnel()
+
+                    if cluster_info:
+                        from mage_ai.services.compute.models import ComputeService
+
+                        cluster_id = cluster_info.get('id') or None
+                        compute_service = ComputeService(project=project)
+                        cluster = compute_service.get_cluster_details(cluster_id=cluster_id)
+
+                        should_set_active = cluster.has_dns_name if cluster else False
+
+            if should_set_active:
+                emr_cluster_manager.set_active_cluster(
+                    auto_creation=auto_creation,
+                    auto_selection=True,
+                    cluster_id=cluster_id,
+                    emr_config=emr_config,
+                )
     except NoSuchKernel as e:
         if kernel_name == KernelName.PYSPARK:
             raise Exception(

mage_ai/server/api/downloads.py

@@ -1,3 +1,8 @@
+import os
+import tempfile
+import zipfile
+
+import jwt
 from tornado import gen, iostream
 
 from mage_ai.api.utils import authenticate_client_and_token
@@ -6,7 +11,8 @@ from mage_ai.data_preparation.models.variable import VariableType
 from mage_ai.orchestration.db.models.oauth import Oauth2Application
 from mage_ai.orchestration.db.models.schedules import PipelineRun
 from mage_ai.server.api.base import BaseHandler
-from mage_ai.settings import REQUIRE_USER_AUTHENTICATION
+from mage_ai.settings import JWT_DOWNLOAD_SECRET, REQUIRE_USER_AUTHENTICATION
+from mage_ai.settings.repo import get_repo_path
 
 
 class ApiDownloadHandler(BaseHandler):
@@ -62,3 +68,72 @@ class ApiDownloadHandler(BaseHandler):
                     await self.flush()
                     # Sleep for a nanosecond so other handlers can run and avoid blocking
                     await gen.sleep(0.000000001)
+
+
+class ApiResourceDownloadHandler(BaseHandler):
+
+    def get(self, token):
+        try:
+            decoded_payload = jwt.decode(token, JWT_DOWNLOAD_SECRET, algorithms=['HS256'])
+
+            file_name = decoded_payload['file_name']
+            file_list = decoded_payload['file_list']
+            self.ignore_folder_structure = decoded_payload['ignore_folder_structure']
+
+            self.abs_repo_path = os.path.abspath(get_repo_path())
+
+            relative_file_list = list(map(self.relative_path_mapping, file_list))
+
+            try:
+                file_pointer = self.get_file_pointer(file_list, relative_file_list)
+
+                while True:
+                    _buffer = file_pointer.read(4096)
+                    if not _buffer:
+                        break
+                    self.write(_buffer)
+            except Exception as e:
+                self.set_status(400)
+                self.write(f'Error fetching file {file_name}.\n{e}')
+            finally:
+                file_pointer.close()
+
+            self.set_header('Content-Type', 'application/force-download')
+            self.set_header('Content-Disposition', f'attachment; filename={file_name}')
+            self.flush()
+        except jwt.exceptions.ExpiredSignatureError:
+            self.set_status(400)
+            self.write('Download token is expired.')
+        except (jwt.exceptions.InvalidSignatureError, jwt.exceptions.DecodeError):
+            self.set_status(400)
+            self.write('Download token is invalid.')
+        except ValueError as e:
+            self.set_status(400)
+            self.write(f'Attepmt at fetching file outside of project folder: {e}')
+
+    # file pointer points to either a singular file or a temporary zip
+    def get_file_pointer(self, file_list, relative_file_list):
+        if len(file_list) == 1:
+            return open(file_list[0])
+        return self.zip_files(file_list, relative_file_list)
+
+    # creates a temporary zip and returns the (open) file pointer
+    def zip_files(self, file_list, relative_file_list):
+        zip_file = tempfile.NamedTemporaryFile(suffix='.zip', mode='w+b')
+        with zipfile.ZipFile(zip_file, 'w') as zipf:
+            for path, relative in zip(file_list, relative_file_list):
+                zipf.write(path, relative)
+        zip_file.seek(0)  # set cursor to start of file to prepare for content extraction
+        return zip_file
+
+    def relative_path_mapping(self, path):
+        abs_path = os.path.abspath(path)
+        common_ground = os.path.commonpath([self.abs_repo_path, abs_path])
+
+        # trying to access files outside of the project folder
+        if common_ground != self.abs_repo_path:
+            raise ValueError(abs_path)
+
+        return (os.path.basename(abs_path)
+                if self.ignore_folder_structure
+                else os.path.relpath(abs_path, common_ground))

mage_ai/server/api/triggers.py

@@ -48,6 +48,7 @@ class ApiTriggerPipelineHandler(BaseHandler):
             pipeline,
             pipeline_schedule,
             payload,
+            should_schedule=False,
         )
 
         self.write(dict(pipeline_run=pipeline_run.to_dict()))

mage_ai/server/constants.py

@@ -12,4 +12,4 @@ DATAFRAME_OUTPUT_SAMPLE_COUNT = 10
 # Dockerfile depends on it because it runs ./scripts/install_mage.sh and uses
 # the last line to determine the version to install.
 VERSION = \
-'0.9.44'
+'0.9.46'

mage_ai/server/frontend_dist/404.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=0" name="viewport"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><link href="/favicon.ico" rel="icon"/><link rel="preload" href="/_next/static/css/d1e8e64d0b07af2f.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d1e8e64d0b07af2f.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="/_next/static/chunks/webpack-fea697dd168c6d0c.js" defer=""></script><script src="/_next/static/chunks/framework-7c365855dab1bf41.js" defer=""></script><script src="/_next/static/chunks/main-77fe248a6fbd12d8.js" defer=""></script><script src="/_next/static/chunks/pages/_app-a4a660fa14cf05a9.js" defer=""></script><script src="/_next/static/chunks/pages/_error-e989623bffcbf724.js" defer=""></script><script src="/_next/static/RPXiX8RpZ7oO-yeaEtV2c/_buildManifest.js" defer=""></script><script src="/_next/static/RPXiX8RpZ7oO-yeaEtV2c/_ssgManifest.js" defer=""></script><style data-styled="" data-styled-version="5.3.6">html{-webkit-box-sizing:border-box;box-sizing:border-box;-ms-overflow-style:scrollbar;}/*!sc*/
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=0" name="viewport"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><link href="/favicon.ico" rel="icon"/><link rel="preload" href="/_next/static/css/d1e8e64d0b07af2f.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d1e8e64d0b07af2f.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="/_next/static/chunks/webpack-fea697dd168c6d0c.js" defer=""></script><script src="/_next/static/chunks/framework-22b71764bf44ede4.js" defer=""></script><script src="/_next/static/chunks/main-77fe248a6fbd12d8.js" defer=""></script><script src="/_next/static/chunks/pages/_app-ebef928183f9a3bb.js" defer=""></script><script src="/_next/static/chunks/pages/_error-e989623bffcbf724.js" defer=""></script><script src="/_next/static/9jB4XPuz6BzxBcG9VNao5/_buildManifest.js" defer=""></script><script src="/_next/static/9jB4XPuz6BzxBcG9VNao5/_ssgManifest.js" defer=""></script><style data-styled="" data-styled-version="5.3.6">html{-webkit-box-sizing:border-box;box-sizing:border-box;-ms-overflow-style:scrollbar;}/*!sc*/
 *,*::before,*::after{-webkit-box-sizing:inherit;box-sizing:inherit;}/*!sc*/
 data-styled.g4[id="sc-global-czSCUT1"]{content:"sc-global-czSCUT1,"}/*!sc*/
 .kOVcuR .Toastify__toast-container{margin-top:24px;padding:0 !important;width:500px !important;}/*!sc*/
@@ -20,4 +20,4 @@ data-styled.g5[id="ToastWrapper-sc-1a33ph1-0"]{content:"kOVcuR,"}/*!sc*/
                   .next-error-h1 {
                     border-right: 1px solid rgba(255, 255, 255, .3);
                   }
-                }</style><h1 class="next-error-h1" style="display:inline-block;margin:0;margin-right:20px;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404<!-- --></h1><div style="display:inline-block;text-align:left;line-height:49px;height:49px;vertical-align:middle"><h2 style="font-size:14px;font-weight:normal;line-height:49px;margin:0;padding:0">This page could not be found<!-- -->.<!-- --></h2></div></div></div><div></div><div></div><div></div><div class="ToastWrapper-sc-1a33ph1-0 kOVcuR"><div class="Toastify"></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404},"currentTheme":{"accent":{"alert":"#F6540B","blue":"#4877FF","blueLight":"rgba(72, 119, 255, 0.5)","contentDefaultTransparent":"rgba(174, 174, 174, 0.5)","cyan":"#65E3FF","cyanLight":"rgba(101, 227, 255, 0.3)","cyanTransparent":"rgba(101, 227, 255, 0.12)","dbt":"#fc6949","dbtLight":"rgba(252, 105, 73, 0.5)","info":"#00ABFF","infoTransparent":"rgba(0, 171, 255, 0.5)","negative":"#FF1E59","negativeTransparent":"rgba(255, 30, 89, 0.3)","pink":"#FF4FF8","pinkLight":"rgb(255, 79, 248, 0.5)","positive":"#00A81A","primaryTransparent":"rgba(155, 108, 167, 0.5)","purple":"#7D55EC","purpleLight":"rgba(125, 85, 236, 0.5)","rose":"#D1A2AB","roseLight":"rgba(209, 162, 171, 0.5)","sky":"#6AA1E0","skyLight":"rgba(106, 161, 224, 0.05)","teal":"#00B4CC","tealLight":"rgba(0, 180, 204, 0.5)","warning":"#DD9900","warningTransparent":"rgba(221, 153, 0, 0.5)","yellow":"#FFCC19","yellowLight":"rgba(255, 204, 25, 0.5)"},"background":{"chartBlock":"#2E3036","codeArea":"#1E1F24","codeTextarea":"#000000","content":"#1B1C20","danger":"#FFD0DB","dark":"#B1B8C3","dashboard":"#18181C","header":"#1B1B1B","menu":"#0F4CFF","muted":"#F9FAFC","navigation":"#EDEDED","output":"#2E3036","page":"#1E1F24","panel":"#232429","popup":"#27292E","row":"#2C2C2C","row2":"#51535C","scrollbarThumb":"rgba(100, 100, 100, 0.5)","scrollbarThumbHover":"rgba(255, 255, 255, 0.3)","scrollbarTrack":"#2E3036","success":"#8ADE00","successLight":"rgb(138, 222, 0, 0.3)","table":"#292A2F","tag":"#3A4550"},"borders":{"button":"#454850","contrast":"#FFFFFF","danger":"#FF144D","dark":"#000000","darkLight":"#2E3036","info":"#FFCC19","light":"#2F3034","medium":"#1C1C1C","medium2":"#141414","success":"#2FCB52"},"brand":{"earth100":"#C6EEDB","earth200":"#9DDFBF","earth300":"#6BBF96","earth400":"#37A46F","earth400Transparent":"rgba(55, 164, 111, 0.4)","earth500":"#00954C","energy100":"#FFF4BA","energy200":"#FFED92","energy300":"#FFE662","energy400":"#FFDA19","energy400Transparent":"rgba(255, 218, 25, 0.04)","energy500":"#F6C000","fire100":"#FFD7E0","fire200":"#FFA3B9","fire300":"#FF547D","fire400":"#FF144D","fire400Transparent":"rgba(255, 20, 77, 0.4)","fire500":"#EB0032","stone100":"#F3E6D7","stone200":"#E3D4C2","stone400":"#BFA78B","stone500":"#AF8859","water100":"#BDCEFF","water200":"#81A1FF","water300":"#517DFF","water400":"#2A60FE","water400Transparent":"rgba(42, 96, 254, 0.4)","water500":"#0F4CFF","wind100":"#EEEAFF","wind200":"#CCC1F4","wind300":"#A698DD","wind400":"#6B50D7","wind400SuperTransparent":"rgba(107, 80, 215, 0.12)","wind400Transparent":"rgba(107, 80, 215, 0.4)","wind500":"#4E32BC"},"chart":{"backgroundPrimary":"#7D55EC","backgroundSecondary":"#FF144D","backgroundTertiary":"#86E2FF","button1":"#4877FF","button2":"#FFCC19","button3":"#8ADE00","button4":"#FF4FF8","button5":"#B98D95","lines":"#9B6CA7","primary":"#6B50D7","secondary":"#FF144D","tertiary":"#2A60FE"},"content":{"active":"#FFFFFF","default":"#AEAEAE","disabled":"rgba(255, 255, 255, 0.3)","inverted":"#2C2C2C","muted":"#787A85"},"elevation":{"visualizationAccent":"#996CFF","visualizationAccentAlt":"#C1ACF7"},"feature":{"active":"rgba(250, 248, 254, 0.14)","disabled":"rgba(201, 206, 218, 0.12)"},"icons":{"neutral":"#787878"},"interactive":{"activeBorder":"#060606","checked":"#060606","dangerBorder":"#FF144D","defaultBackground":"#36383F","defaultBorder":"#ffffff1a","disabledBorder":"#B1B8C3","focusBackground":"#B1B8C3","focusBorder":"#86E2FF","hoverBackground":"#4E4E4E","hoverBorder":"#B9BFCA","hoverOverlay":"rgba(255, 255, 255, 0.1)","linkPrimary":"#1752FF","linkPrimaryHover":"#4877FF","linkPrimaryLight":"#5982ff","linkSecondary":"#6B50D7","linkSecondaryDisabled":"#C4B9EF","linkText":"#6AA1E0","linkTextLight":"#9ECBFF","purple":"#885EFF","rowHoverBackground":"rgba(0, 0, 0, 0.1)","transparent":"rgba(255, 255, 255, 0)"},"loader":{"color":"#EB0032","colorInverted":"#8ADE00"},"logo":{"color":"#FFFFFF"},"monotone":{"black":"#060606","blackTransparent":"rgba(0, 0, 0, 0.6)","gray":"#B1B8C3","grey100":"#F2F2F2","grey200":"#D5D7DC","grey300":"#B4B8C0","grey400":"#70747C","grey500":"#51535C","purple":"#6B50D7","white":"#FFFFFF","whiteTransparent":"rgba(255, 255, 255, 0.6)"},"neutral":{"n100":"#E7E8EA","n200":"#D8DADE","n300":"#CBCCD0","n400":"#BCBEC4","n500":"#AEB0B6"},"progress":{"negative":"#FF144D","positive":"#6B50D7"},"shadow":{"base":"12px 40px 120px rgba(0, 0, 0, 0.3)","frame":"0px 10px 40px rgba(0, 0, 0, 0.26)","menu":"4px 10px 20px rgba(6, 6, 6, 0.12)","popup":"10px 20px 40px rgba(0, 0, 0, 0.2)","small":"0px, 4px, rgba(0, 0, 0, 0.25)","window":"0px 10px 60px rgba(0, 0, 0, 0.7)"},"status":{"negative":"#FF144D","positive":"#24B400"},"text":{"fileBrowser":"#787A85"}}},"page":"/_error","query":{},"buildId":"RPXiX8RpZ7oO-yeaEtV2c","nextExport":true,"isFallback":false,"gip":true,"appGip":true,"scriptLoader":[]}</script></body></html>
+                }</style><h1 class="next-error-h1" style="display:inline-block;margin:0;margin-right:20px;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404<!-- --></h1><div style="display:inline-block;text-align:left;line-height:49px;height:49px;vertical-align:middle"><h2 style="font-size:14px;font-weight:normal;line-height:49px;margin:0;padding:0">This page could not be found<!-- -->.<!-- --></h2></div></div></div><div></div><div></div><div></div><div class="ToastWrapper-sc-1a33ph1-0 kOVcuR"><div class="Toastify"></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404},"currentTheme":{"accent":{"alert":"#F6540B","blue":"#4877FF","blueLight":"rgba(72, 119, 255, 0.5)","contentDefaultTransparent":"rgba(174, 174, 174, 0.5)","cyan":"#65E3FF","cyanLight":"rgba(101, 227, 255, 0.3)","cyanTransparent":"rgba(101, 227, 255, 0.12)","dbt":"#fc6949","dbtLight":"rgba(252, 105, 73, 0.5)","info":"#00ABFF","infoTransparent":"rgba(0, 171, 255, 0.5)","negative":"#FF1E59","negativeTransparent":"rgba(255, 30, 89, 0.3)","pink":"#FF4FF8","pinkLight":"rgb(255, 79, 248, 0.5)","positive":"#00A81A","primaryTransparent":"rgba(155, 108, 167, 0.5)","purple":"#7D55EC","purpleLight":"rgba(125, 85, 236, 0.5)","rose":"#D1A2AB","roseLight":"rgba(209, 162, 171, 0.5)","sky":"#6AA1E0","skyLight":"rgba(106, 161, 224, 0.05)","teal":"#00B4CC","tealLight":"rgba(0, 180, 204, 0.5)","warning":"#DD9900","warningTransparent":"rgba(221, 153, 0, 0.5)","yellow":"#FFCC19","yellowLight":"rgba(255, 204, 25, 0.5)"},"background":{"chartBlock":"#2E3036","codeArea":"#1E1F24","codeTextarea":"#000000","content":"#1B1C20","danger":"#FFD0DB","dark":"#B1B8C3","dashboard":"#18181C","header":"#1B1B1B","menu":"#0F4CFF","muted":"#F9FAFC","navigation":"#EDEDED","output":"#2E3036","page":"#1E1F24","panel":"#232429","popup":"#27292E","row":"#2C2C2C","row2":"#51535C","scrollbarThumb":"rgba(100, 100, 100, 0.5)","scrollbarThumbHover":"rgba(255, 255, 255, 0.3)","scrollbarTrack":"#2E3036","success":"#8ADE00","successLight":"rgb(138, 222, 0, 0.3)","table":"#292A2F","tag":"#3A4550"},"borders":{"button":"#454850","contrast":"#FFFFFF","danger":"#FF144D","dark":"#000000","darkLight":"#2E3036","info":"#FFCC19","light":"#2F3034","medium":"#1C1C1C","medium2":"#141414","success":"#2FCB52"},"brand":{"earth100":"#C6EEDB","earth200":"#9DDFBF","earth300":"#6BBF96","earth400":"#37A46F","earth400Transparent":"rgba(55, 164, 111, 0.4)","earth500":"#00954C","energy100":"#FFF4BA","energy200":"#FFED92","energy300":"#FFE662","energy400":"#FFDA19","energy400Transparent":"rgba(255, 218, 25, 0.04)","energy500":"#F6C000","fire100":"#FFD7E0","fire200":"#FFA3B9","fire300":"#FF547D","fire400":"#FF144D","fire400Transparent":"rgba(255, 20, 77, 0.4)","fire500":"#EB0032","stone100":"#F3E6D7","stone200":"#E3D4C2","stone400":"#BFA78B","stone500":"#AF8859","water100":"#BDCEFF","water200":"#81A1FF","water300":"#517DFF","water400":"#2A60FE","water400Transparent":"rgba(42, 96, 254, 0.4)","water500":"#0F4CFF","wind100":"#EEEAFF","wind200":"#CCC1F4","wind300":"#A698DD","wind400":"#6B50D7","wind400SuperTransparent":"rgba(107, 80, 215, 0.12)","wind400Transparent":"rgba(107, 80, 215, 0.4)","wind500":"#4E32BC"},"chart":{"backgroundPrimary":"#7D55EC","backgroundSecondary":"#FF144D","backgroundTertiary":"#86E2FF","button1":"#4877FF","button2":"#FFCC19","button3":"#8ADE00","button4":"#FF4FF8","button5":"#B98D95","lines":"#9B6CA7","primary":"#6B50D7","secondary":"#FF144D","tertiary":"#2A60FE"},"content":{"active":"#FFFFFF","default":"#AEAEAE","disabled":"rgba(255, 255, 255, 0.3)","inverted":"#2C2C2C","muted":"#787A85"},"elevation":{"visualizationAccent":"#996CFF","visualizationAccentAlt":"#C1ACF7"},"feature":{"active":"rgba(250, 248, 254, 0.14)","disabled":"rgba(201, 206, 218, 0.12)"},"icons":{"neutral":"#787878"},"interactive":{"activeBorder":"#060606","checked":"#060606","dangerBorder":"#FF144D","defaultBackground":"#36383F","defaultBorder":"#ffffff1a","disabledBorder":"#B1B8C3","focusBackground":"#B1B8C3","focusBorder":"#86E2FF","hoverBackground":"#4E4E4E","hoverBorder":"#B9BFCA","hoverOverlay":"rgba(255, 255, 255, 0.1)","linkPrimary":"#1752FF","linkPrimaryHover":"#4877FF","linkPrimaryLight":"#5982ff","linkSecondary":"#6B50D7","linkSecondaryDisabled":"#C4B9EF","linkText":"#6AA1E0","linkTextLight":"#9ECBFF","purple":"#885EFF","rowHoverBackground":"rgba(0, 0, 0, 0.1)","transparent":"rgba(255, 255, 255, 0)"},"loader":{"color":"#EB0032","colorInverted":"#8ADE00"},"logo":{"color":"#FFFFFF"},"monotone":{"black":"#060606","blackTransparent":"rgba(0, 0, 0, 0.6)","gray":"#B1B8C3","grey100":"#F2F2F2","grey200":"#D5D7DC","grey300":"#B4B8C0","grey400":"#70747C","grey500":"#51535C","purple":"#6B50D7","white":"#FFFFFF","whiteTransparent":"rgba(255, 255, 255, 0.6)"},"neutral":{"n100":"#E7E8EA","n200":"#D8DADE","n300":"#CBCCD0","n400":"#BCBEC4","n500":"#AEB0B6"},"progress":{"negative":"#FF144D","positive":"#6B50D7"},"shadow":{"base":"12px 40px 120px rgba(0, 0, 0, 0.3)","frame":"0px 10px 40px rgba(0, 0, 0, 0.26)","menu":"4px 10px 20px rgba(6, 6, 6, 0.12)","popup":"10px 20px 40px rgba(0, 0, 0, 0.2)","small":"0px, 4px, rgba(0, 0, 0, 0.25)","window":"0px 10px 60px rgba(0, 0, 0, 0.7)"},"status":{"negative":"#FF144D","positive":"#24B400"},"text":{"fileBrowser":"#787A85"}}},"page":"/_error","query":{},"buildId":"9jB4XPuz6BzxBcG9VNao5","nextExport":true,"isFallback":false,"gip":true,"appGip":true,"scriptLoader":[]}</script></body></html>