mage-ai 0.9.7__py3-none-any.whl → 0.9.9__py3-none-any.whl

mage_ai/data_preparation/templates/constants.py

@@ -1,8 +1,22 @@
 from mage_ai.data_preparation.models.constants import BlockLanguage, BlockType
+from mage_ai.io.base import DataSource
 from mage_ai.shared.hash import index_by
 
+GROUP_AGGREGATE = 'Aggregate'
+GROUP_COLUMN_ACTIONS = 'Column actions'
+GROUP_COLUMN_REMOVAL = 'Column removal'
+GROUP_DATABASES = 'Databases'
+GROUP_DATABASES_NO_SQL = 'Databases (NoSQL)'
+GROUP_DATA_CLEANING = 'Data cleaning'
+GROUP_DATA_LAKES = 'Data lakes'
+GROUP_DATA_WAREHOUSES = 'Data warehouses'
 GROUP_DELTA_LAKE = 'Delta Lake'
+GROUP_FEATURE_EXTRACTION = 'Feature extraction'
+GROUP_FEATURE_SCALING = 'Feature scaling'
+GROUP_FORMATTING = 'Formatting'
 GROUP_ORCHESTRATION = 'Orchestration'
+GROUP_ROW_ACTIONS = 'Row actions'
+GROUP_SHIFT = 'Shift'
 
 TEMPLATES = [
     dict(
@@ -32,6 +46,7 @@ TEMPLATES = [
     dict(
         block_type=BlockType.DATA_LOADER,
         description='Load data from MongoDB.',
+        groups=[GROUP_DATABASES_NO_SQL],
         language=BlockLanguage.PYTHON,
         name='MongoDB',
         path='data_loaders/mongodb.py',
@@ -107,4 +122,466 @@ TEMPLATES = [
     ),
 ]
 
-TEMPLATES_BY_UUID = index_by(lambda x: x['name'], TEMPLATES)
+TEMPLATES_ONLY_FOR_V2 = [
+    # Data loaders
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        language=BlockLanguage.PYTHON,
+        name='Base template (generic)',
+        path='data_loaders/default.jinja',
+    ),
+    #   Data lakes
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon S3',
+        path='data_loaders/s3.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Azure Blob Storage',
+        path='data_loaders/azure_blob_storage.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Google Cloud Storage',
+        path='data_loaders/google_cloud_storage.py',
+    ),
+    #   Data warehouses
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon Redshift',
+        path='data_loaders/redshift.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        description='Load data from Google BigQuery.',
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Google BigQuery',
+        path='data_loaders/bigquery.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Snowflake',
+        path='data_loaders/snowflake.py',
+    ),
+    #   Databases
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='MySQL',
+        path='data_loaders/mysql.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='Oracle DB',
+        path='data_loaders/oracledb.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='PostgreSQL',
+        path='data_loaders/postgres.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        description='Fetch data from an API request.',
+        language=BlockLanguage.PYTHON,
+        name='API',
+        path='data_loaders/api.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        description='Load data from a file on your machine.',
+        language=BlockLanguage.PYTHON,
+        name='Local file',
+        path='data_loaders/file.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_LOADER,
+        language=BlockLanguage.PYTHON,
+        name='Druid',
+        path='data_loaders/druid.py',
+    ),
+    # Transformers
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        language=BlockLanguage.PYTHON,
+        name='Base template (generic)',
+        path='transformers/default.jinja',
+    ),
+    #   Data warehouses
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon Redshift',
+        path='transformers/data_warehouse_transformer.jinja',
+        template_variables=dict(
+            additional_args='\n        loader.commit() # Permanently apply database changes',
+            data_source=DataSource.REDSHIFT.value,
+            data_source_handler='Redshift',
+        ),
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Google BigQuery',
+        path='transformers/data_warehouse_transformer.jinja',
+        template_variables=dict(
+            additional_args='',
+            data_source=DataSource.BIGQUERY.value,
+            data_source_handler='BigQuery',
+        ),
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Snowflake',
+        path='transformers/data_warehouse_transformer.jinja',
+        template_variables=dict(
+            additional_args='\n        loader.commit() # Permanently apply database changes',
+            data_source=DataSource.SNOWFLAKE.value,
+            data_source_handler='Snowflake',
+        ),
+    ),
+    #   Databases
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='PostgreSQL',
+        path='transformers/data_warehouse_transformer.jinja',
+        template_variables=dict(
+            additional_args='\n        loader.commit() # Permanently apply database changes',
+            data_source=DataSource.POSTGRES.value,
+            data_source_handler='Postgres',
+        ),
+    ),
+    #   Row actions
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_ROW_ACTIONS],
+        language=BlockLanguage.PYTHON,
+        name='Drop duplicate rows',
+        path='transformers/transformer_actions/row/drop_duplicate.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_ROW_ACTIONS],
+        language=BlockLanguage.PYTHON,
+        name='Filter rows',
+        path='transformers/transformer_actions/row/filter.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_ROW_ACTIONS],
+        language=BlockLanguage.PYTHON,
+        name='Remove rows',
+        path='transformers/transformer_actions/row/remove.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_ROW_ACTIONS],
+        language=BlockLanguage.PYTHON,
+        name='Sort rows',
+        path='transformers/transformer_actions/row/sort.py',
+    ),
+    #   Column actions
+    #       Aggregate
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Average value of column',
+        path='transformers/transformer_actions/column/average.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Count unique values in column',
+        path='transformers/transformer_actions/column/count_distinct.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='First value in column',
+        path='transformers/transformer_actions/column/first.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Last value in column',
+        path='transformers/transformer_actions/column/last.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Maximum value in column',
+        path='transformers/transformer_actions/column/max.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Median value in column',
+        path='transformers/transformer_actions/column/median.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Min value in column',
+        path='transformers/transformer_actions/column/min.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Sum of all values in column',
+        path='transformers/transformer_actions/column/sum.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_AGGREGATE],
+        language=BlockLanguage.PYTHON,
+        name='Total count of values in column',
+        path='transformers/transformer_actions/column/count.py',
+    ),
+    #       Formatting
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_FORMATTING],
+        language=BlockLanguage.PYTHON,
+        name='Clean column name',
+        path='transformers/transformer_actions/column/clean_column_name.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_FORMATTING],
+        language=BlockLanguage.PYTHON,
+        name='Fix syntax errors',
+        path='transformers/transformer_actions/column/fix_syntax_errors.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_FORMATTING],
+        language=BlockLanguage.PYTHON,
+        name='Reformat values in column',
+        path='transformers/transformer_actions/column/reformat.py',
+    ),
+    #       Column removal
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_COLUMN_REMOVAL],
+        language=BlockLanguage.PYTHON,
+        name='Keep column(s)',
+        path='transformers/transformer_actions/column/select.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_COLUMN_REMOVAL],
+        language=BlockLanguage.PYTHON,
+        name='Remove column(s)',
+        path='transformers/transformer_actions/column/remove.py',
+    ),
+    #       Shift
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_SHIFT],
+        language=BlockLanguage.PYTHON,
+        name='Shift row values down',
+        path='transformers/transformer_actions/column/shift_down.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_SHIFT],
+        language=BlockLanguage.PYTHON,
+        name='Shift row values up',
+        path='transformers/transformer_actions/column/shift_up.py',
+    ),
+    #       Feature scaling
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_FEATURE_SCALING],
+        language=BlockLanguage.PYTHON,
+        name='Normalize data',
+        path='transformers/transformer_actions/column/normalize.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_FEATURE_SCALING],
+        language=BlockLanguage.PYTHON,
+        name='Standardize data',
+        path='transformers/transformer_actions/column/standardize.py',
+    ),
+    #       Data cleaning
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_DATA_CLEANING],
+        language=BlockLanguage.PYTHON,
+        name='Fill in missing values',
+        path='transformers/transformer_actions/column/impute.py',
+    ),
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_DATA_CLEANING],
+        language=BlockLanguage.PYTHON,
+        name='Remove outliers',
+        path='transformers/transformer_actions/column/remove_outliers.py',
+    ),
+    #       Feature extraction
+    dict(
+        block_type=BlockType.TRANSFORMER,
+        groups=[GROUP_COLUMN_ACTIONS, GROUP_FEATURE_EXTRACTION],
+        language=BlockLanguage.PYTHON,
+        name='Calculate difference between values',
+        path='transformers/transformer_actions/column/diff.py',
+    ),
+    # Data exporters
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        language=BlockLanguage.PYTHON,
+        name='Base template (generic)',
+        path='data_exporters/default.jinja',
+    ),
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        language=BlockLanguage.PYTHON,
+        name='Local file',
+        path='data_exporters/file.py',
+    ),
+    #   Data lakes
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon S3',
+        path='data_exporters/s3.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Azure Blob Storage',
+        path='data_exporters/azure_blob_storage.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Google Cloud Storage',
+        path='data_exporters/google_cloud_storage.py',
+    ),
+    #   Data warehouses
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon Redshift',
+        path='data_exporters/redshift.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Google BigQuery',
+        path='data_exporters/bigquery.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Snowflake',
+        path='data_exporters/snowflake.py',
+    ),
+    #   Databases
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='MySQL',
+        path='data_exporters/mysql.py',
+    ),
+    dict(
+        block_type=BlockType.DATA_EXPORTER,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='PostgreSQL',
+        path='data_exporters/postgres.py',
+    ),
+    # Sensors
+    dict(
+        block_type=BlockType.SENSOR,
+        language=BlockLanguage.PYTHON,
+        name='Base template (generic)',
+        path='sensors/default.py',
+    ),
+    #   Data lakes
+    dict(
+        block_type=BlockType.SENSOR,
+        groups=[GROUP_DATA_LAKES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon S3',
+        path='sensors/s3.py',
+    ),
+    #   Data warehouses
+    dict(
+        block_type=BlockType.SENSOR,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Amazon Redshift',
+        path='sensors/redshift.py',
+    ),
+    dict(
+        block_type=BlockType.SENSOR,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Google BigQuery',
+        path='sensors/bigquery.py',
+    ),
+    dict(
+        block_type=BlockType.SENSOR,
+        groups=[GROUP_DATA_WAREHOUSES],
+        language=BlockLanguage.PYTHON,
+        name='Snowflake',
+        path='sensors/snowflake.py',
+    ),
+    #   Databases
+    dict(
+        block_type=BlockType.SENSOR,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='MySQL',
+        path='sensors/mysql.py',
+    ),
+    dict(
+        block_type=BlockType.SENSOR,
+        groups=[GROUP_DATABASES],
+        language=BlockLanguage.PYTHON,
+        name='PostgreSQL',
+        path='sensors/postgres.py',
+    ),
+]
+
+TEMPLATES_BY_UUID = index_by(lambda x: x['name'], TEMPLATES + TEMPLATES_ONLY_FOR_V2)

mage_ai/data_preparation/templates/template.py

@@ -59,9 +59,17 @@ def fetch_template_source(
         return template_source
 
     if 'template_path' in config:
+        template_variables_to_render = dict(
+            code=config.get('existing_code', ''),
+        )
+
+        template_variables = config.get('template_variables')
+        if template_variables:
+            template_variables_to_render.update(template_variables)
+
         return (
             template_env.get_template(config['template_path']).render(
-                code=config.get('existing_code', ''),
+                **template_variables_to_render,
             )
         )
     elif block_type == BlockType.DATA_LOADER:

mage_ai/orchestration/backfills/service.py

@@ -84,10 +84,25 @@ def __build_variables_list(backfill: Backfill) -> List[Dict]:
         interval_end_datetime = None
         interval_seconds = None
         interval_start_datetime = execution_date
+        interval_start_datetime_previous = None
 
         if idx < number_of_dates - 1:
             interval_end_datetime = dates[idx + 1]
-            interval_seconds = interval_end_datetime.timestamp() - execution_date.timestamp()
+            interval_seconds = (
+                interval_end_datetime.timestamp() - interval_start_datetime.timestamp()
+            )
+        elif idx >= 1 and idx == number_of_dates - 1:
+            # Last date
+            interval_start_datetime_previous = dates[idx - 1]
+            interval_seconds = (
+                interval_start_datetime.timestamp() - interval_start_datetime_previous.timestamp()
+            )
+            interval_end_datetime = interval_start_datetime + timedelta(seconds=interval_seconds)
+
+        if not interval_start_datetime_previous and (interval_seconds and interval_start_datetime):
+            interval_start_datetime_previous = interval_start_datetime - timedelta(
+                seconds=interval_seconds,
+            )
 
         if interval_end_datetime:
             interval_end_datetime = interval_end_datetime.isoformat()
@@ -95,6 +110,9 @@ def __build_variables_list(backfill: Backfill) -> List[Dict]:
         if interval_start_datetime:
             interval_start_datetime = interval_start_datetime.isoformat()
 
+        if interval_start_datetime_previous:
+            interval_start_datetime_previous = interval_start_datetime_previous.isoformat()
+
         arr.append(dict(
             ds=execution_date.strftime('%Y-%m-%d'),
             execution_date=execution_date.isoformat(),
@@ -102,6 +120,7 @@ def __build_variables_list(backfill: Backfill) -> List[Dict]:
             interval_end_datetime=interval_end_datetime,
             interval_seconds=interval_seconds,
             interval_start_datetime=interval_start_datetime,
+            interval_start_datetime_previous=interval_start_datetime_previous,
         ))
 
     return arr

mage_ai/orchestration/db/__init__.py

@@ -4,6 +4,7 @@ import os
 import sqlalchemy
 from sqlalchemy import create_engine
 from sqlalchemy.orm import scoped_session, sessionmaker
+from urllib.parse import parse_qs, urlparse
 
 from mage_ai.data_preparation.repo_manager import get_variables_dir
 from mage_ai.orchestration.constants import (
@@ -71,8 +72,32 @@ class DBConnection:
         self.session = None
 
 
+def get_postgresql_schema(url):
+    parse_result = urlparse(url)
+    if parse_result.scheme == 'postgresql+psycopg2':
+        q = parse_qs(
+            parse_result.query.replace('%%', '%')
+        )
+        options = q.get('options')
+        if options and len(options) >= 1:
+            params = options[0].replace('-c ', '').split(' ')
+            kvs = dict(p.split('=') for p in params)
+            return kvs.get('search_path')
+
+
 db_connection = DBConnection()
 
+if db_connection_url.startswith('postgresql'):
+    db_schema = get_postgresql_schema(db_connection_url)
+    if db_schema:
+        db_connection.start_session()
+        db_connection.session.execute(f'CREATE SCHEMA IF NOT EXISTS {db_schema};')
+        db_connection.session.commit()
+        db_connection.close_session()
+        print(f'Set the default PostgreSQL schema to {db_schema}')
+    else:
+        print('No schema in PostgreSQL connection URL: use the default "public" schema')
+
 
 def safe_db_query(func):
     def func_with_rollback(*args, **kwargs):

mage_ai/orchestration/db/models/schedules.py

@@ -473,8 +473,12 @@ class PipelineRun(BaseModel):
         interval_end_datetime = variables.get('interval_end_datetime')
         interval_seconds = variables.get('interval_seconds')
         interval_start_datetime = variables.get('interval_start_datetime')
+        interval_start_datetime_previous = variables.get('interval_start_datetime_previous')
 
-        if interval_end_datetime or interval_seconds or interval_start_datetime:
+        if interval_end_datetime or \
+                interval_seconds or \
+                interval_start_datetime or \
+                interval_start_datetime_previous:
             if interval_end_datetime:
                 try:
                     variables['interval_end_datetime'] = dateutil.parser.parse(
@@ -490,10 +494,19 @@ class PipelineRun(BaseModel):
                     )
                 except Exception as err:
                     print(f'[ERROR] PipelineRun.get_variables: {err}')
+
+            if interval_start_datetime_previous:
+                try:
+                    variables['interval_start_datetime_previous'] = dateutil.parser.parse(
+                        interval_start_datetime_previous,
+                    )
+                except Exception as err:
+                    print(f'[ERROR] PipelineRun.get_variables: {err}')
         elif self.execution_date and ScheduleType.TIME == self.pipeline_schedule.schedule_type:
             interval_end_datetime = None
             interval_seconds = None
             interval_start_datetime = self.execution_date
+            interval_start_datetime_previous = None
 
             if ScheduleInterval.DAILY == self.pipeline_schedule.schedule_interval:
                 interval_seconds = 60 * 60 * 24
@@ -512,9 +525,15 @@ class PipelineRun(BaseModel):
                     seconds=interval_seconds,
                 )
 
+            if interval_seconds and interval_start_datetime:
+                interval_start_datetime_previous = interval_start_datetime - timedelta(
+                    seconds=interval_seconds,
+                )
+
             variables['interval_end_datetime'] = interval_end_datetime
             variables['interval_seconds'] = interval_seconds
             variables['interval_start_datetime'] = interval_start_datetime
+            variables['interval_start_datetime_previous'] = interval_start_datetime_previous
 
         variables.update(extra_variables)
 

mage_ai/orchestration/pipeline_scheduler.py

@@ -214,6 +214,8 @@ class PipelineScheduler:
                     pipeline=self.pipeline,
                     pipeline_run=self.pipeline_run,
                 )
+                # Cancel block runs that are still in progress for the pipeline run.
+                cancel_block_runs_and_jobs(self.pipeline_run, self.pipeline)
             elif PipelineType.INTEGRATION == self.pipeline.type:
                 self.__schedule_integration_streams(block_runs)
             else:
@@ -1159,6 +1161,27 @@ def stop_pipeline_run(
     pipeline_run.update(status=PipelineRun.PipelineRunStatus.CANCELLED)
 
     # Cancel all the block runs
+    cancel_block_runs_and_jobs(pipeline_run, pipeline)
+
+
+def cancel_block_runs_and_jobs(
+    pipeline_run: PipelineRun,
+    pipeline: Pipeline = None,
+) -> None:
+    """Cancel in progress block runs and jobs for a pipeline run.
+
+    This function cancels blocks runs for the pipeline run. If a pipeline object
+    is provided, it also kills the jobs associated with the pipeline run and its
+    integration streams if applicable.
+
+    Args:
+        pipeline_run (PipelineRun): The pipeline run to stop.
+        pipeline (Pipeline, optional): The pipeline associated with the pipeline run.
+            Defaults to None.
+
+    Returns:
+        None
+    """
     block_runs_to_cancel = []
     running_blocks = []
     for b in pipeline_run.block_runs:

mage_ai/server/constants.py

@@ -12,4 +12,4 @@ DATAFRAME_OUTPUT_SAMPLE_COUNT = 10
 # Dockerfile depends on it because it runs ./scripts/install_mage.sh and uses
 # the last line to determine the version to install.
 VERSION = \
-'0.9.7'
+'0.9.9'