mage-ai 0.8.78__tar.gz → 0.8.80__tar.gz

{mage-ai-0.8.78/mage_ai.egg-info → mage-ai-0.8.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mage-ai
-Version: 0.8.78
+Version: 0.8.80
 Summary: Mage is a tool for building and deploying data pipelines.
 Home-page: https://github.com/mage-ai/mage-ai
 Author: Mage

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/api/policies/PipelineRunPolicy.py

@@ -51,6 +51,7 @@ PipelineRunPolicy.allow_read(PipelineRunPresenter.default_attributes + [], scope
 
 PipelineRunPolicy.allow_write([
     'backfill_id',
+    'event_variables',
     'execution_date',
     'pipeline_schedule_id',
     'pipeline_uuid',

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/api/policies/StatusPolicy.py

@@ -12,10 +12,12 @@ StatusPolicy.allow_actions([
     constants.LIST,
 ], scopes=[
     OauthScope.CLIENT_PRIVATE,
-], condition=lambda policy: policy.has_at_least_viewer_role())
+    OauthScope.CLIENT_PUBLIC,
+])
 
 StatusPolicy.allow_read(StatusPresenter.default_attributes, scopes=[
     OauthScope.CLIENT_PRIVATE,
+    OauthScope.CLIENT_PUBLIC,
 ], on_action=[
     constants.LIST,
-], condition=lambda policy: policy.has_at_least_viewer_role())
+])

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/api/presenters/StatusPresenter.py

@@ -4,6 +4,7 @@ from mage_ai.api.presenters.BasePresenter import BasePresenter
 class StatusPresenter(BasePresenter):
     default_attributes = [
         'is_instance_manager',
+        'max_print_output_lines',
         'repo_path',
         'scheduler_status',
         'instance_type',

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/api/resources/LogResource.py

@@ -1,12 +1,18 @@
 from datetime import datetime
+from typing import Dict, List
+
+from sqlalchemy.orm import aliased
+
 from mage_ai.api.errors import ApiError
 from mage_ai.api.operations.constants import META_KEY_LIMIT
 from mage_ai.api.resources.GenericResource import GenericResource
 from mage_ai.data_preparation.models.pipeline import Pipeline
 from mage_ai.orchestration.db import safe_db_query
-from mage_ai.orchestration.db.models.schedules import BlockRun, PipelineRun, PipelineSchedule
-from sqlalchemy.orm import aliased
-from typing import Dict, List
+from mage_ai.orchestration.db.models.schedules import (
+    BlockRun,
+    PipelineRun,
+    PipelineSchedule,
+)
 
 MAX_LOG_FILES = 20
 
@@ -96,6 +102,7 @@ class LogResource(GenericResource):
             a.pipeline_schedule_id,
             a.pipeline_schedule_id,
             a.pipeline_uuid,
+            a.variables,
         ]
 
         total_pipeline_run_log_count = 0
@@ -155,6 +162,7 @@ class LogResource(GenericResource):
                 model.execution_date = row.execution_date
                 model.pipeline_schedule_id = row.pipeline_schedule_id
                 model.pipeline_uuid = row.pipeline_uuid
+                model.variables = row.variables
                 logs = await model.logs_async()
                 pipeline_log_file_path = logs.get('path')
                 if pipeline_log_file_path not in processed_pipeline_run_log_files:
@@ -235,6 +243,7 @@ class LogResource(GenericResource):
             model.execution_date = row.execution_date
             model.pipeline_schedule_id = row.pipeline_schedule_id
             model.pipeline_uuid = row.pipeline_uuid
+            model.variables = row.variables
 
             model2 = BlockRun()
             model2.block_uuid = row.block_uuid

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/api/resources/StatusResource.py

@@ -1,6 +1,7 @@
 import os
 
 from mage_ai.api.resources.GenericResource import GenericResource
+from mage_ai.data_preparation.models.constants import MAX_PRINT_OUTPUT_LINES
 from mage_ai.data_preparation.repo_manager import (
     ProjectType,
     get_project_type,
@@ -53,6 +54,7 @@ class StatusResource(GenericResource):
             'scheduler_status': scheduler_manager.get_status(),
             'instance_type': instance_type,
             'disable_pipeline_edit_access': is_disable_pipeline_edit_access(),
+            'max_print_output_lines': MAX_PRINT_OUTPUT_LINES,
             'require_user_authentication': REQUIRE_USER_AUTHENTICATION,
         }
         return self.build_result_set([status], user, **kwargs)

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/data_preparation/executors/streaming_pipeline_executor.py

@@ -1,15 +1,18 @@
+import asyncio
+import copy
+import logging
+import os
 from contextlib import redirect_stderr, redirect_stdout
+from typing import Callable, Dict, List, Union
+
+import yaml
+
 from mage_ai.data_preparation.executors.pipeline_executor import PipelineExecutor
 from mage_ai.data_preparation.logging.logger import DictLogger
 from mage_ai.data_preparation.models.constants import BlockType
 from mage_ai.data_preparation.models.pipeline import Pipeline
 from mage_ai.data_preparation.shared.stream import StreamToLogger
 from mage_ai.shared.hash import merge_dict
-from typing import Callable, Dict, List, Union
-import asyncio
-import logging
-import os
-import yaml
 
 
 class StreamingPipelineExecutor(PipelineExecutor):
@@ -21,7 +24,9 @@ class StreamingPipelineExecutor(PipelineExecutor):
     def parse_and_validate_blocks(self):
         """
         Find the first valid streaming pipeline is in the structure:
-        source -> transformer -> sink
+        source -> transformer1 -> sink2
+               -> transformer2 -> sink2
+               -> transformer3 -> sink3
         """
         blocks = self.pipeline.blocks_by_uuid.values()
         source_blocks = []
@@ -29,24 +34,23 @@ class StreamingPipelineExecutor(PipelineExecutor):
         transformer_blocks = []
         for b in blocks:
             if b.type == BlockType.DATA_LOADER:
+                # Data loader block should be root block
                 if len(b.upstream_blocks or []) > 0:
                     raise Exception(f'Data loader {b.uuid} can\'t have upstream blocks.')
-                if len(b.downstream_blocks or []) != 1:
-                    raise Exception(f'Data loader {b.uuid} must have one transformer or data'
-                                    ' exporter as the downstream block.')
+                if len(b.downstream_blocks or []) < 1:
+                    raise Exception(f'Data loader {b.uuid} must have at least one transformer or'
+                                    ' data exporter as the downstream block.')
                 source_blocks.append(b)
             if b.type == BlockType.DATA_EXPORTER:
+                # Data exporter block should be leaf block
                 if len(b.downstream_blocks or []) > 0:
                     raise Exception(f'Data expoter {b.uuid} can\'t have downstream blocks.')
                 if len(b.upstream_blocks or []) != 1:
-                    raise Exception(f'Data loader {b.uuid} must have a transformer or data'
-                                    ' exporter as the upstream block.')
+                    raise Exception(f'Data exporter {b.uuid} must have a transformer or data'
+                                    ' loader as the upstream block.')
                 sink_blocks.append(b)
             if b.type == BlockType.TRANSFORMER:
-                if len(b.downstream_blocks or []) != 1:
-                    raise Exception(
-                        f'Transformer {b.uuid} should (only) have one downstream block.',
-                    )
+                # Each transformer block can only have one upstream block
                 if len(b.upstream_blocks or []) != 1:
                     raise Exception(f'Transformer {b.uuid} should (only) have one upstream block.')
                 transformer_blocks.append(b)
@@ -54,15 +58,8 @@ class StreamingPipelineExecutor(PipelineExecutor):
         if len(source_blocks) != 1:
             raise Exception('Please provide (only) one data loader block as the source.')
 
-        if len(transformer_blocks) > 1:
-            raise Exception('Please provide no more than one transformer block.')
-
-        if len(sink_blocks) != 1:
-            raise Exception('Please provide (only) one data expoter block as the sink.')
-
         self.source_block = source_blocks[0]
-        self.sink_block = sink_blocks[0]
-        self.transformer_block = transformer_blocks[0] if len(transformer_blocks) > 0 else None
+        self.sink_blocks = sink_blocks
 
     def execute(
         self,
@@ -97,11 +94,10 @@ class StreamingPipelineExecutor(PipelineExecutor):
             raise e
 
     def __execute_in_python(self, build_block_output_stdout: Callable[..., object] = None):
+        from mage_ai.streaming.sinks.sink_factory import SinkFactory
         from mage_ai.streaming.sources.base import SourceConsumeMethod
         from mage_ai.streaming.sources.source_factory import SourceFactory
-        from mage_ai.streaming.sinks.sink_factory import SinkFactory
         source_config = yaml.safe_load(self.source_block.content)
-        sink_config = yaml.safe_load(self.sink_block.content)
         source = SourceFactory.get_source(
             source_config,
             checkpoint_path=os.path.join(
@@ -109,47 +105,67 @@ class StreamingPipelineExecutor(PipelineExecutor):
                 'streaming_checkpoint',
             ),
         )
-        sink = SinkFactory.get_sink(
-            sink_config,
-            buffer_path=os.path.join(
-                self.pipeline.pipeline_variables_dir,
-                'buffer',
-            ),
-        )
+
+        sinks_by_uuid = dict()
+        for sink_block in self.sink_blocks:
+            sink_config = yaml.safe_load(sink_block.content)
+            sinks_by_uuid[sink_block.uuid] = SinkFactory.get_sink(
+                sink_config,
+                buffer_path=os.path.join(
+                    self.pipeline.pipeline_variables_dir,
+                    'buffer',
+                ),
+            )
+
+        def handle_batch_events_recursively(curr_block, outputs_by_block: Dict, **kwargs):
+            curr_block_output = outputs_by_block[curr_block.uuid]
+            for downstream_block in curr_block.downstream_blocks:
+                if downstream_block.type == BlockType.TRANSFORMER:
+                    execute_block_kwargs = dict(
+                        global_vars=kwargs,
+                        input_args=[copy.deepcopy(curr_block_output)],
+                        logger=self.logger,
+                    )
+                    if build_block_output_stdout:
+                        execute_block_kwargs['build_block_output_stdout'] = \
+                            build_block_output_stdout
+                    outputs_by_block[downstream_block.uuid] = \
+                        downstream_block.execute_block(
+                            **execute_block_kwargs,
+                    )['output']
+                elif downstream_block.type == BlockType.DATA_EXPORTER:
+                    sinks_by_uuid[downstream_block.uuid].batch_write(
+                        copy.deepcopy(curr_block_output))
+                if downstream_block.downstream_blocks:
+                    handle_batch_events_recursively(
+                        downstream_block,
+                        outputs_by_block,
+                        **kwargs,
+                    )
 
         def handle_batch_events(messages: List[Union[Dict, str]], **kwargs):
-            if self.transformer_block is not None:
-                execute_block_kwargs = dict(
-                    global_vars=kwargs,
-                    input_args=[messages],
-                    logger=self.logger,
-                )
-                if build_block_output_stdout:
-                    execute_block_kwargs['build_block_output_stdout'] = build_block_output_stdout
-                messages = self.transformer_block.execute_block(
-                    **execute_block_kwargs,
-                )['output']
-            sink.batch_write(messages)
+            # Handle the events with DFS
+
+            outputs_by_block = dict()
+            outputs_by_block[self.source_block.uuid] = messages
+
+            handle_batch_events_recursively(self.source_block, outputs_by_block)
 
         async def handle_event_async(message, **kwargs):
-            if self.transformer_block is not None:
-                execute_block_kwargs = dict(
-                    global_vars=kwargs,
-                    input_args=[[message]],
-                    logger=self.logger,
-                )
-                if build_block_output_stdout:
-                    execute_block_kwargs['build_block_output_stdout'] = build_block_output_stdout
-                messages = self.transformer_block.execute_block(
-                    **execute_block_kwargs,
-                )['output']
-            sink.batch_write(messages)
+            outputs_by_block = dict()
+            outputs_by_block[self.source_block.uuid] = [message]
+
+            handle_batch_events_recursively(self.source_block, outputs_by_block)
 
         # Long running method
         if source.consume_method == SourceConsumeMethod.BATCH_READ:
             source.batch_read(handler=handle_batch_events)
         elif source.consume_method == SourceConsumeMethod.READ_ASYNC:
-            asyncio.run(source.read_async(handler=handle_event_async))
+            loop = asyncio.get_event_loop()
+            if loop is not None:
+                loop.run_until_complete(source.read_async(handler=handle_event_async))
+            else:
+                asyncio.run(source.read_async(handler=handle_event_async))
 
     def __excute_in_flink(self):
         """

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/data_preparation/models/block/__init__.py

@@ -227,9 +227,11 @@ class Block:
         status: BlockStatus = BlockStatus.NOT_EXECUTED,
         pipeline=None,
         language: BlockLanguage = BlockLanguage.PYTHON,
-        configuration: Dict = dict(),
+        configuration: Dict = None,
         has_callback: bool = False,
     ):
+        if configuration is None:
+            configuration = dict()
         self.name = name or uuid
         self._uuid = uuid
         self.type = block_type
@@ -614,7 +616,7 @@ class Block:
         self,
         global_vars: Dict = None,
         logger: Logger = None,
-        logging_tags: Dict = dict(),
+        logging_tags: Dict = None,
         **kwargs
     ):
         """
@@ -623,6 +625,9 @@ class Block:
         websocket as a way to test the code in the callback. To run a block in a pipeline
         run, use a BlockExecutor.
         """
+        if logging_tags is None:
+            logging_tags = dict()
+
         arr = []
         if self.callback_block:
             arr.append(self.callback_block)
@@ -643,6 +648,7 @@ class Block:
                     global_vars=global_vars,
                     logger=logger,
                     logging_tags=logging_tags,
+                    parent_block=self,
                 )
             raise e
 
@@ -652,6 +658,7 @@ class Block:
                 global_vars=global_vars,
                 logger=logger,
                 logging_tags=logging_tags,
+                parent_block=self,
             )
 
         return output
@@ -664,7 +671,7 @@ class Block:
         execution_partition: str = None,
         global_vars: Dict = None,
         logger: Logger = None,
-        logging_tags: Dict = dict(),
+        logging_tags: Dict = None,
         run_all_blocks: bool = False,
         test_execution: bool = False,
         update_status: bool = True,
@@ -678,6 +685,9 @@ class Block:
         run_settings: Dict = None,
         **kwargs,
     ) -> Dict:
+        if logging_tags is None:
+            logging_tags = dict()
+
         try:
             if not run_all_blocks:
                 not_executed_upstream_blocks = list(
@@ -873,7 +883,7 @@ class Block:
         execution_partition: str = None,
         input_args: List = None,
         logger: Logger = None,
-        logging_tags: Dict = dict(),
+        logging_tags: Dict = None,
         global_vars: Dict = None,
         test_execution: bool = False,
         input_from_output: Dict = None,
@@ -883,6 +893,9 @@ class Block:
         run_settings: Dict = None,
         **kwargs,
     ) -> Dict:
+        if logging_tags is None:
+            logging_tags = dict()
+
         # Add pipeline uuid and block uuid to global_vars
         global_vars = merge_dict(
             global_vars or dict(),
@@ -951,7 +964,7 @@ class Block:
         execution_partition: str = None,
         input_vars: List = None,
         logger: Logger = None,
-        logging_tags: Dict = dict(),
+        logging_tags: Dict = None,
         global_vars: Dict = None,
         test_execution: bool = False,
         input_from_output: Dict = None,
@@ -960,6 +973,9 @@ class Block:
         run_settings: Dict = None,
         **kwargs,
     ) -> List:
+        if logging_tags is None:
+            logging_tags = dict()
+
         decorated_functions = []
         test_functions = []
 
@@ -1531,13 +1547,18 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
         build_block_output_stdout: Callable[..., object] = None,
         custom_code: str = None,
         execution_partition: str = None,
-        global_vars: Dict = {},
+        global_vars: Dict = None,
         logger: Logger = None,
-        logging_tags: Dict = dict(),
+        logging_tags: Dict = None,
         update_tests: bool = True,
         dynamic_block_uuid: str = None,
         from_notebook: bool = False,
     ) -> None:
+        if global_vars is None:
+            global_vars = dict()
+        if logging_tags is None:
+            logging_tags = dict()
+
         self.dynamic_block_uuid = dynamic_block_uuid
 
         if self.pipeline \
@@ -2133,6 +2154,8 @@ class CallbackBlock(Block):
                     pipeline_run=pipeline_run,
                 ),
             )
+            if parent_block:
+                global_vars['parent_block_uuid'] = parent_block.uuid
 
             if parent_block and \
                     parent_block.pipeline and \

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/data_preparation/models/constants.py

@@ -1,3 +1,4 @@
+import os
 from enum import Enum
 
 PIPELINES_FOLDER = 'pipelines'
@@ -19,6 +20,7 @@ DATAFRAME_ANALYSIS_MAX_COLUMNS = 100
 DATAFRAME_SAMPLE_COUNT_PREVIEW = 10
 DATAFRAME_SAMPLE_COUNT = 1000
 DATAFRAME_SAMPLE_MAX_COLUMNS = 1000
+MAX_PRINT_OUTPUT_LINES = int(os.getenv('MAX_PRINT_OUTPUT_LINES', 1000) or 1000)
 VARIABLE_DIR = '.variables'
 LOGS_DIR = '.logs'
 

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/data_preparation/models/widget/charts.py

@@ -82,10 +82,29 @@ def build_time_series_buckets(df, datetime_column, time_interval, metrics):
         return []
 
     datetimes = datetimes.unique()
-    min_value_datetime = dateutil.parser.parse(datetimes.min())
-    max_value_datetime = dateutil.parser.parse(datetimes.max())
-
-    a, b = [dateutil.parser.parse(d) for d in sorted(datetimes)[:2]]
+    min_value_datetime = datetimes.min()
+    max_value_datetime = datetimes.max()
+
+    if type(min_value_datetime) is str:
+        min_value_datetime = dateutil.parser.parse(min_value_datetime)
+    if type(max_value_datetime) is str:
+        max_value_datetime = dateutil.parser.parse(max_value_datetime)
+
+    # If you manually convert the datetime column to a datetime, Pandas will use numpy.datetime64
+    # type. This type does not have the methods year, month, day, etc that is used down below.
+    datetimes_temp = []
+    for dt in datetimes:
+        if type(dt) is np.datetime64:
+            datetimes_temp.append(pd.to_datetime(dt.astype(datetime)).to_pydatetime())
+        else:
+            datetimes_temp.append(dt)
+    datetimes = datetimes_temp
+    if type(min_value_datetime) is np.datetime64:
+        min_value_datetime = pd.to_datetime(min_value_datetime.astype(datetime)).to_pydatetime()
+    if type(max_value_datetime) is np.datetime64:
+        max_value_datetime = pd.to_datetime(max_value_datetime.astype(datetime)).to_pydatetime()
+
+    a, b = [dateutil.parser.parse(d) if type(d) is str else d for d in sorted(datetimes)[:2]]
 
     year = min_value_datetime.year
     month = min_value_datetime.month

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/orchestration/db/models/schedules.py

@@ -237,6 +237,8 @@ class PipelineRun(BaseModel):
 
     @property
     def execution_partition(self) -> str:
+        if self.variables and self.variables.get('execution_partition'):
+            return self.variables.get('execution_partition')
         if self.execution_date is None:
             return str(self.pipeline_schedule_id)
         else:

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/orchestration/notification/sender.py

@@ -1,11 +1,12 @@
+import os
+
 from mage_ai.orchestration.notification.config import AlertOn, NotificationConfig
 from mage_ai.services.email.email import send_email
+from mage_ai.services.google_chat.google_chat import send_google_chat_message
 from mage_ai.services.opsgenie.opsgenie import send_opsgenie_alert
 from mage_ai.services.slack.slack import send_slack_message
 from mage_ai.services.teams.teams import send_teams_message
-from mage_ai.services.google_chat.google_chat import send_google_chat_message
 from mage_ai.settings import MAGE_PUBLIC_HOST
-import os
 
 
 class NotificationSender:
@@ -18,10 +19,17 @@ class NotificationSender:
         summary: str = None,
         details: str = None,
     ) -> None:
+        """Send messages to the notification channels.
+
+        Args:
+            title (str, optional): Short sentence, used as title (e.g. Email subject)
+            summary (str, optional): Mid-length sentences, used as the summary of the message.
+            details (str, optional): Long message, used as the body of the message (e.g. Email body)
+        """
         if summary is None:
             return
         if self.config.slack_config is not None and self.config.slack_config.is_valid:
-            send_slack_message(self.config.slack_config, summary)
+            send_slack_message(self.config.slack_config, details or summary)
 
         if self.config.teams_config is not None and self.config.teams_config.is_valid:
             send_teams_message(self.config.teams_config, summary)
@@ -75,9 +83,14 @@ class NotificationSender:
                 f'at execution time `{pipeline_run.execution_date}`.'
             )
             email_content = f'{message}\n'
-            if os.getenv('ENV') != 'production':
+            if os.getenv('ENV') != 'production' or MAGE_PUBLIC_HOST != 'http://localhost:6789':
+                """
+                Include the URL for the following cases
+                1. Dev environment: Use the default localhost as host in URL
+                2. Production environment: If MAGE_PUBLIC_HOST is set, use it as host.
+                """
                 email_content += f'Open {self.__pipeline_run_url(pipeline, pipeline_run)} '\
-                                  'to check pipeline run results and logs.'
+                                 'to check pipeline run results and logs.'
             self.send(
                 title=f'Failed to run Mage pipeline {pipeline.uuid}',
                 summary=message,

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/orchestration/pipeline_scheduler.py

@@ -1,3 +1,4 @@
+import os
 import traceback
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Tuple
@@ -274,7 +275,9 @@ class PipelineScheduler:
                     tags=merge_dict(tags, dict(metrics=self.pipeline_run.metrics)),
                 )
 
-    def memory_usage_failure(self, tags: Dict = {}) -> None:
+    def memory_usage_failure(self, tags: Dict = None) -> None:
+        if tags is None:
+            tags = dict()
         msg = 'Memory usage across all pipeline runs has reached or exceeded the maximum '\
             f'limit of {int(MEMORY_USAGE_MAXIMUM * 100)}%.'
         self.logger.info(msg, tags=tags)
@@ -603,7 +606,8 @@ def run_integration_pipeline(
             while True:
                 block_runs_in_order.append(
                     find(
-                        lambda b: b.block_uuid == f'{current_block.uuid}:{tap_stream_id}:{idx}',
+                        lambda b: b.block_uuid ==
+                        f'{current_block.uuid}:{tap_stream_id}:{idx}',  # noqa: B023
                         all_block_runs,
                     )
                 )
@@ -616,11 +620,11 @@ def run_integration_pipeline(
             data_exporter_uuid = f'{data_exporter_block.uuid}:{tap_stream_id}:{idx}'
 
             data_loader_block_run = find(
-                lambda b: b.block_uuid == data_loader_uuid,
+                lambda b: b.block_uuid == data_loader_uuid,     # noqa: B023
                 all_block_runs,
             )
             data_exporter_block_run = find(
-                lambda b: b.block_uuid == data_exporter_uuid,
+                lambda b: b.block_uuid == data_exporter_uuid,   # noqa: B023
                 block_runs_for_stream,
             )
             if not data_loader_block_run or not data_exporter_block_run:
@@ -653,7 +657,7 @@ def run_integration_pipeline(
                 block_runs_and_configs = block_runs_and_configs[1:]
 
             block_failed = False
-            for idx2, tup in enumerate(block_runs_and_configs):
+            for _, tup in enumerate(block_runs_and_configs):
                 block_run, template_runtime_configuration = tup
 
                 tags_updated = merge_dict(tags, dict(
@@ -813,7 +817,7 @@ def run_block(
                     block_grandparent_uuid,
                 )
 
-                for idx, value in enumerate(values):
+                for idx, _ in enumerate(values):
                     if idx < len(block_metadata):
                         metadata = block_metadata[idx].copy()
                     else:
@@ -880,8 +884,10 @@ def run_pipeline(pipeline_run_id, variables, tags):
 def configure_pipeline_run_payload(
     pipeline_schedule: PipelineSchedule,
     pipeline_type: PipelineType,
-    payload: Dict = {},
+    payload: Dict = None,
 ) -> Tuple[Dict, bool]:
+    if payload is None:
+        payload = dict()
     if 'variables' not in payload:
         payload['variables'] = {}
 
@@ -893,6 +899,13 @@ def configure_pipeline_run_payload(
     elif not isinstance(execution_date, datetime):
         payload['execution_date'] = datetime.fromisoformat(execution_date)
 
+    # Set execution_partition in variables
+    payload['variables']['execution_partition'] = \
+        os.sep.join([
+            str(pipeline_schedule.id),
+            payload['execution_date'].strftime(format='%Y%m%dT%H%M%S_%f'),
+        ])
+
     is_integration = PipelineType.INTEGRATION == pipeline_type
     if is_integration:
         payload['create_block_runs'] = False
@@ -1115,7 +1128,9 @@ def schedule_all():
     job_manager.clean_up_jobs()
 
 
-def schedule_with_event(event: Dict = dict()):
+def schedule_with_event(event: Dict = None):
+    if event is None:
+        event = dict()
     logger.info(f'Schedule with event {event}')
     all_event_matchers = EventMatcher.active_event_matchers()
     for e in all_event_matchers:
@@ -1145,7 +1160,9 @@ def sync_schedules(pipeline_uuids: List[str]):
             PipelineSchedule.create_or_update(pipeline_trigger)
 
 
-def get_variables(pipeline_run, extra_variables: Dict = {}) -> Dict:
+def get_variables(pipeline_run, extra_variables: Dict = None) -> Dict:
+    if extra_variables is None:
+        extra_variables = dict()
     if not pipeline_run:
         return {}
 

{mage-ai-0.8.78 → mage-ai-0.8.80}/mage_ai/server/constants.py

@@ -12,4 +12,4 @@ DATAFRAME_OUTPUT_SAMPLE_COUNT = 10
 # Dockerfile depends on it because it runs ./scripts/install_mage.sh and uses
 # the last line to determine the version to install.
 VERSION = \
-'0.8.78'
+'0.8.80'