mage-ai 0.9.58__py3-none-any.whl → 0.9.59__py3-none-any.whl

mage_ai/data_preparation/models/block/__init__.py

@@ -18,6 +18,7 @@ from typing import Any, Callable, Dict, Generator, List, Set, Tuple, Union
 
 import inflection
 import pandas as pd
+import polars as pl
 import simplejson
 import yaml
 from jinja2 import Template
@@ -28,12 +29,20 @@ from mage_ai.data_cleaner.shared.utils import is_geo_dataframe, is_spark_datafra
 from mage_ai.data_integrations.sources.constants import SQL_SOURCES_MAPPING
 from mage_ai.data_preparation.logging.logger import DictLogger
 from mage_ai.data_preparation.logging.logger_manager_factory import LoggerManagerFactory
+from mage_ai.data_preparation.models.block.content import hydrate_block_outputs
 from mage_ai.data_preparation.models.block.data_integration.mixins import (
     DataIntegrationMixin,
 )
 from mage_ai.data_preparation.models.block.data_integration.utils import (
     execute_data_integration,
 )
+from mage_ai.data_preparation.models.block.dynamic.utils import (
+    DynamicBlockFlag,
+    is_dynamic_block,
+    is_dynamic_block_child,
+    should_reduce_output,
+    uuid_for_output_variables,
+)
 from mage_ai.data_preparation.models.block.errors import HasDownstreamDependencies
 from mage_ai.data_preparation.models.block.extension.utils import handle_run_tests
 from mage_ai.data_preparation.models.block.platform.mixins import (
@@ -45,12 +54,9 @@ from mage_ai.data_preparation.models.block.utils import (
     clean_name,
     fetch_input_variables,
     input_variables,
-    is_dynamic_block,
-    is_dynamic_block_child,
     is_output_variable,
     is_valid_print_variable,
     output_variables,
-    should_reduce_output,
 )
 from mage_ai.data_preparation.models.constants import (
     BLOCK_LANGUAGE_TO_FILE_EXTENSION,
@@ -76,20 +82,22 @@ from mage_ai.data_preparation.shared.stream import StreamToLogger
 from mage_ai.data_preparation.shared.utils import get_template_vars
 from mage_ai.data_preparation.templates.data_integrations.utils import get_templates
 from mage_ai.data_preparation.templates.template import load_template
-from mage_ai.data_preparation.templates.utils import get_variable_for_template
 from mage_ai.server.kernel_output_parser import DataType
 from mage_ai.services.spark.config import SparkConfig
 from mage_ai.services.spark.spark import get_spark_session
 from mage_ai.settings.platform.constants import project_platform_activated
 from mage_ai.settings.repo import get_repo_path
 from mage_ai.shared.constants import ENV_DEV, ENV_TEST
+from mage_ai.shared.custom_logger import DX_PRINTER
 from mage_ai.shared.environments import get_env, is_debug
 from mage_ai.shared.hash import extract, ignore_keys, merge_dict
 from mage_ai.shared.logger import BlockFunctionExec
 from mage_ai.shared.parsers import encode_complex
 from mage_ai.shared.path_fixer import (
+    add_absolute_path,
     add_root_repo_path_to_relative_path,
     get_path_parts,
+    remove_base_repo_path,
 )
 from mage_ai.shared.strings import format_enum
 from mage_ai.shared.utils import clean_name as clean_name_orig
@@ -337,14 +345,8 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
         # Replicate block
         self.replicated_block = replicated_block
-        self.replicated_block_object = None
-        if replicated_block:
-            self.replicated_block_object = Block(
-                self.replicated_block,
-                self.replicated_block,
-                self.type,
-                language=self.language,
-            )
+        self.replicated_blocks = {}
+        self._replicated_block_object = None
 
         # Module for the block functions. Will be set when the block is executed from a notebook.
         self.module = None
@@ -399,9 +401,26 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
     def configuration(self, x) -> None:
         self._configuration = self.clean_file_paths(x) if x else x
 
+    def get_original_block(self) -> 'Block':
+        if self.replicated_block:
+            return self.replicated_block_object.get_original_block()
+        return self
+
+    @property
+    def replicated_block_object(self) -> 'Block':
+        if self._replicated_block_object:
+            return self._replicated_block_object
+
+        if self.replicated_block and self.pipeline:
+            self._replicated_block_object = self.pipeline.get_block(self.replicated_block)
+            if self._replicated_block_object:
+                self._replicated_block_object.replicated_blocks[self.uuid] = self
+
+        return self._replicated_block_object
+
     @property
     def content(self) -> str:
-        if self.replicated_block:
+        if self.replicated_block and self.replicated_block_object:
             self._content = self.replicated_block_object.content
 
         if self._content is None:
@@ -422,7 +441,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         return None
 
     async def content_async(self) -> str:
-        if self.replicated_block:
+        if self.replicated_block and self.replicated_block_object:
             self._content = await self.replicated_block_object.content_async()
 
         if self._content is None:
@@ -430,6 +449,44 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
         return self._content
 
+    def interpolate_content(
+        self,
+        content: str,
+        dynamic_block_index: int = None,
+        dynamic_block_indexes: int = None,
+        dynamic_upstream_block_uuids: int = None,
+        execution_partition: str = None,
+        from_notebook: bool = False,
+        outputs_from_input_vars: Dict = None,
+        upstream_block_uuids: List[str] = None,
+        variables: Dict = None,
+        **kwargs,
+    ) -> str:
+        variables = variables or {}
+        if self.pipeline and self.pipeline.variables:
+            variables.update(self.pipeline.variables)
+
+        if upstream_block_uuids is None:
+            upstream_block_uuids = self.upstream_block_uuids
+
+        if outputs_from_input_vars is None:
+            outputs_from_input_vars, _input_vars, _kwargs_vars, upstream_block_uuids = \
+                self.__get_outputs_from_input_vars(
+                    dynamic_block_index=dynamic_block_index,
+                    dynamic_block_indexes=dynamic_block_indexes,
+                    dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                    execution_partition=execution_partition,
+                    from_notebook=from_notebook,
+                    global_vars=variables,
+                )
+
+        return hydrate_block_outputs(
+            content,
+            outputs_from_input_vars=outputs_from_input_vars,
+            upstream_block_uuids=upstream_block_uuids,
+            variables=variables,
+        )
+
     async def metadata_async(self) -> Dict:
         if self.is_data_integration():
             grouped_templates = get_templates(group_templates=True)
@@ -437,59 +494,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             if BlockLanguage.YAML == self.language:
                 content = await self.content_async()
                 if content:
-                    variables = {}
-                    if self.pipeline and self.pipeline.variables:
-                        variables.update(self.pipeline.variables)
-
-                    def _block_output(
-                        block_uuid: str,
-                        parse: str = None,
-                        current_block=self,
-                        global_vars=variables,
-                    ) -> Any:
-                        data = None
-
-                        if not self.fetched_inputs_from_blocks:
-                            input_vars_fetched, _kwargs_vars, _upstream_block_uuids = \
-                                self.fetch_input_variables_and_catalog(
-                                    None,
-                                    None,
-                                    global_vars=global_vars,
-                                    from_notebook=True,
-                                )
-                            self.fetched_inputs_from_blocks = input_vars_fetched
-
-                        if block_uuid in self.upstream_block_uuids and \
-                                self.data_integration_inputs and \
-                                block_uuid in self.data_integration_inputs:
-
-                            up_uuids = [i for i in
-                                        self.upstream_block_uuids if i in
-                                        self.data_integration_inputs]
-
-                            index = up_uuids.index(block_uuid)
-                            data = self.fetched_inputs_from_blocks[index]
-
-                        if parse:
-                            results = {}
-                            exec(f'_parse_func = {parse}', results)
-                            try:
-                                return results['_parse_func'](data)
-                            except Exception:
-                                pass
-
-                        return data
-
-                    text = Template(content).render(
-                        block_output=_block_output,
-                        variables=lambda x, p=None, v=variables: get_variable_for_template(
-                            x,
-                            parse=p,
-                            variables=v,
-                        ),
-                        **get_template_vars(),
-                    )
-
+                    text = self.interpolate_content(content)
                     settings = yaml.safe_load(text)
                     uuid = settings.get('source') or settings.get('destination')
                     mapping = grouped_templates.get(uuid) or {}
@@ -570,6 +575,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
     @property
     def repo_path(self) -> str:
+        if self.project_platform_activated:
+            repo_path = self.get_repo_path_from_configuration()
+            if repo_path:
+                return repo_path
+
         return self.pipeline.repo_path if self.pipeline is not None else get_repo_path()
 
     @classmethod
@@ -591,9 +601,15 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
     @property
     def file_path(self) -> str:
+        if self.replicated_block and self.replicated_block_object:
+            return self.replicated_block_object.file_path
+
         file_path = self.get_file_path_from_source()
+        if not file_path:
+            file_path = self.configuration.get('file_path')
+
         if file_path:
-            return add_root_repo_path_to_relative_path(file_path)
+            return add_absolute_path(file_path)
 
         return self.__build_file_path(
             self.repo_path or os.getcwd(),
@@ -604,6 +620,9 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
     @property
     def file(self) -> File:
+        if self.replicated_block and self.replicated_block_object:
+            return self.replicated_block_object.file
+
         if self.project_platform_activated:
             file = self.build_file()
             if file:
@@ -660,23 +679,16 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             downstream_block_uuids = kwargs.get('downstream_block_uuids', [])
             upstream_block_uuids = kwargs.get('upstream_block_uuids', [])
 
-            if BlockType.DBT == block.type and block.language == BlockLanguage.SQL:
-                upstream_dbt_blocks = block.upstream_dbt_blocks() or []
-                upstream_dbt_blocks_by_uuid = {
-                    block.uuid: block
-                    for block in upstream_dbt_blocks
-                }
-                pipeline.blocks_by_uuid.update(upstream_dbt_blocks_by_uuid)
-                pipeline.validate('A cycle was formed while adding a block')
-                pipeline.save()
-            else:
-                pipeline.add_block(
-                    block,
-                    downstream_block_uuids=downstream_block_uuids,
-                    upstream_block_uuids=upstream_block_uuids,
-                    priority=priority,
-                    widget=widget,
-                )
+            if BlockType.DBT == block.type:
+                block.set_default_configurations()
+
+            pipeline.add_block(
+                block,
+                downstream_block_uuids=downstream_block_uuids,
+                upstream_block_uuids=upstream_block_uuids,
+                priority=priority,
+                widget=widget,
+            )
 
     @classmethod
     def block_class_from_type(self, block_type: str, language=None, pipeline=None) -> 'Block':
@@ -1052,8 +1064,10 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                     logger=logger,
                     logging_tags=logging_tags,
                 )
-                conditional_message += \
-                    f'Conditional block {conditional_block.uuid} evaluated to {block_result}.\n'
+                conditional_message = (
+                    f'{conditional_message}Conditional block '
+                    f'{conditional_block.uuid} evaluated to {block_result}.\n'
+                )
                 result = result and block_result
 
             # Print result to block output
@@ -1071,6 +1085,33 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             callback_arr += self.callback_blocks
 
         try:
+            # {
+            #   "dynamic_block_index": 1,
+            #   "dynamic_block_indexes": {
+            #     "dynamic_b": 0
+            #   },
+            #   "dynamic_upstream_block_uuids": [
+            #     "child_upstreams:0:b0_0"
+            #   ],
+            #   "metadata": {
+            #     "dynamic_b": {
+            #       "block_uuid": "b1_1",
+            #       "upstream_blocks": [
+            #         "b0_0"
+            #       ]
+            #     }
+            #   }
+            # }
+            if from_notebook:
+                for upstream_block in self.upstream_blocks:
+                    if is_dynamic_block(upstream_block) or is_dynamic_block_child(upstream_block):
+                        if 'dynamic_block_index' not in kwargs:
+                            kwargs['dynamic_block_index'] = 0
+                        if 'dynamic_block_indexes' not in kwargs:
+                            kwargs['dynamic_block_indexes'] = {}
+                        if upstream_block.uuid not in kwargs['dynamic_block_indexes']:
+                            kwargs['dynamic_block_indexes'][upstream_block.uuid] = 0
+
             output = self.execute_sync(
                 global_vars=global_vars,
                 logger=logger,
@@ -1129,6 +1170,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         disable_json_serialization: bool = False,
         data_integration_runtime_settings: Dict = None,
         execution_partition_previous: str = None,
+        metadata: Dict = None,
         **kwargs,
     ) -> Dict:
         if logging_tags is None:
@@ -1185,55 +1227,68 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 run_settings=run_settings,
                 data_integration_runtime_settings=data_integration_runtime_settings,
                 execution_partition_previous=execution_partition_previous,
+                metadata=metadata,
                 **kwargs,
             )
 
-            block_output = self.post_process_output(output)
-            variable_mapping = dict()
-
-            if BlockType.CHART == self.type:
-                variable_mapping = block_output
-                output = dict(
-                    output=simplejson.dumps(
-                        block_output,
-                        default=encode_complex,
-                        ignore_nan=True,
-                    ) if not disable_json_serialization else block_output,
-                )
+            if self.configuration and self.configuration.get('disable_query_preprocessing'):
+                output = dict(output=None)
             else:
-                output_count = len(block_output)
-                variable_keys = [f'output_{idx}' for idx in range(output_count)]
-                variable_mapping = dict(zip(variable_keys, block_output))
+                block_output = self.post_process_output(output)
+                variable_mapping = dict()
+
+                if BlockType.CHART == self.type:
+                    variable_mapping = block_output
+                    output = dict(
+                        output=simplejson.dumps(
+                            block_output,
+                            default=encode_complex,
+                            ignore_nan=True,
+                        ) if not disable_json_serialization else block_output,
+                    )
+                else:
+                    output_count = len(block_output)
+                    variable_keys = [f'output_{idx}' for idx in range(output_count)]
+                    variable_mapping = dict(zip(variable_keys, block_output))
 
-            if store_variables and \
-                    self.pipeline and \
-                    self.pipeline.type != PipelineType.INTEGRATION:
+                if store_variables and \
+                        self.pipeline and \
+                        self.pipeline.type != PipelineType.INTEGRATION:
 
-                try:
-                    self.store_variables(
-                        variable_mapping,
-                        execution_partition=execution_partition,
-                        override_outputs=True,
-                        spark=self.__get_spark_session_from_global_vars(global_vars=global_vars),
-                        dynamic_block_uuid=dynamic_block_uuid,
-                    )
-                except ValueError as e:
-                    if str(e) == 'Circular reference detected':
-                        raise ValueError(
-                            'Please provide dataframe or json serializable data as output.'
+                    try:
+                        self.store_variables(
+                            variable_mapping,
+                            execution_partition=execution_partition,
+                            override_outputs=True,
+                            spark=self.__get_spark_session_from_global_vars(
+                                global_vars=global_vars,
+                            ),
+                            dynamic_block_uuid=dynamic_block_uuid,
+                        )
+                    except ValueError as e:
+                        if str(e) == 'Circular reference detected':
+                            raise ValueError(
+                                'Please provide dataframe or json serializable data as output.'
+                            )
+                        raise e
+                # Reset outputs cache
+                self._outputs = None
+
+                if BlockType.CHART != self.type:
+                    if analyze_outputs:
+                        self.analyze_outputs(
+                            variable_mapping,
+                            execution_partition=execution_partition,
+                        )
+                    else:
+                        self.analyze_outputs(
+                            variable_mapping,
+                            execution_partition=execution_partition,
+                            shape_only=True,
                         )
-                    raise e
-            # Reset outputs cache
-            self._outputs = None
 
             if update_status:
                 self.status = BlockStatus.EXECUTED
-
-            if BlockType.CHART != self.type:
-                if analyze_outputs:
-                    self.analyze_outputs(variable_mapping)
-                else:
-                    self.analyze_outputs(variable_mapping, shape_only=True)
         except Exception as err:
             if update_status:
                 self.status = BlockStatus.FAILED
@@ -1348,6 +1403,59 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
             return block_function
 
+    def __get_outputs_from_input_vars(
+        self,
+        block_run_outputs_cache: Dict[str, List] = None,
+        dynamic_block_index: int = None,
+        dynamic_block_indexes: Dict = None,
+        dynamic_upstream_block_uuids: List[str] = None,
+        execution_partition: str = None,
+        from_notebook: bool = False,
+        global_vars: Dict = None,
+        input_args: List = None,
+        metadata: Dict = None,
+    ) -> Tuple[Dict, List, Dict, List[str]]:
+        # Only fetch the input variables that the destination block explicitly declares.
+        # If all the input variables are fetched, there is a chance that a lot of data from
+        # an upstream source block is loaded just to be used as inputs for the block’s
+        # decorated functions. Only do this for the notebook because
+        if from_notebook and self.is_data_integration():
+            input_vars, kwargs_vars, upstream_block_uuids = \
+                self.fetch_input_variables_and_catalog(
+                    input_args,
+                    execution_partition,
+                    global_vars,
+                    dynamic_block_index=dynamic_block_index,
+                    dynamic_block_indexes=dynamic_block_indexes,
+                    dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                    from_notebook=from_notebook,
+                )
+        else:
+            input_vars, kwargs_vars, upstream_block_uuids = self.fetch_input_variables(
+                input_args,
+                block_run_outputs_cache=block_run_outputs_cache,
+                dynamic_block_index=dynamic_block_index,
+                dynamic_block_indexes=dynamic_block_indexes,
+                dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                execution_partition=execution_partition,
+                from_notebook=from_notebook,
+                global_vars=global_vars,
+                metadata=metadata,
+            )
+
+        outputs_from_input_vars = {}
+        if input_args is None:
+            upstream_block_uuids_length = len(upstream_block_uuids)
+            for idx, input_var in enumerate(input_vars):
+                if idx < upstream_block_uuids_length:
+                    upstream_block_uuid = upstream_block_uuids[idx]
+                    outputs_from_input_vars[upstream_block_uuid] = input_var
+                    outputs_from_input_vars[f'df_{idx + 1}'] = input_var
+        else:
+            outputs_from_input_vars = dict()
+
+        return outputs_from_input_vars, input_vars, kwargs_vars, upstream_block_uuids
+
     def execute_block(
         self,
         block_run_outputs_cache: Dict[str, List] = None,
@@ -1367,6 +1475,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         run_settings: Dict = None,
         data_integration_runtime_settings: str = None,
         execution_partition_previous: str = None,
+        metadata: Dict = None,
         **kwargs,
     ) -> Dict:
         if logging_tags is None:
@@ -1388,25 +1497,8 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             logging_tags=logging_tags,
         ):
             # Fetch input variables
-
-            # Only fetch the input variables that the destination block explicitly declares.
-            # If all the input variables are fetched, there is a chance that a lot of data from
-            # an upstream source block is loaded just to be used as inputs for the block’s
-            # decorated functions. Only do this for the notebook because
-            if from_notebook and self.is_data_integration():
-                input_vars, kwargs_vars, upstream_block_uuids = \
-                    self.fetch_input_variables_and_catalog(
-                        input_args,
-                        execution_partition,
-                        global_vars,
-                        dynamic_block_index=dynamic_block_index,
-                        dynamic_block_indexes=dynamic_block_indexes,
-                        dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
-                        from_notebook=from_notebook,
-                    )
-            else:
-                input_vars, kwargs_vars, upstream_block_uuids = self.fetch_input_variables(
-                    input_args,
+            outputs_from_input_vars, input_vars, kwargs_vars, upstream_block_uuids = \
+                self.__get_outputs_from_input_vars(
                     block_run_outputs_cache=block_run_outputs_cache,
                     dynamic_block_index=dynamic_block_index,
                     dynamic_block_indexes=dynamic_block_indexes,
@@ -1414,23 +1506,15 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                     execution_partition=execution_partition,
                     from_notebook=from_notebook,
                     global_vars=global_vars,
+                    input_args=input_args,
+                    metadata=metadata,
                 )
 
-            outputs_from_input_vars = {}
-            if input_args is None:
-                upstream_block_uuids_length = len(upstream_block_uuids)
-                for idx, input_var in enumerate(input_vars):
-                    if idx < upstream_block_uuids_length:
-                        upstream_block_uuid = upstream_block_uuids[idx]
-                        outputs_from_input_vars[upstream_block_uuid] = input_var
-                        outputs_from_input_vars[f'df_{idx + 1}'] = input_var
-            else:
-                outputs_from_input_vars = dict()
-
             global_vars_copy = global_vars.copy()
             for kwargs_var in kwargs_vars:
                 if kwargs_var:
-                    global_vars_copy.update(kwargs_var)
+                    if isinstance(global_vars_copy, dict) and isinstance(kwargs_var, dict):
+                        global_vars_copy.update(kwargs_var)
 
             outputs = self._execute_block(
                 outputs_from_input_vars,
@@ -1641,9 +1725,9 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         # Initialize module
         block_uuid = self.uuid
         block_file_path = self.file_path
-        if self.replicated_block:
-            block_uuid = self.replicated_block
+        if self.replicated_block and self.replicated_block_object:
             block_file_path = self.replicated_block_object.file_path
+            block_uuid = self.replicated_block
 
         spec = importlib.util.spec_from_file_location(block_uuid, block_file_path)
         module = importlib.util.module_from_spec(spec)
@@ -1686,6 +1770,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         from_notebook: bool = False,
         global_vars: Dict = None,
         upstream_block_uuids: List[str] = None,
+        metadata: Dict = None,
     ) -> Tuple[List, List, List]:
         """
         Fetch input variables for the current block's execution.
@@ -1709,6 +1794,13 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             Tuple[List, List, List]: A tuple containing the input variables, kwargs variables, and
                 upstream block UUIDs.
         """
+
+        dynamic_block_flags = []
+        if is_dynamic_block(self):
+            dynamic_block_flags.append(DynamicBlockFlag.DYNAMIC)
+        if is_dynamic_block_child(self):
+            dynamic_block_flags.append(DynamicBlockFlag.DYNAMIC_CHILD)
+
         variables = fetch_input_variables(
             self.pipeline,
             upstream_block_uuids or self.upstream_block_uuids,
@@ -1721,6 +1813,22 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             execution_partition=execution_partition,
             from_notebook=from_notebook,
             global_vars=global_vars,
+            dynamic_block_flags=dynamic_block_flags,
+            metadata=metadata,
+        )
+
+        DX_PRINTER.critical(
+            block=self,
+            metadata=metadata,
+            dynamic_block_index=dynamic_block_index,
+            dynamic_block_indexes=dynamic_block_indexes,
+            dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+            dynamic_block_flags=dynamic_block_flags,
+            execution_partition=execution_partition,
+            upstream_block_uuids=self.upstream_block_uuids,
+            upstream_block_uuids_override=upstream_block_uuids,
+            variables=len(variables) if variables else 'nothing',
+            __uuid='fetch_input_variables',
         )
 
         return variables
@@ -1753,19 +1861,30 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
     ):
         variable_manager = self.pipeline.variable_manager
 
-        block_uuid_use = block_uuid
-        clean_block_uuid = True
-        if dynamic_block_index is not None or is_dynamic_block_child(self):
-            block_uuid_use = os.path.join(*block_uuid.split(':'))
-            clean_block_uuid = False
+        block_uuid_use, changed = uuid_for_output_variables(
+            self,
+            block_uuid=block_uuid,
+            dynamic_block_index=dynamic_block_index,
+        )
 
-        return variable_manager.get_variables_by_block(
+        res = variable_manager.get_variables_by_block(
             self.pipeline.uuid,
             block_uuid=block_uuid_use,
-            clean_block_uuid=clean_block_uuid,
+            clean_block_uuid=not changed,
             partition=partition,
         )
 
+        DX_PRINTER.debug(
+            str(res),
+            block=self,
+            block_uuid_use=block_uuid_use,
+            clean_block_uuid=not changed,
+            partition=partition,
+            __uuid='get_variables_by_block',
+        )
+
+        return res
+
     def get_variable(
         self,
         block_uuid: str,
@@ -1777,22 +1896,34 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
     ):
         variable_manager = self.pipeline.variable_manager
 
-        block_uuid_use = block_uuid
-        clean_block_uuid = True
-        if dynamic_block_index is not None or is_dynamic_block_child(self):
-            block_uuid_use = os.path.join(*block_uuid.split(':'))
-            clean_block_uuid = False
+        block_uuid_use, changed = uuid_for_output_variables(
+            self,
+            block_uuid=block_uuid,
+            dynamic_block_index=dynamic_block_index,
+        )
 
-        return variable_manager.get_variable(
+        value = variable_manager.get_variable(
             self.pipeline.uuid,
             block_uuid=block_uuid_use,
-            clean_block_uuid=clean_block_uuid,
+            clean_block_uuid=not changed,
             partition=partition,
             raise_exception=raise_exception,
             spark=spark,
             variable_uuid=variable_uuid,
         )
 
+        DX_PRINTER.debug(
+            'get_variable',
+            block=self,
+            block_uuid_use=block_uuid_use,
+            clean_block_uuid=not changed,
+            partition=partition,
+            value=value,
+            __uuid='output_variables'
+        )
+
+        return value
+
     def get_variable_object(
         self,
         block_uuid: str,
@@ -1802,16 +1933,16 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
     ):
         variable_manager = self.pipeline.variable_manager
 
-        block_uuid_use = block_uuid
-        clean_block_uuid = True
-        if dynamic_block_index is not None or is_dynamic_block_child(self):
-            block_uuid_use = os.path.join(*block_uuid.split(':'))
-            clean_block_uuid = False
+        block_uuid, changed = uuid_for_output_variables(
+            self,
+            block_uuid=block_uuid,
+            dynamic_block_index=dynamic_block_index,
+        )
 
         return variable_manager.get_variable_object(
             self.pipeline.uuid,
-            block_uuid=block_uuid_use,
-            clean_block_uuid=clean_block_uuid,
+            block_uuid=block_uuid,
+            clean_block_uuid=not changed,
             partition=partition,
             spark=self.get_spark_session(),
             variable_uuid=variable_uuid,
@@ -1855,6 +1986,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         variable_type: VariableType = None,
         block_uuid: str = None,
         selected_variables: List[str] = None,
+        metadata: Dict = None,
     ) -> List[Dict]:
         if self.pipeline is None:
             return
@@ -1862,9 +1994,17 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         if not block_uuid:
             block_uuid = self.uuid
 
+        # The block_run’s block_uuid for replicated blocks will be in this format:
+        # [block_uuid]:[replicated_block_uuid]
+        # We need to use the original block_uuid to get the proper output.
+
+        # Block runs for dynamic child blocks will have the following block UUID:
+        # [block.uuid]:[index]
+        # Don’t use the original UUID even if the block is a replica because it will get rid of
+        # the dynamic child block index.
+
         data_products = []
         outputs = []
-        variable_manager = self.pipeline.variable_manager
 
         all_variables = self.get_variables_by_block(
             block_uuid=block_uuid,
@@ -1876,6 +2016,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 execution_partition=execution_partition,
             )
 
+        DX_PRINTER.debug(
+            all_variables=all_variables,
+            __uuid='get_variables_by_block',
+        )
+
         for v in all_variables:
             if selected_variables and v not in selected_variables:
                 continue
@@ -1894,91 +2039,22 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 sample_count=sample_count,
                 spark=self.get_spark_session(),
             )
-            if type(data) is pd.DataFrame:
-                if csv_lines_only:
-                    data = dict(
-                        table=data.to_csv(header=True, index=False).strip('\n').split('\n')
-                    )
-                else:
-                    try:
-                        analysis = variable_manager.get_variable(
-                            self.pipeline.uuid,
-                            block_uuid,
-                            v,
-                            dataframe_analysis_keys=['metadata', 'statistics'],
-                            partition=execution_partition,
-                            variable_type=VariableType.DATAFRAME_ANALYSIS,
-                        )
-                    except Exception:
-                        analysis = None
-                    if analysis is not None and \
-                            (analysis.get('statistics') or analysis.get('metadata')):
-
-                        stats = analysis.get('statistics', {})
-                        column_types = (analysis.get('metadata') or {}).get('column_types', {})
-                        row_count = stats.get('original_row_count', stats.get('count'))
-                        column_count = stats.get('original_column_count', len(column_types))
-                    else:
-                        row_count, column_count = data.shape
-
-                    columns_to_display = data.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-                    data = dict(
-                        sample_data=dict(
-                            columns=columns_to_display,
-                            rows=json.loads(
-                                data[columns_to_display].to_json(orient='split')
-                            )['data']
-                        ),
-                        shape=[row_count, column_count],
-                        type=DataType.TABLE,
-                        variable_uuid=v,
-                    )
-                data_products.append(data)
-                continue
-            elif is_geo_dataframe(data):
-                data = dict(
-                    text_data=f'''Use the code in a scratchpad to get the output of the block:
-
-from mage_ai.data_preparation.variable_manager import get_variable
-df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
-''',
-                    type=DataType.TEXT,
-                    variable_uuid=v,
-                )
-            elif type(data) is str:
-                data = dict(
-                    text_data=data,
-                    type=DataType.TEXT,
-                    variable_uuid=v,
-                )
-            elif type(data) is dict or type(data) is list:
-                data = dict(
-                    text_data=simplejson.dumps(
-                        data,
-                        default=datetime.isoformat,
-                        ignore_nan=True,
-                    ),
-                    type=DataType.TEXT,
-                    variable_uuid=v,
-                )
-            elif is_spark_dataframe(data):
-                df = data.toPandas()
-                columns_to_display = df.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-                data = dict(
-                    sample_data=dict(
-                        columns=columns_to_display,
-                        rows=json.loads(df[columns_to_display].to_json(orient='split'))['data']
-                    ),
-                    type=DataType.TABLE,
-                    variable_uuid=v,
-                )
+            data, is_data_product = self.__format_output_data(
+                data,
+                v,
+                block_uuid=block_uuid,
+                csv_lines_only=csv_lines_only,
+                execution_partition=execution_partition,
+            )
+            if is_data_product:
                 data_products.append(data)
-                continue
-            outputs.append(data)
+            else:
+                outputs.append(data)
         return outputs + data_products
 
     async def __get_outputs_async(
         self,
+        csv_lines_only: bool = False,
         execution_partition: str = None,
         include_print_outputs: bool = True,
         sample_count: int = DATAFRAME_SAMPLE_COUNT_PREVIEW,
@@ -2021,19 +2097,55 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 sample_count=sample_count,
                 spark=self.get_spark_session(),
             )
-            if type(data) is pd.DataFrame:
+            data, is_data_product = self.__format_output_data(
+                data,
+                v,
+                block_uuid=block_uuid,
+                csv_lines_only=csv_lines_only,
+                execution_partition=execution_partition,
+            )
+            if is_data_product:
+                data_products.append(data)
+            else:
+                outputs.append(data)
+        return outputs + data_products
+
+    def __format_output_data(
+        self,
+        data: Any,
+        variable_uuid: str,
+        block_uuid: str = None,
+        csv_lines_only: bool = False,
+        execution_partition: str = None,
+    ) -> Tuple[Dict, bool]:
+        """
+        Takes variable data and formats it to return to the frontend.
+
+        Returns:
+            Tuple[Dict, bool]: Tuple of the formatted data and is_data_product boolean. Data product
+                outputs and non data product outputs are handled slightly differently.
+        """
+        variable_manager = self.pipeline.variable_manager
+        if isinstance(data, pd.DataFrame):
+            if csv_lines_only:
+                data = dict(
+                    table=data.to_csv(header=True, index=False).strip('\n').split('\n')
+                )
+            else:
                 try:
                     analysis = variable_manager.get_variable(
                         self.pipeline.uuid,
                         block_uuid,
-                        v,
+                        variable_uuid,
                         dataframe_analysis_keys=['metadata', 'statistics'],
                         partition=execution_partition,
                         variable_type=VariableType.DATAFRAME_ANALYSIS,
                     )
                 except Exception:
                     analysis = None
-                if analysis is not None:
+                if analysis is not None and \
+                        (analysis.get('statistics') or analysis.get('metadata')):
+
                     stats = analysis.get('statistics', {})
                     column_types = (analysis.get('metadata') or {}).get('column_types', {})
                     row_count = stats.get('original_row_count', stats.get('count'))
@@ -2045,55 +2157,90 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 data = dict(
                     sample_data=dict(
                         columns=columns_to_display,
-                        rows=json.loads(data[columns_to_display].to_json(orient='split'))['data']
+                        rows=json.loads(
+                            data[columns_to_display].to_json(orient='split')
+                        )['data']
                     ),
                     shape=[row_count, column_count],
                     type=DataType.TABLE,
-                    variable_uuid=v,
+                    variable_uuid=variable_uuid,
                 )
-                data_products.append(data)
-                continue
-            elif is_geo_dataframe(data):
-                data = dict(
-                    text_data=f'''Use the code in a scratchpad to get the output of the block:
+            return data, True
+        elif isinstance(data, pl.DataFrame):
+            try:
+                analysis = variable_manager.get_variable(
+                    self.pipeline.uuid,
+                    block_uuid,
+                    variable_uuid,
+                    dataframe_analysis_keys=['statistics'],
+                    partition=execution_partition,
+                    variable_type=VariableType.DATAFRAME_ANALYSIS,
+                )
+            except Exception:
+                analysis = None
+            if analysis is not None:
+                stats = analysis.get('statistics', {})
+                row_count = stats.get('original_row_count')
+                column_count = stats.get('original_column_count')
+            else:
+                row_count, column_count = data.shape
+            columns_to_display = data.columns[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
+            data = dict(
+                sample_data=dict(
+                    columns=columns_to_display,
+                    rows=[
+                        list(row.values()) for row in json.loads(
+                            data[columns_to_display].write_json(row_oriented=True)
+                        )
+                    ]
+                ),
+                shape=[row_count, column_count],
+                type=DataType.TABLE,
+                variable_uuid=variable_uuid,
+            )
+            return data, True
+        elif is_geo_dataframe(data):
+            data = dict(
+                text_data=f'''Use the code in a scratchpad to get the output of the block:
 
 from mage_ai.data_preparation.variable_manager import get_variable
-df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
+df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
 ''',
-                    type=DataType.TEXT,
-                    variable_uuid=v,
-                )
-            elif type(data) is str:
-                data = dict(
-                    text_data=data,
-                    type=DataType.TEXT,
-                    variable_uuid=v,
-                )
-            elif type(data) is dict or type(data) is list:
-                data = dict(
-                    text_data=simplejson.dumps(
-                        data,
-                        default=datetime.isoformat,
-                        ignore_nan=True,
-                    ),
-                    type=DataType.TEXT,
-                    variable_uuid=v,
-                )
-            elif is_spark_dataframe(data):
-                df = data.toPandas()
-                columns_to_display = df.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-                data = dict(
-                    sample_data=dict(
-                        columns=columns_to_display,
-                        rows=json.loads(df[columns_to_display].to_json(orient='split'))['data']
-                    ),
-                    type=DataType.TABLE,
-                    variable_uuid=v,
-                )
-                data_products.append(data)
-                continue
-            outputs.append(data)
-        return outputs + data_products
+                type=DataType.TEXT,
+                variable_uuid=variable_uuid,
+            )
+            return data, False
+        elif type(data) is str:
+            data = dict(
+                text_data=data,
+                type=DataType.TEXT,
+                variable_uuid=variable_uuid,
+            )
+            return data, False
+        elif type(data) is dict or type(data) is list:
+            data = dict(
+                text_data=simplejson.dumps(
+                    data,
+                    default=datetime.isoformat,
+                    ignore_nan=True,
+                ),
+                type=DataType.TEXT,
+                variable_uuid=variable_uuid,
+            )
+            return data, False
+        elif is_spark_dataframe(data):
+            df = data.toPandas()
+            columns_to_display = df.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
+            data = dict(
+                sample_data=dict(
+                    columns=columns_to_display,
+                    rows=json.loads(df[columns_to_display].to_json(orient='split'))['data']
+                ),
+                type=DataType.TABLE,
+                variable_uuid=variable_uuid,
+            )
+            return data, True
+        return data, False
 
     def __save_outputs_prepare(self, outputs, override_output_variable: bool = False) -> Dict:
         variable_mapping = dict()
@@ -2225,7 +2372,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                         self.replicated_block and \
                         BlockType.GLOBAL_DATA_PRODUCT != self.type:
 
-                    file_path = self.file.file_path
+                    file_path = self.file_path
                     if not os.path.isfile(file_path):
                         data['error'] = dict(
                             error='No such file or directory',
@@ -2384,6 +2531,9 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
         widget=False,
         error_if_file_missing: bool = True,
     ) -> 'Block':
+        if self.replicated_block:
+            return self
+
         if error_if_file_missing and not self.file.exists():
             raise Exception(f'File for block {self.uuid} does not exist at {self.file.file_path}.')
 
@@ -2400,6 +2550,9 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
         error_if_file_missing: bool = True,
         widget: bool = False,
     ) -> 'Block':
+        if self.replicated_block:
+            return self
+
         if error_if_file_missing and not self.file.exists():
             raise Exception(f'File for block {self.uuid} does not exist at {self.file.file_path}.')
 
@@ -2583,7 +2736,12 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                 logging_tags=logging_tags,
             )
 
-    def analyze_outputs(self, variable_mapping, shape_only: bool = False) -> None:
+    def analyze_outputs(
+        self,
+        variable_mapping,
+        execution_partition: str = None,
+        shape_only: bool = False,
+    ) -> None:
         if self.pipeline is None:
             return
         for uuid, data in variable_mapping.items():
@@ -2599,6 +2757,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                                 original_column_count=data.shape[1],
                             ),
                         ),
+                        partition=execution_partition,
                         variable_type=VariableType.DATAFRAME_ANALYSIS,
                     )
                     continue
@@ -2626,6 +2785,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                             insights=analysis['insights'],
                             suggestions=analysis['suggestions'],
                         ),
+                        partition=execution_partition,
                         variable_type=VariableType.DATAFRAME_ANALYSIS,
                     )
                 except Exception:
@@ -2633,6 +2793,20 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                     # TODO: we use to silently fail, but it looks bad when using BigQuery
                     # print('\nFailed to analyze dataframe:')
                     # print(traceback.format_exc())
+            elif type(data) is pl.DataFrame:
+                self.pipeline.variable_manager.add_variable(
+                    self.pipeline.uuid,
+                    self.uuid,
+                    uuid,
+                    dict(
+                        statistics=dict(
+                            original_row_count=data.shape[0],
+                            original_column_count=data.shape[1],
+                        ),
+                    ),
+                    partition=execution_partition,
+                    variable_type=VariableType.DATAFRAME_ANALYSIS,
+                )
 
     def set_global_vars(self, global_vars: Dict) -> None:
         self.global_vars = global_vars
@@ -2794,11 +2968,11 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
     ) -> Dict:
         self.dynamic_block_uuid = dynamic_block_uuid
 
-        block_uuid = self.uuid
-        clean_block_uuid = True
-        if dynamic_block_uuid is not None:
-            block_uuid = os.path.join(*dynamic_block_uuid.split(':'))
-            clean_block_uuid = False
+        block_uuid, changed = uuid_for_output_variables(
+            self,
+            block_uuid=self.uuid,
+            dynamic_block_uuid=dynamic_block_uuid,
+        )
 
         if self.pipeline is None:
             return
@@ -2807,7 +2981,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
             self.pipeline.uuid,
             block_uuid=block_uuid,
             partition=execution_partition,
-            clean_block_uuid=clean_block_uuid,
+            clean_block_uuid=not changed,
         )
 
         variable_mapping = self.__consolidate_variables(variable_mapping)
@@ -2844,24 +3018,23 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
             dynamic_block_uuid=dynamic_block_uuid,
         )
 
-        block_uuid = self.uuid
-        clean_block_uuid = True
-        if dynamic_block_uuid is not None:
-            block_uuid = os.path.join(*dynamic_block_uuid.split(':'))
-            clean_block_uuid = False
+        block_uuid, changed = uuid_for_output_variables(
+            self,
+            block_uuid=self.uuid,
+            dynamic_block_uuid=dynamic_block_uuid,
+        )
 
         for uuid, data in variables_data['variable_mapping'].items():
             if spark is not None and self.pipeline.type == PipelineType.PYSPARK \
                     and type(data) is pd.DataFrame:
                 data = spark.createDataFrame(data)
-
             self.pipeline.variable_manager.add_variable(
                 self.pipeline.uuid,
                 block_uuid,
                 uuid,
                 data,
                 partition=execution_partition,
-                clean_block_uuid=clean_block_uuid,
+                clean_block_uuid=not changed,
             )
 
         for uuid in variables_data['removed_variables']:
@@ -2888,11 +3061,11 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
             dynamic_block_uuid,
         )
 
-        block_uuid = self.uuid
-        clean_block_uuid = True
-        if dynamic_block_uuid is not None:
-            block_uuid = os.path.join(*dynamic_block_uuid.split(':'))
-            clean_block_uuid = False
+        block_uuid, changed = uuid_for_output_variables(
+            self,
+            block_uuid=self.uuid,
+            dynamic_block_uuid=dynamic_block_uuid,
+        )
 
         for uuid, data in variables_data['variable_mapping'].items():
             if spark is not None and type(data) is pd.DataFrame:
@@ -2904,7 +3077,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                 uuid,
                 data,
                 partition=execution_partition,
-                clean_block_uuid=clean_block_uuid,
+                clean_block_uuid=not changed,
             )
 
         for uuid in variables_data['removed_variables']:
@@ -2912,7 +3085,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                 self.pipeline.uuid,
                 block_uuid,
                 uuid,
-                clean_block_uuid=clean_block_uuid,
+                clean_block_uuid=not changed,
             )
 
     def input_variables(self, execution_partition: str = None) -> Dict[str, List[str]]:
@@ -3055,18 +3228,72 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
         2. Update the folder of variable
         3. Update upstream and downstream relationships
         """
+        file_extension = Path(self.file_path).suffix if self.file_path else ''
+        directory_name = self.file_directory_name(self.type)
+
         old_uuid = self.uuid
         # This has to be here
         old_file_path = self.file_path
         block_content = self.content
 
+        # load_titanic
         new_uuid = clean_name(name)
         self.name = name
         self.uuid = new_uuid
+
+        # This file has a path in its file_source that must be updated.
+        if project_platform_activated() and \
+                self.file_source_path() and \
+                add_absolute_path(self.file_source_path()) == self.file_path:
+
+            #  /home/src/data-vault/perftools/mage/data_loaders/team/illusory_glitter
+            old_file_path_without_extension = str(Path(old_file_path).with_suffix(''))
+            #  /home/src/data-vault/perftools/mage/data_loaders/team
+            old_file_path_without_uuid = str(Path(old_file_path_without_extension.replace(
+                str(Path(old_uuid)),
+                '',
+            )))
+
+            #  perftools/mage/data_loaders/team
+            old_file_path_without_repo_path = remove_base_repo_path(old_file_path_without_uuid)
+            #  perftools/mage
+            path_without_block_directory = str(old_file_path_without_repo_path).split(
+                directory_name,
+            )[0]
+
+            file_extension_new = Path(self.uuid).suffix or file_extension
+            # perftools/mage/data_loaders/load_titanic.py
+            new_path = str(Path(os.path.join(
+                path_without_block_directory,
+                directory_name,
+                str(Path(self.uuid).with_suffix('')),
+            )).with_suffix(file_extension_new))
+
+            configuration = self.configuration or {}
+            if not configuration.get('file_source'):
+                configuration['file_source'] = {}
+            configuration['file_source']['path'] = new_path
+            self.configuration = configuration
+
         # This has to be here
         new_file_path = self.file_path
 
         if self.pipeline is not None:
+            DX_PRINTER.critical(
+                block=self,
+                old_uuid=old_uuid,
+                old_file_path=old_file_path,
+                block_content=block_content,
+                new_uuid=new_uuid,
+                name=self.name,
+                uuid=self.uuid,
+                file_path=new_file_path,
+                pipeline=self.pipeline.uuid,
+                repo_path=self.pipeline.repo_path,
+                configuration=self.configuration,
+                __uuid='__update_name',
+            )
+
             if self.pipeline.has_block(
                 new_uuid,
                 block_type=self.type,
@@ -3110,7 +3337,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
                     language=self.language,
                     pipeline=self.pipeline,
                 )
-                cache.remove_pipeline(old_block, self.pipeline.uuid)
+                cache.remove_pipeline(old_block, self.pipeline.uuid, self.pipeline.repo_path)
             else:
                 cache.move_pipelines(self, dict(
                     type=self.type,