mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl

mage_ai/data_preparation/models/block/__init__.py

@@ -14,7 +14,18 @@ from inspect import Parameter, isfunction, signature
 from logging import Logger
 from pathlib import Path
 from queue import Queue
-from typing import Any, Callable, Dict, Generator, List, Set, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 import inflection
 import pandas as pd
@@ -25,7 +36,8 @@ from jinja2 import Template
 
 import mage_ai.data_preparation.decorators
 from mage_ai.cache.block import BlockCache
-from mage_ai.data_cleaner.shared.utils import is_geo_dataframe, is_spark_dataframe
+from mage_ai.data.constants import InputDataType
+from mage_ai.data.tabular.models import BatchSettings
 from mage_ai.data_integrations.sources.constants import SQL_SOURCES_MAPPING
 from mage_ai.data_preparation.logging.logger import DictLogger
 from mage_ai.data_preparation.logging.logger_manager_factory import LoggerManagerFactory
@@ -43,6 +55,7 @@ from mage_ai.data_preparation.models.block.dynamic.utils import (
     uuid_for_output_variables,
 )
 from mage_ai.data_preparation.models.block.dynamic.variables import (
+    LazyVariableSet,
     delete_variable_objects_for_dynamic_child,
     fetch_input_variables_for_dynamic_upstream_blocks,
     get_outputs_for_dynamic_block,
@@ -51,10 +64,26 @@ from mage_ai.data_preparation.models.block.dynamic.variables import (
 )
 from mage_ai.data_preparation.models.block.errors import HasDownstreamDependencies
 from mage_ai.data_preparation.models.block.extension.utils import handle_run_tests
+from mage_ai.data_preparation.models.block.outputs import (
+    format_output_data,
+    get_outputs_for_display_async,
+    get_outputs_for_display_dynamic_block,
+    get_outputs_for_display_sync,
+)
 from mage_ai.data_preparation.models.block.platform.mixins import (
     ProjectPlatformAccessible,
 )
 from mage_ai.data_preparation.models.block.platform.utils import from_another_project
+from mage_ai.data_preparation.models.block.settings.dynamic.mixins import DynamicMixin
+from mage_ai.data_preparation.models.block.settings.global_data_products.mixins import (
+    GlobalDataProductsMixin,
+)
+from mage_ai.data_preparation.models.block.settings.variables.mixins import (
+    VariablesMixin,
+)
+from mage_ai.data_preparation.models.block.settings.variables.models import (
+    ChunkKeyTypeUnion,
+)
 from mage_ai.data_preparation.models.block.spark.mixins import SparkBlock
 from mage_ai.data_preparation.models.block.utils import (
     clean_name,
@@ -64,13 +93,14 @@ from mage_ai.data_preparation.models.block.utils import (
     is_valid_print_variable,
     output_variables,
 )
-from mage_ai.data_preparation.models.constants import (
+from mage_ai.data_preparation.models.constants import (  # PIPELINES_FOLDER,
     BLOCK_LANGUAGE_TO_FILE_EXTENSION,
     CALLBACK_STATUSES,
     CUSTOM_EXECUTION_BLOCK_TYPES,
     DATAFRAME_ANALYSIS_MAX_COLUMNS,
     DATAFRAME_ANALYSIS_MAX_ROWS,
     DATAFRAME_SAMPLE_COUNT_PREVIEW,
+    DYNAMIC_CHILD_BLOCK_SAMPLE_COUNT_PREVIEW,
     FILE_EXTENSION_TO_BLOCK_LANGUAGE,
     NON_PIPELINE_EXECUTABLE_BLOCK_TYPES,
     BlockColor,
@@ -82,18 +112,36 @@ from mage_ai.data_preparation.models.constants import (
     PipelineType,
 )
 from mage_ai.data_preparation.models.file import File
-from mage_ai.data_preparation.models.variable import VariableType
+from mage_ai.data_preparation.models.utils import is_basic_iterable, warn_for_repo_path
+from mage_ai.data_preparation.models.variable import Variable
+from mage_ai.data_preparation.models.variables.cache import (
+    AggregateInformation,
+    AggregateInformationData,
+    InformationData,
+    VariableAggregateCache,
+)
+from mage_ai.data_preparation.models.variables.constants import (
+    VariableAggregateDataType,
+    VariableAggregateSummaryGroupType,
+    VariableType,
+)
+from mage_ai.data_preparation.models.variables.summarizer import (
+    aggregate_summary_info_for_all_variables,
+    get_aggregate_summary_info,
+)
 from mage_ai.data_preparation.repo_manager import RepoConfig
 from mage_ai.data_preparation.shared.stream import StreamToLogger
 from mage_ai.data_preparation.shared.utils import get_template_vars
 from mage_ai.data_preparation.templates.data_integrations.utils import get_templates
 from mage_ai.data_preparation.templates.template import load_template
-from mage_ai.server.kernel_output_parser import DataType
+from mage_ai.data_preparation.variable_manager import VariableManager
+from mage_ai.io.base import ExportWritePolicy
 from mage_ai.services.spark.config import SparkConfig
 from mage_ai.services.spark.spark import SPARK_ENABLED, get_spark_session
 from mage_ai.settings.platform.constants import project_platform_activated
-from mage_ai.settings.repo import get_repo_path
-from mage_ai.shared.array import unique_by
+from mage_ai.settings.repo import base_repo_path_directory_name, get_repo_path
+from mage_ai.settings.server import VARIABLE_DATA_OUTPUT_META_CACHE
+from mage_ai.shared.array import is_iterable, unique_by
 from mage_ai.shared.constants import ENV_DEV, ENV_TEST
 from mage_ai.shared.custom_logger import DX_PRINTER
 from mage_ai.shared.environments import get_env, is_debug
@@ -110,6 +158,10 @@ from mage_ai.shared.strings import format_enum
 from mage_ai.shared.utils import clean_name as clean_name_orig
 from mage_ai.shared.utils import is_spark_env
 
+# from mage_ai.system.memory.manager import MemoryManager
+from mage_ai.system.memory.wrappers import execute_with_memory_tracking
+from mage_ai.system.models import ResourceUsage
+
 PYTHON_COMMAND = 'python3'
 BLOCK_EXISTS_ERROR = '[ERR_BLOCK_EXISTS]'
 
@@ -174,8 +226,11 @@ async def run_blocks(
     while not blocks.empty():
         block = blocks.get()
 
-        if block.type in NON_PIPELINE_EXECUTABLE_BLOCK_TYPES or \
-                not run_sensors and block.type == BlockType.SENSOR:
+        if (
+            block.type in NON_PIPELINE_EXECUTABLE_BLOCK_TYPES
+            or not run_sensors
+            and block.type == BlockType.SENSOR
+        ):
             continue
 
         if tries_by_block_uuid.get(block.uuid, None) is None:
@@ -290,7 +345,14 @@ def run_blocks_sync(
                 blocks.put(downstream_block)
 
 
-class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
+class Block(
+    DataIntegrationMixin,
+    SparkBlock,
+    ProjectPlatformAccessible,
+    DynamicMixin,
+    GlobalDataProductsMixin,
+    VariablesMixin,
+):
     def __init__(
         self,
         name: str,
@@ -376,6 +438,13 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         # Needs to after self._project_platform_activated = None
         self.configuration = configuration
 
+        self.resource_usage = None
+        self._store_variables_in_block_function: Optional[
+            Callable[..., Optional[List[VariableType]]]
+        ] = None
+
+        self._variable_aggregate_cache = None
+
     @property
     def uuid(self) -> str:
         return self._uuid
@@ -449,6 +518,166 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             )
         return None
 
+    @property
+    def variable_manager(self) -> VariableManager:
+        if not self.pipeline:
+            return
+        return self.pipeline.variable_manager
+
+    @property
+    def pipeline_uuid(self) -> str:
+        return self.pipeline.uuid if self.pipeline else ''
+
+    def __load_variable_aggregate_cache(self, variable_uuid: str) -> VariableAggregateCache:
+        if not self._variable_aggregate_cache:
+            self._variable_aggregate_cache = {variable_uuid: VariableAggregateCache()}
+
+        if variable_uuid not in self._variable_aggregate_cache:
+            self._variable_aggregate_cache[variable_uuid] = VariableAggregateCache()
+
+        return self._variable_aggregate_cache[variable_uuid]
+
+    def get_variable_aggregate_cache(
+        self,
+        variable_uuid: str,
+        data_type: VariableAggregateDataType,
+        default_group_type: Optional[VariableAggregateSummaryGroupType] = None,
+        group_type: Optional[VariableAggregateSummaryGroupType] = None,
+        infer_group_type: Optional[bool] = None,
+        partition: Optional[str] = None,
+    ) -> Optional[Union[AggregateInformationData, InformationData]]:
+        if not VARIABLE_DATA_OUTPUT_META_CACHE:
+            return
+
+        cache = self.__load_variable_aggregate_cache(variable_uuid)
+        cache = VariableAggregateCache.load(cache)
+
+        if infer_group_type:
+            group_type = (
+                VariableAggregateSummaryGroupType.DYNAMIC
+                if is_dynamic_block_child(self)
+                else VariableAggregateSummaryGroupType.PARTS
+            )
+
+        keys = [v.value for v in [group_type, data_type] if v is not None]
+        value = functools.reduce(getattr, keys, cache)
+
+        if not value:
+            cache_new = get_aggregate_summary_info(
+                self.variable_manager,
+                self.pipeline_uuid,
+                self.uuid,
+                variable_uuid,
+                data_type,
+                default_group_type=default_group_type,
+                group_type=group_type,
+                partition=partition,
+            )
+            group_value_use = (group_type.value if group_type else None) or (
+                default_group_type.value if default_group_type else None
+            )
+            if group_value_use is not None:
+                cache_group = AggregateInformation.load(getattr(cache, group_value_use))
+                cache_group_new = AggregateInformation.load(getattr(cache_new, group_value_use))
+                if cache_group_new:
+                    for data in VariableAggregateDataType:
+                        val = getattr(cache_group, data.value)
+                        val_new = getattr(cache_group_new, data.value)
+                        cache_group.update_attributes(**{
+                            data.value: val_new or val,
+                        })
+                cache.update_attributes(**{
+                    group_value_use: AggregateInformation.load(cache_group)
+                })
+
+            for data in VariableAggregateDataType:
+                val = getattr(cache, data.value)
+                val_new = getattr(cache_new, data.value)
+                cache.update_attributes(**{
+                    data.value: val_new or val,
+                })
+
+            cache = VariableAggregateCache.load(cache)
+            self._variable_aggregate_cache = merge_dict(
+                self._variable_aggregate_cache or {},
+                {variable_uuid: cache},
+            )
+            value = functools.reduce(getattr, keys, cache)
+
+        return value
+
+    def get_resource_usage(
+        self,
+        block_uuid: Optional[str] = None,
+        index: Optional[int] = None,
+        partition: Optional[str] = None,
+        variable_uuid: Optional[str] = None,
+    ) -> Optional[ResourceUsage]:
+        try:
+            if not VARIABLE_DATA_OUTPUT_META_CACHE:
+                variable = self.get_variable_object(
+                    block_uuid or self.uuid, partition=partition, variable_uuid=variable_uuid
+                )
+                return variable.get_resource_usage(index=index)
+
+            values = self.get_variable_aggregate_cache(
+                variable_uuid,
+                VariableAggregateDataType.RESOURCE_USAGE,
+                infer_group_type=index is not None,
+                partition=partition,
+            )
+
+            if index is not None:
+                if values and isinstance(values, list) and len(values) > index:
+                    values = values[index]
+            else:
+                values = values
+
+            if isinstance(values, Iterable) and len(values) >= 1:
+                values = values[0]
+
+            return values
+        except Exception as err:
+            print(f'[ERROR] Block.get_resource_usage: {err}')
+            return ResourceUsage()
+
+    def get_analysis(
+        self,
+        block_uuid: Optional[str] = None,
+        index: Optional[int] = None,
+        partition: Optional[str] = None,
+        variable_uuid: Optional[str] = None,
+    ) -> Optional[Dict]:
+        try:
+            if not VARIABLE_DATA_OUTPUT_META_CACHE:
+                variable = self.get_variable_object(
+                    block_uuid or self.uuid, partition=partition, variable_uuid=variable_uuid
+                )
+                return variable.get_analysis(index=index)
+
+            values = self.get_variable_aggregate_cache(
+                variable_uuid,
+                VariableAggregateDataType.STATISTICS,
+                infer_group_type=index is not None,
+                partition=partition,
+            )
+
+            value = None
+            if index is not None:
+                if values and isinstance(values, list) and len(values) > index:
+                    value = values[index]
+            else:
+                value = values
+
+            if isinstance(value, Iterable) and len(value) >= 1:
+                value = value[0]
+
+            if value is not None:
+                return dict(statistics=value.to_dict() if value else {})
+        except Exception as err:
+            print(f'[ERROR] Block.get_analysis: {err}')
+            return {}
+
     async def content_async(self) -> str:
         if self.replicated_block and self.replicated_block_object:
             self._content = await self.replicated_block_object.content_async()
@@ -483,32 +712,42 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             upstream_block_uuids = self.upstream_block_uuids
 
         if outputs_from_input_vars is None:
-
-            if BlockLanguage.SQL == self.language and any([is_dynamic_block(
-                upstream_block,
-            ) or is_dynamic_block_child(
-                upstream_block,
-            ) for upstream_block in self.upstream_blocks]):
-                outputs_from_input_vars, _kwargs_vars, upstream_block_uuids = \
-                    fetch_input_variables_for_dynamic_upstream_blocks(
-                        self,
-                        None,
-                        dynamic_block_index=dynamic_block_index,
-                        dynamic_block_indexes=dynamic_block_indexes,
-                        execution_partition=execution_partition,
-                        from_notebook=from_notebook,
-                        global_vars=variables,
-                    )
+            if BlockLanguage.SQL == self.language and any([
+                is_dynamic_block(
+                    upstream_block,
+                )
+                or is_dynamic_block_child(
+                    upstream_block,
+                )
+                for upstream_block in self.upstream_blocks
+            ]):
+                (
+                    outputs_from_input_vars,
+                    _kwargs_vars,
+                    upstream_block_uuids,
+                ) = fetch_input_variables_for_dynamic_upstream_blocks(
+                    self,
+                    None,
+                    dynamic_block_index=dynamic_block_index,
+                    dynamic_block_indexes=dynamic_block_indexes,
+                    execution_partition=execution_partition,
+                    from_notebook=from_notebook,
+                    global_vars=variables,
+                )
             else:
-                outputs_from_input_vars, _input_vars, _kwargs_vars, upstream_block_uuids = \
-                    self.__get_outputs_from_input_vars(
-                        dynamic_block_index=dynamic_block_index,
-                        dynamic_block_indexes=dynamic_block_indexes,
-                        dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
-                        execution_partition=execution_partition,
-                        from_notebook=from_notebook,
-                        global_vars=variables,
-                    )
+                (
+                    outputs_from_input_vars,
+                    _input_vars,
+                    _kwargs_vars,
+                    upstream_block_uuids,
+                ) = self.__get_outputs_from_input_vars(
+                    dynamic_block_index=dynamic_block_index,
+                    dynamic_block_indexes=dynamic_block_indexes,
+                    dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                    execution_partition=execution_partition,
+                    from_notebook=from_notebook,
+                    global_vars=variables,
+                )
 
         return hydrate_block_outputs(
             content,
@@ -524,7 +763,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             if BlockLanguage.YAML == self.language:
                 content = await self.content_async()
                 if content:
-                    text = self.interpolate_content(content)
+                    try:
+                        text = self.interpolate_content(content)
+                    except Exception:
+                        traceback.print_exc()
+                        text = content
                     settings = yaml.safe_load(text)
                     uuid = settings.get('source') or settings.get('destination')
                     mapping = grouped_templates.get(uuid) or {}
@@ -550,11 +793,14 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
                     di_metadata = merge_dict(
                         extract(mapping or {}, ['name']),
-                        ignore_keys(di_settings or {}, [
-                            'catalog',
-                            'config',
-                            'data_integration_uuid',
-                        ]),
+                        ignore_keys(
+                            di_settings or {},
+                            [
+                                'catalog',
+                                'config',
+                                'data_integration_uuid',
+                            ],
+                        ),
                     )
                     di_metadata['sql'] = uuid in SQL_SOURCES_MAPPING
 
@@ -572,9 +818,8 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
     @property
     def executable(self) -> bool:
-        return (
-            self.type not in NON_PIPELINE_EXECUTABLE_BLOCK_TYPES
-            and (self.pipeline is None or self.pipeline.type != PipelineType.STREAMING)
+        return self.type not in NON_PIPELINE_EXECUTABLE_BLOCK_TYPES and (
+            self.pipeline is None or self.pipeline.type != PipelineType.STREAMING
         )
 
     @property
@@ -584,10 +829,15 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 self._outputs = self.get_outputs()
         return self._outputs
 
-    async def __outputs_async(self) -> List:
+    async def __outputs_async(
+        self, exclude_blank_variable_uuids: bool = False, max_results: Optional[int] = None
+    ) -> List:
         if not self._outputs_loaded:
             if self._outputs is None or len(self._outputs) == 0:
-                self._outputs = await self.__get_outputs_async()
+                self._outputs = await self.__get_outputs_async(
+                    exclude_blank_variable_uuids=exclude_blank_variable_uuids,
+                    max_results=max_results,
+                )
         return self._outputs
 
     @property
@@ -622,15 +872,20 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         block_uuid: str,
         block_type: BlockType,
         language: BlockLanguage,
+        relative_path: bool = False,
     ) -> str:
         file_extension = BLOCK_LANGUAGE_TO_FILE_EXTENSION[language]
         block_directory = f'{block_type}s' if block_type != BlockType.CUSTOM else block_type
 
-        return os.path.join(
-            repo_path or os.getcwd(),
+        parts = []
+        if not relative_path:
+            parts.append(repo_path or os.getcwd())
+        parts += [
             block_directory,
             f'{block_uuid}.{file_extension}',
-        )
+        ]
+
+        return os.path.join(*parts)
 
     @property
     def file_path(self) -> str:
@@ -645,12 +900,59 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             return add_absolute_path(file_path)
 
         return self.__build_file_path(
-            self.repo_path or os.getcwd(),
+            self.repo_path,
             self.uuid,
             self.type,
             self.language,
         )
 
+    def build_file_path_directory(
+        self,
+        block_uuid: Optional[str] = None,
+    ) -> Tuple[Optional[str], Optional[str]]:
+        file_path = None
+        file_path_absolute = None
+
+        if self.replicated_block and self.replicated_block_object:
+            (
+                file_path_absolute,
+                file_path,
+            ) = self.replicated_block_object.build_file_path_directory(
+                block_uuid=block_uuid,
+            )
+
+        if not file_path:
+            file_path = self.get_file_path_from_source() or self.configuration.get('file_path')
+
+        if not file_path:
+            file_path = self.__build_file_path(
+                self.repo_path or os.getcwd(),
+                self.uuid,
+                self.type,
+                self.language,
+                relative_path=True,
+            )
+
+        if block_uuid:
+            old_file_path = Path(file_path)
+            old_file_extension = old_file_path.suffix
+            new_file_name = f'{block_uuid}{old_file_extension}'
+            file_path = str(old_file_path.with_name(new_file_name))
+
+        if file_path:
+            file_path_absolute = add_absolute_path(file_path)
+
+            if not file_path_absolute and block_uuid:
+                file_path_absolute = self.__build_file_path(
+                    self.repo_path or os.getcwd(),
+                    block_uuid,
+                    self.type,
+                    self.language,
+                    relative_path=False,
+                )
+
+        return file_path_absolute, file_path
+
     @property
     def file(self) -> File:
         if self.replicated_block and self.replicated_block_object:
@@ -661,15 +963,29 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             if file:
                 return file
 
-        return File.from_path(self.file_path)
+        repo_path = self.pipeline.repo_path if self.pipeline else None
+        new_file = File.from_path(self.file_path, repo_path=repo_path)
+
+        if not new_file.filename or not new_file.dir_path:
+            new_file = File.from_path(
+                self.__build_file_path(
+                    new_file.repo_path,
+                    self.uuid,
+                    self.type,
+                    self.language,
+                )
+            )
+
+        return new_file
 
     @property
     def table_name(self) -> str:
         if self.configuration and self.configuration.get('data_provider_table'):
             return self.configuration['data_provider_table']
 
-        table_name = f'{self.pipeline.uuid}_{clean_name_orig(self.uuid)}_'\
-                     f'{self.pipeline.version_name}'
+        table_name = (
+            f'{self.pipeline_uuid}_{clean_name_orig(self.uuid)}_' f'{self.pipeline.version_name}'
+        )
 
         env = (self.global_vars or dict()).get('env')
         if env == ENV_DEV:
@@ -680,31 +996,15 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         return table_name
 
     @property
-    def full_table_name(self) -> str:
+    def full_table_name(self) -> Optional[str]:
         from mage_ai.data_preparation.models.block.sql.utils.shared import (
-            extract_create_statement_table_name,
-            extract_insert_statement_table_names,
-            extract_update_statement_table_names,
+            extract_full_table_name,
         )
 
-        if not self.content:
-            return None
-
-        table_name = extract_create_statement_table_name(self.content)
-        if table_name:
-            return table_name
-
-        matches = extract_insert_statement_table_names(self.content)
-        if len(matches) == 0:
-            matches = extract_update_statement_table_names(self.content)
-
-        if len(matches) == 0:
-            return None
-
-        return matches[len(matches) - 1]
+        return extract_full_table_name(self.content)
 
     @classmethod
-    def after_create(self, block: 'Block', **kwargs) -> None:
+    def after_create(cls, block: 'Block', **kwargs) -> None:
         widget = kwargs.get('widget')
         pipeline = kwargs.get('pipeline')
         if pipeline is not None:
@@ -766,28 +1066,30 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         # Don’t create a file if it’s from another project.
 
         file_path_from_source = (
-            configuration and
-            configuration.get('file_source') and
-            (configuration.get('file_source') or {}).get('path')
+            configuration
+            and configuration.get('file_source')
+            and (configuration.get('file_source') or {}).get('path')
         )
-        file_is_from_another_project = (
-            file_path_from_source and
-            from_another_project(
-                file_path=file_path_from_source,
-                other_file_path=pipeline.dir_path if pipeline else None,
+        file_is_from_another_project = file_path_from_source and from_another_project(
+            file_path=file_path_from_source,
+            other_file_path=pipeline.dir_path if pipeline else None,
+        )
+        absolute_file_path = (
+            add_root_repo_path_to_relative_path(
+                file_path_from_source,
             )
+            if file_path_from_source
+            else None
         )
-        absolute_file_path = add_root_repo_path_to_relative_path(
-            file_path_from_source,
-        ) if file_path_from_source else None
-
-        if not file_is_from_another_project and \
-                (not absolute_file_path or not os.path.exists(absolute_file_path)):
-
-            if not replicated_block and \
-                    (BlockType.DBT != block_type or BlockLanguage.YAML == language) and \
-                    BlockType.GLOBAL_DATA_PRODUCT != block_type:
 
+        if not file_is_from_another_project and (
+            not absolute_file_path or not os.path.exists(absolute_file_path)
+        ):
+            if (
+                not replicated_block
+                and (BlockType.DBT != block_type or BlockLanguage.YAML == language)
+                and BlockType.GLOBAL_DATA_PRODUCT != block_type
+            ):
                 block_directory = self.file_directory_name(block_type)
                 if absolute_file_path:
                     block_dir_path = os.path.dirname(absolute_file_path)
@@ -803,19 +1105,24 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 file_path = os.path.join(block_dir_path, f'{uuid}.{file_extension}')
                 if os.path.exists(file_path):
                     already_exists = True
-                    if (pipeline is not None and pipeline.has_block(
-                        uuid,
-                        block_type=block_type,
-                        extension_uuid=extension_uuid,
-                    )) or require_unique_name:
+                    if (
+                        pipeline is not None
+                        and pipeline.has_block(
+                            uuid,
+                            block_type=block_type,
+                            extension_uuid=extension_uuid,
+                        )
+                    ) or require_unique_name:
                         """
                         The BLOCK_EXISTS_ERROR constant is used on the frontend to identify when
                         a user is trying to create a new block with an existing block name, and
                         link them to the existing block file so the user can choose to add the
                         existing block to their pipeline.
                         """
-                        raise Exception(f'{BLOCK_EXISTS_ERROR} Block {uuid} already exists. \
-                                        Please use a different name.')
+                        raise Exception(
+                            f'{BLOCK_EXISTS_ERROR} Block {uuid} already exists. \
+                                        Please use a different name.'
+                        )
                 else:
                     load_template(
                         block_type,
@@ -830,8 +1137,9 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             if not configuration.get('file_source'):
                 configuration['file_source'] = {}
             if not configuration['file_source'].get('path'):
+                relative_path = str(Path(repo_path).relative_to(base_repo_path_directory_name()))
                 configuration['file_source']['path'] = self.__build_file_path(
-                    get_repo_path(absolute_path=False, root_project=False),
+                    relative_path,
                     uuid,
                     block_type,
                     language,
@@ -875,8 +1183,13 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         return f'{block_type}s' if block_type != BlockType.CUSTOM else block_type
 
     @classmethod
-    def block_type_from_path(self, block_file_absolute_path: str) -> BlockType:
-        file_path = str(block_file_absolute_path).replace(get_repo_path(), '')
+    def block_type_from_path(
+        self, block_file_absolute_path: str, repo_path: str = None
+    ) -> BlockType:
+        warn_for_repo_path(repo_path)
+        if not repo_path:
+            repo_path = get_repo_path()
+        file_path = str(block_file_absolute_path).replace(repo_path, '')
         if file_path.startswith(os.sep):
             file_path = file_path[1:]
 
@@ -981,9 +1294,13 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             )
             # For block_type SCRATCHPAD and MARKDOWN, also delete the file if possible
             if self.type in NON_PIPELINE_EXECUTABLE_BLOCK_TYPES:
-                pipelines = Pipeline.get_pipelines_by_block(self, widget=widget)
+                pipelines = Pipeline.get_pipelines_by_block(
+                    self,
+                    repo_path=self.pipeline.repo_path,
+                    widget=widget,
+                )
                 pipelines = [
-                    pipeline for pipeline in pipelines if self.pipeline.uuid != pipeline.uuid
+                    pipeline for pipeline in pipelines if self.pipeline_uuid != pipeline.uuid
                 ]
                 if len(pipelines) == 0:
                     os.remove(self.file_path)
@@ -991,7 +1308,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
         # TODO (tommy dang): delete this block from all pipelines in all projects
         # If pipeline is not specified, delete the block from all pipelines and delete the file.
-        pipelines = Pipeline.get_pipelines_by_block(self, widget=widget)
+        pipelines = Pipeline.get_pipelines_by_block(self, repo_path=self.repo_path, widget=widget)
         if not force:
             for p in pipelines:
                 if not p.block_deletable(self):
@@ -1017,7 +1334,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         logging_tags: Dict = None,
         execution_uuid: str = None,
         from_notebook: bool = False,
-        **kwargs
+        **kwargs,
     ) -> Dict:
         """
         This method will execute the block and run the callback functions if they exist
@@ -1051,9 +1368,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             # Print result to block output
             if not result:
                 conditional_message += 'This block would not be executed in a trigger run.\n'
-            conditional_json = json.dumps(dict(
-                message=conditional_message,
-            ))
+            conditional_json = json.dumps(
+                dict(
+                    message=conditional_message,
+                )
+            )
             print(f'[__internal_test__]{conditional_json}')
 
         callback_arr = []
@@ -1068,7 +1387,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 logger=logger,
                 logging_tags=logging_tags,
                 from_notebook=from_notebook,
-                **kwargs
+                **kwargs,
             )
         except Exception as error:
             for callback_block in callback_arr:
@@ -1122,145 +1441,235 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         data_integration_runtime_settings: Dict = None,
         execution_partition_previous: str = None,
         metadata: Dict = None,
+        override_outputs: bool = True,
         **kwargs,
     ) -> Dict:
-        if logging_tags is None:
-            logging_tags = dict()
+        def __execute(
+            self=self,
+            analyze_outputs=analyze_outputs,
+            block_run_outputs_cache=block_run_outputs_cache,
+            build_block_output_stdout=build_block_output_stdout,
+            custom_code=custom_code,
+            data_integration_runtime_settings=data_integration_runtime_settings,
+            disable_json_serialization=disable_json_serialization,
+            dynamic_block_index=dynamic_block_index,
+            dynamic_block_indexes=dynamic_block_indexes,
+            dynamic_block_uuid=dynamic_block_uuid,
+            dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+            execution_partition=execution_partition,
+            execution_partition_previous=execution_partition_previous,
+            from_notebook=from_notebook,
+            global_vars=global_vars,
+            input_from_output=input_from_output,
+            kwargs=kwargs,
+            logger=logger,
+            logging_tags=logging_tags,
+            metadata=metadata,
+            output_messages_to_logs=output_messages_to_logs,
+            override_outputs=override_outputs,
+            run_all_blocks=run_all_blocks,
+            run_settings=run_settings,
+            runtime_arguments=runtime_arguments,
+            store_variables=store_variables,
+            update_status=update_status,
+            verify_output=verify_output,
+        ) -> Dict:
+            if logging_tags is None:
+                logging_tags = dict()
 
-        try:
-            if not run_all_blocks:
-                not_executed_upstream_blocks = list(
-                    filter(lambda b: b.status == BlockStatus.NOT_EXECUTED, self.upstream_blocks)
-                )
-                all_upstream_is_dbt = all([BlockType.DBT == b.type
-                                           for b in not_executed_upstream_blocks])
-                if not all_upstream_is_dbt and len(not_executed_upstream_blocks) > 0:
-                    upstream_block_uuids = list(map(lambda b: b.uuid, not_executed_upstream_blocks))
-                    raise Exception(
-                        f"Block {self.uuid}'s upstream blocks have not been executed yet. "
-                        f'Please run upstream blocks {upstream_block_uuids} '
-                        'before running the current block.'
+            try:
+                if not run_all_blocks:
+                    not_executed_upstream_blocks = list(
+                        filter(
+                            lambda b: b.status == BlockStatus.NOT_EXECUTED,
+                            self.upstream_blocks,
+                        )
                     )
-            global_vars = self.enrich_global_vars(
-                global_vars,
-                dynamic_block_index=dynamic_block_index,
-            )
-
-            if output_messages_to_logs and not logger:
-                from mage_ai.data_preparation.models.block.constants import (
-                    LOG_PARTITION_EDIT_PIPELINE,
+                    all_upstream_is_dbt = all([
+                        BlockType.DBT == b.type for b in not_executed_upstream_blocks
+                    ])
+                    if not all_upstream_is_dbt and len(not_executed_upstream_blocks) > 0:
+                        upstream_block_uuids = list(
+                            map(lambda b: b.uuid, not_executed_upstream_blocks)
+                        )
+                        raise Exception(
+                            f"Block {self.uuid}'s upstream blocks have not been executed yet. "
+                            f'Please run upstream blocks {upstream_block_uuids} '
+                            'before running the current block.'
+                        )
+                global_vars = self.enrich_global_vars(
+                    global_vars,
+                    dynamic_block_index=dynamic_block_index,
                 )
 
-                logger_manager = LoggerManagerFactory.get_logger_manager(
-                    block_uuid=datetime.utcnow().strftime(format='%Y%m%dT%H%M%S'),
-                    partition=LOG_PARTITION_EDIT_PIPELINE,
-                    pipeline_uuid=self.pipeline.uuid if self.pipeline else None,
-                    subpartition=clean_name(self.uuid),
-                )
-                logger = DictLogger(logger_manager.logger)
-                logging_tags = dict(
-                    block_type=self.type,
-                    block_uuid=self.uuid,
-                    pipeline_uuid=self.pipeline.uuid if self.pipeline else None,
-                )
+                def __store_variables(
+                    variable_mapping: Dict[str, Any],
+                    skip_delete: Optional[bool] = None,
+                    save_variable_types_only: Optional[bool] = None,
+                    clean_variable_uuid: Optional[bool] = None,
+                    dynamic_block_index=dynamic_block_index,
+                    dynamic_block_uuid=dynamic_block_uuid,
+                    execution_partition=execution_partition,
+                    global_vars=global_vars,
+                    override_outputs=override_outputs,
+                    self=self,
+                ) -> Optional[List[Variable]]:
+                    return self.store_variables(
+                        variable_mapping,
+                        clean_variable_uuid=clean_variable_uuid,
+                        execution_partition=execution_partition,
+                        override_outputs=override_outputs,
+                        skip_delete=skip_delete,
+                        spark=self.__get_spark_session_from_global_vars(
+                            global_vars=global_vars,
+                        ),
+                        dynamic_block_index=dynamic_block_index,
+                        dynamic_block_uuid=dynamic_block_uuid,
+                        save_variable_types_only=save_variable_types_only,
+                    )
 
-            output = self.execute_block(
-                block_run_outputs_cache=block_run_outputs_cache,
-                build_block_output_stdout=build_block_output_stdout,
-                custom_code=custom_code,
-                execution_partition=execution_partition,
-                from_notebook=from_notebook,
-                global_vars=global_vars,
-                logger=logger,
-                logging_tags=logging_tags,
-                input_from_output=input_from_output,
-                runtime_arguments=runtime_arguments,
-                dynamic_block_index=dynamic_block_index,
-                dynamic_block_indexes=dynamic_block_indexes,
-                dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
-                run_settings=run_settings,
-                data_integration_runtime_settings=data_integration_runtime_settings,
-                execution_partition_previous=execution_partition_previous,
-                metadata=metadata,
-                **kwargs,
-            )
+                self._store_variables_in_block_function = __store_variables
 
-            if self.configuration and self.configuration.get('disable_query_preprocessing'):
-                output = dict(output=None)
-            else:
-                block_output = self.post_process_output(output)
-                variable_mapping = dict()
-
-                if BlockType.CHART == self.type:
-                    variable_mapping = block_output
-                    output = dict(
-                        output=simplejson.dumps(
-                            block_output,
-                            default=encode_complex,
-                            ignore_nan=True,
-                        ) if not disable_json_serialization else block_output,
+                if output_messages_to_logs and not logger:
+                    from mage_ai.data_preparation.models.block.constants import (
+                        LOG_PARTITION_EDIT_PIPELINE,
                     )
-                else:
-                    output_count = len(block_output)
-                    variable_keys = [f'output_{idx}' for idx in range(output_count)]
-                    variable_mapping = dict(zip(variable_keys, block_output))
 
-                if store_variables and \
-                        self.pipeline and \
-                        self.pipeline.type != PipelineType.INTEGRATION:
+                    logger_manager = LoggerManagerFactory.get_logger_manager(
+                        block_uuid=datetime.utcnow().strftime(format='%Y%m%dT%H%M%S'),
+                        partition=LOG_PARTITION_EDIT_PIPELINE,
+                        pipeline_uuid=self.pipeline_uuid,
+                        subpartition=clean_name(self.uuid),
+                    )
+                    logger = DictLogger(logger_manager.logger)
+                    logging_tags = dict(
+                        block_type=self.type,
+                        block_uuid=self.uuid,
+                        pipeline_uuid=self.pipeline_uuid,
+                    )
 
-                    try:
-                        DX_PRINTER.critical(
-                            block=self,
-                            execution_partition=execution_partition,
-                            override_outputs=True,
-                            dynamic_block_uuid=dynamic_block_uuid,
-                            __uuid='store_variables',
-                        )
+                output = self.execute_block(
+                    block_run_outputs_cache=block_run_outputs_cache,
+                    build_block_output_stdout=build_block_output_stdout,
+                    custom_code=custom_code,
+                    execution_partition=execution_partition,
+                    from_notebook=from_notebook,
+                    global_vars=global_vars,
+                    logger=logger,
+                    logging_tags=logging_tags,
+                    input_from_output=input_from_output,
+                    runtime_arguments=runtime_arguments,
+                    dynamic_block_index=dynamic_block_index,
+                    dynamic_block_indexes=dynamic_block_indexes,
+                    dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                    run_settings=run_settings,
+                    data_integration_runtime_settings=data_integration_runtime_settings,
+                    execution_partition_previous=execution_partition_previous,
+                    metadata=metadata,
+                    override_outputs=override_outputs,
+                    **kwargs,
+                )
 
-                        self.store_variables(
-                            variable_mapping,
-                            execution_partition=execution_partition,
-                            override_outputs=True,
-                            spark=self.__get_spark_session_from_global_vars(
-                                global_vars=global_vars,
+                if self.configuration and self.configuration.get('disable_query_preprocessing'):
+                    output = dict(output=None)
+                else:
+                    block_output = self.post_process_output(output)
+                    variable_mapping = dict()
+
+                    if BlockType.CHART == self.type:
+                        variable_mapping = block_output
+                        output = dict(
+                            output=(
+                                simplejson.dumps(
+                                    block_output,
+                                    default=encode_complex,
+                                    ignore_nan=True,
+                                )
+                                if not disable_json_serialization
+                                else block_output
                             ),
-                            dynamic_block_index=dynamic_block_index,
-                            dynamic_block_uuid=dynamic_block_uuid,
-                        )
-                    except ValueError as e:
-                        if str(e) == 'Circular reference detected':
-                            raise ValueError(
-                                'Please provide dataframe or json serializable data as output.'
-                            )
-                        raise e
-                # Reset outputs cache
-                self._outputs = None
-
-                if BlockType.CHART != self.type:
-                    if analyze_outputs:
-                        self.analyze_outputs(
-                            variable_mapping,
-                            execution_partition=execution_partition,
                         )
                     else:
-                        self.analyze_outputs(
-                            variable_mapping,
-                            execution_partition=execution_partition,
-                            shape_only=True,
-                        )
+                        output_count = len(block_output)
+                        variable_keys = [f'output_{idx}' for idx in range(output_count)]
+                        variable_mapping = dict(zip(variable_keys, block_output))
+
+                    if (
+                        store_variables
+                        and self.pipeline
+                        and self.pipeline.type != PipelineType.INTEGRATION
+                    ):
+                        try:
+                            DX_PRINTER.critical(
+                                block=self,
+                                execution_partition=execution_partition,
+                                override_outputs=override_outputs,
+                                dynamic_block_uuid=dynamic_block_uuid,
+                                __uuid='store_variables',
+                            )
 
-            if update_status:
-                self.status = BlockStatus.EXECUTED
-        except Exception as err:
-            if update_status:
-                self.status = BlockStatus.FAILED
-            raise err
-        finally:
-            if update_status:
-                self.__update_pipeline_block(widget=BlockType.CHART == self.type)
+                            if self._store_variables_in_block_function and isinstance(
+                                variable_mapping, dict
+                            ):
+                                self._store_variables_in_block_function(variable_mapping)
+
+                        except ValueError as e:
+                            if str(e) == 'Circular reference detected':
+                                raise ValueError(
+                                    'Please provide dataframe or json serializable data as output.'
+                                )
+                            raise e
+
+                    if not is_dynamic_block_child(self):
+                        # This will be handled in the execute_custom_code file so that it’s only
+                        # invoked once.
+                        self.aggregate_summary_info()
+
+                    # Reset outputs cache
+                    self._outputs = None
+
+                    if BlockType.CHART != self.type:
+                        if analyze_outputs:
+                            self.analyze_outputs(
+                                variable_mapping,
+                                execution_partition=execution_partition,
+                            )
+                        else:
+                            self.analyze_outputs(
+                                variable_mapping,
+                                execution_partition=execution_partition,
+                                shape_only=True,
+                            )
 
-        return output
+                if update_status:
+                    self.status = BlockStatus.EXECUTED
+            except Exception as err:
+                if update_status:
+                    self.status = BlockStatus.FAILED
+                raise err
+            finally:
+                if update_status:
+                    self.__update_pipeline_block(widget=BlockType.CHART == self.type)
+
+            return output
+
+        # if MEMORY_MANAGER_V2:
+        #     metadata = {}
+        #     if execution_partition:
+        #         metadata['execution_partition'] = execution_partition
+        #     if from_notebook:
+        #         metadata['origin'] = 'ide'
+        #     with MemoryManager(
+        #         scope_uuid=os.path.join(
+        #             *([PIPELINES_FOLDER, self.pipeline_uuid] if self.pipeline else ['']),
+        #             self.uuid,
+        #         ),
+        #         process_uuid='block.execute_sync',
+        #         repo_path=self.repo_path,
+        #         metadata=metadata,
+        #     ):
+        #         return __execute()
+        return __execute()
 
     def post_process_output(self, output: Dict) -> List:
         return output['output'] or []
@@ -1289,7 +1698,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                     global_vars=global_vars,
                     run_all_blocks=run_all_blocks,
                     update_status=update_status,
-                )
+                ),
             )
         else:
             self.execute_sync(
@@ -1334,19 +1743,19 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             if num_args > num_inputs:
                 if num_upstream < num_args:
                     raise Exception(
-                        f'Block {self.uuid} may be missing upstream dependencies. '
+                        f"Block {self.uuid} may be missing upstream dependencies. "
                         f'It expected to have {"at least " if has_var_args else ""}{num_args} '
-                        f'arguments, but only received {num_inputs}. '
-                        f'Confirm that the @{self.type} method declaration has the correct number '
-                        'of arguments.'
+                        f"arguments, but only received {num_inputs}. "
+                        f"Confirm that the @{self.type} method declaration has the correct number "
+                        "of arguments."
                     )
                 else:
                     raise Exception(
-                        f'Block {self.uuid} is missing input arguments. '
+                        f"Block {self.uuid} is missing input arguments. "
                         f'It expected to have {"at least " if has_var_args else ""}{num_args} '
-                        f'arguments, but only received {num_inputs}. '
-                        f'Double check the @{self.type} method declaration has the correct number '
-                        'of arguments and that the upstream blocks have been executed.'
+                        f"arguments, but only received {num_inputs}. "
+                        f"Double check the @{self.type} method declaration has the correct number "
+                        "of arguments and that the upstream blocks have been executed."
                     )
             elif num_args < num_inputs and not has_var_args:
                 if num_upstream > num_args:
@@ -1378,21 +1787,26 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         input_args: List = None,
         metadata: Dict = None,
     ) -> Tuple[Dict, List, Dict, List[str]]:
-        # Only fetch the input variables that the destination block explicitly declares.
-        # If all the input variables are fetched, there is a chance that a lot of data from
-        # an upstream source block is loaded just to be used as inputs for the block’s
-        # decorated functions. Only do this for the notebook because
+        """
+        Only fetch the input variables that the destination block explicitly declares.
+        If all the input variables are fetched, there is a chance that a lot of data from
+        an upstream source block is loaded just to be used as inputs for the block’s
+        decorated functions. Only do this for the notebook because
+        """
         if from_notebook and self.is_data_integration():
-            input_vars, kwargs_vars, upstream_block_uuids = \
-                self.fetch_input_variables_and_catalog(
-                    input_args,
-                    execution_partition,
-                    global_vars,
-                    dynamic_block_index=dynamic_block_index,
-                    dynamic_block_indexes=dynamic_block_indexes,
-                    dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
-                    from_notebook=from_notebook,
-                )
+            (
+                input_vars,
+                kwargs_vars,
+                upstream_block_uuids,
+            ) = self.fetch_input_variables_and_catalog(
+                input_args,
+                execution_partition,
+                global_vars,
+                dynamic_block_index=dynamic_block_index,
+                dynamic_block_indexes=dynamic_block_indexes,
+                dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                from_notebook=from_notebook,
+            )
         else:
             input_vars, kwargs_vars, upstream_block_uuids = self.fetch_input_variables(
                 input_args,
@@ -1451,18 +1865,22 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             logging_tags=logging_tags,
         ):
             # Fetch input variables
-            outputs_from_input_vars, input_vars, kwargs_vars, upstream_block_uuids = \
-                self.__get_outputs_from_input_vars(
-                    block_run_outputs_cache=block_run_outputs_cache,
-                    dynamic_block_index=dynamic_block_index,
-                    dynamic_block_indexes=dynamic_block_indexes,
-                    dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
-                    execution_partition=execution_partition,
-                    from_notebook=from_notebook,
-                    global_vars=global_vars,
-                    input_args=input_args,
-                    metadata=metadata,
-                )
+            (
+                outputs_from_input_vars,
+                input_vars,
+                kwargs_vars,
+                upstream_block_uuids,
+            ) = self.__get_outputs_from_input_vars(
+                block_run_outputs_cache=block_run_outputs_cache,
+                dynamic_block_index=dynamic_block_index,
+                dynamic_block_indexes=dynamic_block_indexes,
+                dynamic_upstream_block_uuids=dynamic_upstream_block_uuids,
+                execution_partition=execution_partition,
+                from_notebook=from_notebook,
+                global_vars=global_vars,
+                input_args=input_args,
+                metadata=metadata,
+            )
 
             global_vars_copy = global_vars.copy()
             for kwargs_var in kwargs_vars:
@@ -1588,11 +2006,14 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         preprocesser_functions = []
         test_functions = []
 
-        results = merge_dict({
-            'preprocesser': self._block_decorator(preprocesser_functions),
-            'test': self._block_decorator(test_functions),
-            self.type: self._block_decorator(decorated_functions),
-        }, outputs_from_input_vars)
+        results = merge_dict(
+            {
+                'preprocesser': self._block_decorator(preprocesser_functions),
+                'test': self._block_decorator(test_functions),
+                self.type: self._block_decorator(decorated_functions),
+            },
+            outputs_from_input_vars,
+        )
 
         if custom_code is not None and custom_code.strip():
             if BlockType.CHART != self.type:
@@ -1610,13 +2031,17 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 self.execute_block_function(
                     preprocesser_function,
                     input_vars,
+                    dynamic_block_index=dynamic_block_index,
+                    execution_partition=execution_partition,
                     from_notebook=from_notebook,
                     global_vars=global_vars,
+                    logger=logger,
+                    logging_tags=logging_tags,
                 )
 
         block_function = self._validate_execution(decorated_functions, input_vars)
         if block_function is not None:
-            if logger and 'logger' not in global_vars:
+            if logger:
                 global_vars['logger'] = logger
 
             track_spark = from_notebook and self.should_track_spark()
@@ -1629,8 +2054,12 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             outputs = self.execute_block_function(
                 block_function,
                 input_vars,
+                dynamic_block_index=dynamic_block_index,
+                execution_partition=execution_partition,
                 from_notebook=from_notebook,
                 global_vars=global_vars,
+                logger=logger,
+                logging_tags=logging_tags,
             )
 
             if track_spark:
@@ -1640,7 +2069,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
         if outputs is None:
             outputs = []
-        if type(outputs) is not list:
+
+        if isinstance(outputs, tuple):
+            outputs = list(outputs)
+
+        if not isinstance(outputs, list):
             outputs = [outputs]
 
         return outputs
@@ -1649,26 +2082,148 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         self,
         block_function: Callable,
         input_vars: List,
+        dynamic_block_index: Optional[int] = None,
+        dynamic_block_uuid: Optional[str] = None,
+        execution_partition: Optional[str] = None,
         from_notebook: bool = False,
-        global_vars: Dict = None,
+        global_vars: Optional[Dict] = None,
         initialize_decorator_modules: bool = True,
-    ) -> Dict:
-        block_function_updated = block_function
+        logger: Optional[Logger] = None,
+        logging_tags: Optional[Dict] = None,
+    ) -> List[Dict[str, Any]]:
+        from mage_ai.settings.server import (
+            MEMORY_MANAGER_V2,  # Need here to mock in tests
+        )
+
+        sig = signature(block_function)
+        has_kwargs = any([p.kind == p.VAR_KEYWORD for p in sig.parameters.values()])
 
+        block_function_updated = block_function
         if from_notebook and initialize_decorator_modules:
             block_function_updated = self.__initialize_decorator_modules(
                 block_function,
-                [self.type],
+                [str(self.type.value) if not isinstance(self.type, str) else str(self.type)]
+                if self.type
+                else [],
             )
 
-        sig = signature(block_function)
-        has_kwargs = any([p.kind == p.VAR_KEYWORD for p in sig.parameters.values()])
-
-        if has_kwargs and global_vars is not None and len(global_vars) != 0:
+        write_policy = (
+            self.write_settings.batch_settings.mode
+            if self.write_settings and self.write_settings.batch_settings
+            else None
+        )
+        append_data = ExportWritePolicy.APPEND == write_policy
+        part_index = None
+        if append_data:
+            block_uuid, changed = uuid_for_output_variables(
+                self,
+                block_uuid=self.uuid,
+                dynamic_block_index=dynamic_block_index,
+            )
+            variable_object = self.get_variable_object(
+                block_uuid=block_uuid,
+                partition=execution_partition,
+            )
+            part_uuids = variable_object.part_uuids
+            if part_uuids is not None:
+                part_index = len(part_uuids)
+                if global_vars:
+                    global_vars.update(part_index=part_index)
+
+        if MEMORY_MANAGER_V2:
+            log_message_prefix = self.uuid
+            if self.pipeline:
+                log_message_prefix = f'{self.pipeline_uuid}:{log_message_prefix}'
+            log_message_prefix = f'[{log_message_prefix}:execute_block_function]'
+
+            output, self.resource_usage = execute_with_memory_tracking(
+                block_function_updated,
+                args=input_vars,
+                kwargs=global_vars
+                if has_kwargs and global_vars is not None and len(global_vars) != 0
+                else None,
+                logger=logger,
+                logging_tags=logging_tags,
+                log_message_prefix=log_message_prefix,
+            )
+        elif has_kwargs and global_vars is not None and len(global_vars) != 0:
             output = block_function_updated(*input_vars, **global_vars)
         else:
             output = block_function_updated(*input_vars)
 
+        if MEMORY_MANAGER_V2 and inspect.isgeneratorfunction(block_function_updated):
+            variable_types = []
+            dynamic_child = is_dynamic_block_child(self)
+            output_count = part_index if part_index is not None else 0
+            if output is not None and is_iterable(output):
+                if dynamic_child or self.is_dynamic_child:
+                    # Each child will delete its own data
+                    # How do we delete everything ahead of time?
+                    delete_variable_objects_for_dynamic_child(
+                        self,
+                        dynamic_block_index=dynamic_block_index,
+                        execution_partition=execution_partition,
+                    )
+                else:
+                    self.delete_variables(
+                        dynamic_block_index=dynamic_block_index,
+                        dynamic_block_uuid=dynamic_block_uuid,
+                        execution_partition=execution_partition,
+                    )
+
+                for data in output:
+                    if self._store_variables_in_block_function is None:
+                        raise Exception(
+                            'Store variables function isn’t defined, '
+                            'don’t proceed or else no data will be persisted'
+                        )
+
+                    store_options = {}
+                    if output_count >= 1:
+                        store_options['override_outputs'] = False
+
+                    variable_mapping = {}
+
+                    def __output_key(order: int, output_count=output_count):
+                        return os.path.join(f'output_{order}', str(output_count))
+
+                    if is_basic_iterable(data):
+                        if data is None or len(data) == 1:
+                            variable_mapping[__output_key(0)] = data
+                        elif len(data) == 2 and isinstance(data[1], dict):
+                            variable_mapping[__output_key(0)] = data[0]
+                            variable_mapping[__output_key(1)] = data[1]
+                        else:
+                            for idx, item in enumerate(data):
+                                variable_mapping[__output_key(idx)] = item
+                    else:
+                        variable_mapping[__output_key(0)] = data
+
+                    variables = self._store_variables_in_block_function(
+                        variable_mapping=variable_mapping,
+                        clean_variable_uuid=False,
+                        skip_delete=True,
+                        **store_options,
+                    )
+                    if variables is not None and isinstance(variables, list):
+                        variable_types += [
+                            variable.variable_type
+                            for variable in variables
+                            if isinstance(variable, Variable)
+                        ]
+
+                    output_count += 1
+
+                if len(variable_types) >= 1 and self._store_variables_in_block_function:
+                    self._store_variables_in_block_function(
+                        {'output_0': variable_types},
+                        save_variable_types_only=True,
+                    )
+
+                self._store_variables_in_block_function = None
+
+        if output is None:
+            return []
         return output
 
     def __initialize_decorator_modules(
@@ -1747,11 +2302,18 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 upstream block UUIDs.
         """
 
-        if any([is_dynamic_block(
-            upstream_block,
-        ) or is_dynamic_block_child(
-            upstream_block,
-        ) for upstream_block in self.upstream_blocks]):
+        if (self.is_dynamic_v2 and self.is_dynamic_child) or (
+            not self.is_dynamic_v2
+            and any([
+                is_dynamic_block(
+                    upstream_block,
+                )
+                or is_dynamic_block_child(
+                    upstream_block,
+                )
+                for upstream_block in self.upstream_blocks
+            ])
+        ):
             return fetch_input_variables_for_dynamic_upstream_blocks(
                 self,
                 input_args,
@@ -1760,7 +2322,6 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 execution_partition=execution_partition,
                 from_notebook=from_notebook,
                 global_vars=global_vars,
-                # For non-dynamic upstream blocks
                 block_run_outputs_cache=block_run_outputs_cache,
                 data_integration_settings_mapping=data_integration_settings_mapping,
                 upstream_block_uuids_override=upstream_block_uuids_override,
@@ -1780,6 +2341,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             global_vars=global_vars,
             metadata=metadata,
             upstream_block_uuids_override=upstream_block_uuids_override,
+            current_block=self,
         )
 
         return variables
@@ -1794,8 +2356,8 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         for v in output_variable_objects:
             if v.variable_type != VariableType.DATAFRAME:
                 continue
-            data = self.pipeline.variable_manager.get_variable(
-                self.pipeline.uuid,
+            data = self.variable_manager.get_variable(
+                self.pipeline_uuid,
                 self.uuid,
                 v.uuid,
                 variable_type=VariableType.DATAFRAME_ANALYSIS,
@@ -1807,12 +2369,12 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
     def get_variables_by_block(
         self,
         block_uuid: str,
-        dynamic_block_index: int = None,
-        dynamic_block_uuid: str = None,
-        partition: str = None,
-    ):
-        variable_manager = self.pipeline.variable_manager
-
+        clean_block_uuid: bool = True,
+        dynamic_block_index: Optional[int] = None,
+        dynamic_block_uuid: Optional[str] = None,
+        max_results: Optional[int] = None,
+        partition: Optional[str] = None,
+    ) -> List[str]:
         block_uuid_use, changed = uuid_for_output_variables(
             self,
             block_uuid=block_uuid,
@@ -1820,10 +2382,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             dynamic_block_uuid=dynamic_block_uuid,
         )
 
-        res = variable_manager.get_variables_by_block(
-            self.pipeline.uuid,
+        res = self.variable_manager.get_variables_by_block(
+            self.pipeline_uuid,
             block_uuid=block_uuid_use,
-            clean_block_uuid=not changed,
+            clean_block_uuid=not changed and clean_block_uuid,
+            max_results=max_results,
             partition=partition,
         )
 
@@ -1847,9 +2410,12 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         partition: str = None,
         raise_exception: bool = False,
         spark=None,
+        input_data_types: Optional[List[InputDataType]] = None,
+        read_batch_settings: Optional[BatchSettings] = None,
+        read_chunks: Optional[List[ChunkKeyTypeUnion]] = None,
+        write_batch_settings: Optional[BatchSettings] = None,
+        write_chunks: Optional[List[ChunkKeyTypeUnion]] = None,
     ):
-        variable_manager = self.pipeline.variable_manager
-
         block_uuid_use, changed = uuid_for_output_variables(
             self,
             block_uuid=block_uuid,
@@ -1857,40 +2423,151 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
             dynamic_block_uuid=dynamic_block_uuid,
         )
 
-        value = variable_manager.get_variable(
-            self.pipeline.uuid,
+        value = self.variable_manager.get_variable(
+            self.pipeline_uuid,
             block_uuid=block_uuid_use,
             clean_block_uuid=not changed,
             partition=partition,
             raise_exception=raise_exception,
             spark=spark,
             variable_uuid=variable_uuid,
+            input_data_types=input_data_types,
+            read_batch_settings=read_batch_settings,
+            read_chunks=read_chunks,
+            write_batch_settings=write_batch_settings,
+            write_chunks=write_chunks,
         )
 
         return value
 
-    def get_variable_object(
+    def read_partial_data(
         self,
-        block_uuid: str,
-        variable_uuid: str = None,
-        dynamic_block_index: int = None,
-        partition: str = None,
+        variable_uuid: str,
+        batch_settings: Optional[BatchSettings] = None,
+        chunks: Optional[List[ChunkKeyTypeUnion]] = None,
+        input_data_types: Optional[List[InputDataType]] = None,
+        part_uuid: Optional[Union[int, str]] = None,
+        partition: Optional[str] = None,
     ):
-        variable_manager = self.pipeline.variable_manager
+        return self.get_variable_object(
+            self.uuid,
+            variable_uuid,
+            partition=partition,
+        ).read_partial_data(
+            batch_settings=batch_settings,
+            chunks=chunks,
+            input_data_types=input_data_types,
+            part_uuid=part_uuid,
+        )
 
+    def get_variable_object(
+        self,
+        block_uuid: Optional[str] = None,
+        variable_uuid: Optional[str] = None,
+        clean_block_uuid: bool = True,
+        dynamic_block_index: Optional[int] = None,
+        input_data_types: Optional[List[InputDataType]] = None,
+        ordinal_position: Optional[int] = None,  # Used to get cached variable information
+        partition: Optional[str] = None,
+        read_batch_settings: Optional[BatchSettings] = None,
+        read_chunks: Optional[List[ChunkKeyTypeUnion]] = None,
+        skip_check_variable_type: Optional[bool] = None,
+        write_batch_settings: Optional[BatchSettings] = None,
+        write_chunks: Optional[List[ChunkKeyTypeUnion]] = None,
+    ) -> Variable:
+        block_uuid = block_uuid or self.uuid
         block_uuid, changed = uuid_for_output_variables(
             self,
             block_uuid=block_uuid,
             dynamic_block_index=dynamic_block_index,
         )
 
-        return variable_manager.get_variable_object(
-            self.pipeline.uuid,
+        variable_type_information = None
+        variable_types_information = None
+        skip_check_variable_type = skip_check_variable_type or False
+        if VARIABLE_DATA_OUTPUT_META_CACHE:
+            dynamic_child = is_dynamic_block_child(self)
+            group_type = (
+                VariableAggregateSummaryGroupType.DYNAMIC
+                if dynamic_child
+                else VariableAggregateSummaryGroupType.PARTS
+            )
+            variable_type_information = self.get_variable_aggregate_cache(
+                variable_uuid, VariableAggregateDataType.TYPE, default_group_type=group_type
+            )
+            variable_types_information = self.get_variable_aggregate_cache(
+                variable_uuid,
+                VariableAggregateDataType.TYPE,
+                group_type=group_type,
+                partition=partition,
+            )
+
+            if (
+                variable_type_information
+                and variable_type_information.type == VariableType.ITERABLE
+                and not variable_types_information
+            ):
+                # If the dynamic parent block is an interable with no types information,
+                # then the data from the parent block won’t have any type information.
+                # Skip variable type check when instantiating a variable object for the children.
+                skip_check_variable_type = True
+            elif (
+                dynamic_child
+                and variable_types_information is not None
+                and isinstance(variable_types_information, Iterable)
+                and (
+                    (
+                        dynamic_block_index is not None
+                        and int(dynamic_block_index) < len(variable_types_information)
+                    )
+                    or (
+                        ordinal_position is not None
+                        and int(ordinal_position) < len(variable_types_information)
+                    )
+                )
+            ):
+                position = (
+                    int(ordinal_position)
+                    if ordinal_position is not None
+                    else int(dynamic_block_index)
+                    if dynamic_block_index is not None
+                    else None
+                )
+                if position is not None and isinstance(variable_types_information, Iterable):
+                    variable_type_information = variable_types_information[position]
+                    if (
+                        isinstance(variable_type_information, Iterable)
+                        and len(variable_type_information) >= 1
+                    ):
+                        variable_type_information = variable_type_information[0]
+
+                    variable_types_information = None
+
+        variable_types = []
+        if isinstance(variable_types_information, Iterable):
+            for v in variable_types_information:
+                if isinstance(v, list):
+                    variable_types += [vv.type for vv in v]
+                else:
+                    variable_types.append(v.type)
+
+        return self.variable_manager.get_variable_object(
+            self.pipeline_uuid,
             block_uuid=block_uuid,
-            clean_block_uuid=not changed,
+            clean_block_uuid=not changed and clean_block_uuid,
             partition=partition,
             spark=self.get_spark_session(),
+            skip_check_variable_type=skip_check_variable_type,
+            variable_type=variable_type_information.type
+            if variable_type_information is not None
+            else None,
+            variable_types=variable_types,
             variable_uuid=variable_uuid,
+            input_data_types=input_data_types,
+            read_batch_settings=read_batch_settings,
+            read_chunks=read_chunks,
+            write_batch_settings=write_batch_settings,
+            write_chunks=write_chunks,
         )
 
     def get_raw_outputs(
@@ -1901,6 +2578,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         global_vars: Dict = None,
         dynamic_block_index: int = None,
         dynamic_block_uuid: str = None,
+        input_data_types: Optional[List[InputDataType]] = None,
+        read_batch_settings: Optional[BatchSettings] = None,
+        read_chunks: Optional[List[ChunkKeyTypeUnion]] = None,
+        write_batch_settings: Optional[BatchSettings] = None,
+        write_chunks: Optional[List[ChunkKeyTypeUnion]] = None,
     ) -> List[Any]:
         all_variables = self.get_variables_by_block(
             block_uuid=block_uuid,
@@ -1912,6 +2594,9 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         outputs = []
 
         for variable_uuid in all_variables:
+            if not is_output_variable(variable_uuid):
+                continue
+
             variable = self.pipeline.get_block_variable(
                 block_uuid,
                 variable_uuid,
@@ -1922,6 +2607,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 spark=self.__get_spark_session_from_global_vars(global_vars),
                 dynamic_block_index=dynamic_block_index,
                 dynamic_block_uuid=dynamic_block_uuid,
+                input_data_types=input_data_types,
+                read_batch_settings=read_batch_settings,
+                read_chunks=read_chunks,
+                write_batch_settings=write_batch_settings,
+                write_chunks=write_chunks,
             )
             outputs.append(variable)
 
@@ -1929,411 +2619,181 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
 
     def get_outputs(
         self,
-        execution_partition: str = None,
-        include_print_outputs: bool = True,
+        block_uuid: Optional[str] = None,
         csv_lines_only: bool = False,
+        dynamic_block_index: Optional[int] = None,
+        exclude_blank_variable_uuids: bool = False,
+        execution_partition: Optional[str] = None,
+        include_print_outputs: bool = True,
+        metadata: Optional[Dict] = None,
         sample: bool = True,
-        sample_count: int = DATAFRAME_SAMPLE_COUNT_PREVIEW,
-        variable_type: VariableType = None,
-        block_uuid: str = None,
-        selected_variables: List[str] = None,
-        metadata: Dict = None,
-        dynamic_block_index: int = None,
-    ) -> List[Dict]:
-        data_products = []
-        outputs = []
-
+        sample_count: Optional[int] = None,
+        selected_variables: Optional[List[str]] = None,
+        variable_type: Optional[VariableType] = None,
+    ) -> List[Dict[str, Any]]:
         is_dynamic_child = is_dynamic_block_child(self)
         is_dynamic = is_dynamic_block(self)
 
-        if is_dynamic_child or is_dynamic:
-            pairs = []
-
-            if is_dynamic_child:
-                lazy_variable_controller = get_outputs_for_dynamic_child(
-                    self,
-                    execution_partition=execution_partition,
-                    sample=sample,
-                    sample_count=sample_count,
-                )
-                pairs = lazy_variable_controller.render(
-                    dynamic_block_index=dynamic_block_index,
-                )
-            elif is_dynamic:
-                tup = get_outputs_for_dynamic_block(
-                    self,
-                    execution_partition=execution_partition,
-                    sample=sample,
-                    sample_count=sample_count,
-                )
-                pairs.append(tup)
-
-            for pair in pairs:
-                child_data = None
-                metadata = None
-                if len(pair) >= 1:
-                    child_data = pair[0]
-                    if len(pair) >= 2:
-                        metadata = pair[1]
-
-                for output, variable_uuid in [
-                    (child_data, 'child_data'),
-                    (metadata, 'metadata'),
-                ]:
-                    if output is None:
-                        continue
-
-                    data, is_data_product = self.__format_output_data(
-                        output,
-                        variable_uuid,
-                        block_uuid=self.uuid,
-                        csv_lines_only=csv_lines_only,
-                        execution_partition=execution_partition,
-                    )
-
-                    outputs_below_limit = not sample or not sample_count
-                    if is_data_product:
-                        outputs_below_limit = outputs_below_limit or \
-                            (sample_count is not None and len(data_products) < sample_count)
-                    else:
-                        outputs_below_limit = outputs_below_limit or \
-                            (sample_count is not None and len(outputs) < sample_count)
-
-                    if outputs_below_limit:
-                        if is_data_product:
-                            data_products.append(data)
-                        else:
-                            outputs.append(data)
-        else:
-            if self.pipeline is None:
-                return
-
-            if not block_uuid:
-                block_uuid = self.uuid
-
-            # The block_run’s block_uuid for replicated blocks will be in this format:
-            # [block_uuid]:[replicated_block_uuid]
-            # We need to use the original block_uuid to get the proper output.
-
-            # Block runs for dynamic child blocks will have the following block UUID:
-            # [block.uuid]:[index]
-            # Don’t use the original UUID even if the block is a replica because it will get rid of
-            # the dynamic child block index.
-
-            data_products = []
-            outputs = []
-
-            all_variables = self.get_variables_by_block(
+        if not is_dynamic and not is_dynamic_child:
+            return get_outputs_for_display_sync(
+                self,
                 block_uuid=block_uuid,
-                partition=execution_partition,
+                csv_lines_only=csv_lines_only,
+                exclude_blank_variable_uuids=exclude_blank_variable_uuids,
+                execution_partition=execution_partition,
+                include_print_outputs=include_print_outputs,
+                sample=sample,
+                sample_count=sample_count or DATAFRAME_SAMPLE_COUNT_PREVIEW,
+                selected_variables=selected_variables,
+                variable_type=variable_type,
             )
 
-            if not include_print_outputs:
-                all_variables = self.output_variables(
-                    execution_partition=execution_partition,
-                )
-
-            for v in all_variables:
-                if selected_variables and v not in selected_variables:
-                    continue
-
-                variable_object = self.get_variable_object(
-                    block_uuid=block_uuid,
-                    partition=execution_partition,
-                    variable_uuid=v,
-                )
+        sample_count_use = sample_count or DYNAMIC_CHILD_BLOCK_SAMPLE_COUNT_PREVIEW
+        output_sets = []
+        variable_sets = []
 
-                if variable_type is not None and variable_object.variable_type != variable_type:
-                    continue
-
-                data = variable_object.read_data(
-                    sample=sample,
-                    sample_count=sample_count,
-                    spark=self.get_spark_session(),
-                )
-                data, is_data_product = self.__format_output_data(
-                    data,
-                    v,
-                    block_uuid=block_uuid,
-                    csv_lines_only=csv_lines_only,
-                    execution_partition=execution_partition,
-                )
-                if is_data_product:
-                    data_products.append(data)
-                else:
-                    outputs.append(data)
-
-        return outputs + data_products
-
-    async def __get_outputs_async(
-        self,
-        csv_lines_only: bool = False,
-        execution_partition: str = None,
-        include_print_outputs: bool = True,
-        sample: bool = True,
-        sample_count: int = DATAFRAME_SAMPLE_COUNT_PREVIEW,
-        variable_type: VariableType = None,
-        block_uuid: str = None,
-        dynamic_block_index: int = None,
-    ) -> List[Dict]:
-        data_products = []
-        outputs = []
-
-        is_dynamic_child = is_dynamic_block_child(self)
-        is_dynamic = is_dynamic_block(self)
-
-        if is_dynamic_child or is_dynamic:
-            pairs = []
-
-            if is_dynamic_child:
-                lazy_variable_controller = get_outputs_for_dynamic_child(
-                    self,
-                    execution_partition=execution_partition,
-                    sample=sample,
-                    sample_count=sample_count,
-                )
-                pairs = await lazy_variable_controller.render_async(
-                    dynamic_block_index=dynamic_block_index,
-                )
-            elif is_dynamic:
-                tup = await get_outputs_for_dynamic_block_async(
-                    self,
-                    execution_partition=execution_partition,
-                    sample=sample,
-                    sample_count=sample_count,
-                )
-                pairs.append(tup)
-
-            if len(pairs) > 10:
-                # Limit the number of dynamic block children we display output for in the UI
-                pairs = pairs[:DATAFRAME_SAMPLE_COUNT_PREVIEW]
-            for pair in pairs:
-                child_data = None
-                metadata = None
-                if len(pair) >= 1:
-                    child_data = pair[0]
-                    if len(pair) >= 2:
-                        metadata = pair[1]
-
-                for output, variable_uuid in [
-                    (child_data, 'child_data'),
-                    (metadata, 'metadata'),
-                ]:
-                    if output is None:
-                        continue
-
-                    data, is_data_product = self.__format_output_data(
-                        output,
-                        variable_uuid,
-                        block_uuid=self.uuid,
-                        csv_lines_only=csv_lines_only,
-                        execution_partition=execution_partition,
-                    )
-
-                    if is_data_product:
-                        data_products.append(data)
-                    else:
-                        outputs.append(data)
-        else:
-            if self.pipeline is None:
-                return
-
-            if not block_uuid:
-                block_uuid = self.uuid
-
-            variable_manager = self.pipeline.variable_manager
-
-            all_variables = variable_manager.get_variables_by_block(
-                self.pipeline.uuid,
-                block_uuid,
-                partition=execution_partition,
-                max_results=DATAFRAME_SAMPLE_COUNT_PREVIEW,
+        if is_dynamic_child:
+            lazy_variable_controller = get_outputs_for_dynamic_child(
+                self,
+                execution_partition=execution_partition,
+                sample=sample,
+                sample_count=sample_count_use,
             )
+            variable_sets: List[
+                Union[
+                    Tuple[Optional[Any], Dict],
+                    List[LazyVariableSet],
+                ],
+            ] = lazy_variable_controller.render(
+                dynamic_block_index=dynamic_block_index,
+                lazy_load=True,
+            )
+        elif is_dynamic:
+            output_pair: List[
+                Optional[
+                    Union[
+                        Any,
+                        Dict,
+                        int,
+                        pd.DataFrame,
+                        str,
+                    ]
+                ]
+            ] = get_outputs_for_dynamic_block(
+                self,
+                execution_partition=execution_partition,
+                sample=sample,
+                sample_count=sample_count_use,
+            )
+            output_sets.append(output_pair)
 
-            if not include_print_outputs:
-                all_variables = self.output_variables(execution_partition=execution_partition)
-
-            for v in all_variables:
-                variable_object = variable_manager.get_variable_object(
-                    self.pipeline.uuid,
-                    block_uuid,
-                    v,
-                    partition=execution_partition,
-                    spark=self.get_spark_session(),
-                )
-
-                if variable_type is not None and variable_object.variable_type != variable_type:
-                    continue
-
-                data = await variable_object.read_data_async(
-                    sample=True,
-                    sample_count=sample_count,
-                    spark=self.get_spark_session(),
-                )
-                data, is_data_product = self.__format_output_data(
-                    data,
-                    v,
-                    block_uuid=block_uuid,
-                    csv_lines_only=csv_lines_only,
-                    execution_partition=execution_partition,
-                )
-                if is_data_product:
-                    data_products.append(data)
-                else:
-                    outputs.append(data)
-
-        return outputs + data_products
+        output_sets = output_sets[:DATAFRAME_SAMPLE_COUNT_PREVIEW]
+        variable_sets = variable_sets[:DATAFRAME_SAMPLE_COUNT_PREVIEW]
+        child_data_sets = [lazy_variable_set.read_data() for lazy_variable_set in variable_sets]
 
-    def __format_output_data(
+        return get_outputs_for_display_dynamic_block(
+            self,
+            output_sets,
+            child_data_sets,
+            block_uuid=block_uuid,
+            csv_lines_only=csv_lines_only,
+            dynamic_block_index=dynamic_block_index,
+            exclude_blank_variable_uuids=exclude_blank_variable_uuids,
+            execution_partition=execution_partition,
+            metadata=metadata,
+            sample=sample,
+            sample_count=sample_count_use,
+        )
+
+    async def __get_outputs_async(
         self,
-        data: Any,
-        variable_uuid: str,
-        block_uuid: str = None,
+        execution_partition: Optional[str] = None,
+        include_print_outputs: bool = True,
         csv_lines_only: bool = False,
-        execution_partition: str = None,
-        skip_dynamic_block: bool = False,
-    ) -> Tuple[Dict, bool]:
-        """
-        Takes variable data and formats it to return to the frontend.
-
-        Returns:
-            Tuple[Dict, bool]: Tuple of the formatted data and is_data_product boolean. Data product
-                outputs and non data product outputs are handled slightly differently.
-        """
-        variable_manager = self.pipeline.variable_manager
-
+        sample: bool = True,
+        sample_count: Optional[int] = None,
+        variable_type: Optional[VariableType] = None,
+        block_uuid: Optional[str] = None,
+        selected_variables: Optional[List[str]] = None,
+        metadata: Optional[Dict] = None,
+        dynamic_block_index: Optional[int] = None,
+        exclude_blank_variable_uuids: bool = False,
+        max_results: Optional[int] = None,
+    ) -> List[Dict[str, Any]]:
         is_dynamic_child = is_dynamic_block_child(self)
         is_dynamic = is_dynamic_block(self)
 
-        if (is_dynamic_child or is_dynamic) and not skip_dynamic_block:
-            from mage_ai.data_preparation.models.block.dynamic.utils import (
-                coerce_into_dataframe,
+        if not is_dynamic and not is_dynamic_child:
+            return await get_outputs_for_display_async(
+                self,
+                block_uuid=block_uuid,
+                csv_lines_only=csv_lines_only,
+                exclude_blank_variable_uuids=exclude_blank_variable_uuids,
+                execution_partition=execution_partition,
+                include_print_outputs=include_print_outputs,
+                sample=sample,
+                sample_count=sample_count or DATAFRAME_SAMPLE_COUNT_PREVIEW,
+                selected_variables=selected_variables,
+                variable_type=variable_type,
+                max_results=max_results,
             )
 
-            data, is_data_product = self.__format_output_data(
-                coerce_into_dataframe(data),
-                variable_uuid=variable_uuid,
-                skip_dynamic_block=True,
+        sample_count_use = sample_count or DYNAMIC_CHILD_BLOCK_SAMPLE_COUNT_PREVIEW
+        output_sets = []
+        variable_sets = []
+
+        if is_dynamic_child:
+            lazy_variable_controller = get_outputs_for_dynamic_child(
+                self,
+                execution_partition=execution_partition,
+                limit_parts=max_results,
+                sample=sample,
+                sample_count=sample_count_use,
+            )
+            variable_sets: List[
+                Union[
+                    Tuple[Optional[Any], Dict],
+                    List[LazyVariableSet],
+                ],
+            ] = await lazy_variable_controller.render_async(
+                dynamic_block_index=dynamic_block_index,
+                lazy_load=True,
             )
 
-            return merge_dict(data, dict(multi_output=True)), is_data_product
-        elif isinstance(data, pd.DataFrame):
-            if csv_lines_only:
-                data = dict(
-                    table=data.to_csv(header=True, index=False).strip('\n').split('\n')
-                )
-            else:
-                try:
-                    analysis = variable_manager.get_variable(
-                        self.pipeline.uuid,
-                        block_uuid,
-                        variable_uuid,
-                        dataframe_analysis_keys=['metadata', 'statistics'],
-                        partition=execution_partition,
-                        variable_type=VariableType.DATAFRAME_ANALYSIS,
-                    )
-                except Exception:
-                    analysis = None
-                if analysis is not None and \
-                        (analysis.get('statistics') or analysis.get('metadata')):
-
-                    stats = analysis.get('statistics', {})
-                    column_types = (analysis.get('metadata') or {}).get('column_types', {})
-                    row_count = stats.get('original_row_count', stats.get('count'))
-                    column_count = stats.get('original_column_count', len(column_types))
-                else:
-                    row_count, column_count = data.shape
-
-                columns_to_display = data.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-                data = dict(
-                    sample_data=dict(
-                        columns=columns_to_display,
-                        rows=json.loads(
-                            data[columns_to_display].to_json(orient='split', date_format='iso'),
-                        )['data']
-                    ),
-                    shape=[row_count, column_count],
-                    type=DataType.TABLE,
-                    variable_uuid=variable_uuid,
-                )
-            return data, True
-        elif isinstance(data, pl.DataFrame):
-            try:
-                analysis = variable_manager.get_variable(
-                    self.pipeline.uuid,
-                    block_uuid,
-                    variable_uuid,
-                    dataframe_analysis_keys=['statistics'],
-                    partition=execution_partition,
-                    variable_type=VariableType.DATAFRAME_ANALYSIS,
-                )
-            except Exception:
-                analysis = None
-            if analysis is not None:
-                stats = analysis.get('statistics', {})
-                row_count = stats.get('original_row_count')
-                column_count = stats.get('original_column_count')
-            else:
-                row_count, column_count = data.shape
-            columns_to_display = data.columns[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-            data = dict(
-                sample_data=dict(
-                    columns=columns_to_display,
-                    rows=[
-                        list(row.values()) for row in json.loads(
-                            data[columns_to_display].write_json(row_oriented=True)
-                        )
-                    ]
-                ),
-                shape=[row_count, column_count],
-                type=DataType.TABLE,
-                variable_uuid=variable_uuid,
-            )
-            return data, True
-        elif is_geo_dataframe(data):
-            data = dict(
-                text_data=f'''Use the code in a scratchpad to get the output of the block:
-
-from mage_ai.data_preparation.variable_manager import get_variable
-df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
-''',
-                type=DataType.TEXT,
-                variable_uuid=variable_uuid,
-            )
-            return data, False
-        elif type(data) is str:
-            data = dict(
-                text_data=data,
-                type=DataType.TEXT,
-                variable_uuid=variable_uuid,
-            )
-            return data, False
-        elif type(data) is dict or type(data) is list:
-            data = dict(
-                text_data=simplejson.dumps(
-                    data,
-                    default=encode_complex,
-                    ignore_nan=True,
-                ),
-                type=DataType.TEXT,
-                variable_uuid=variable_uuid,
-            )
-            return data, False
-        elif is_spark_dataframe(data):
-            df = data.toPandas()
-            columns_to_display = df.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-            data = dict(
-                sample_data=dict(
-                    columns=columns_to_display,
-                    rows=json.loads(
-                        df[columns_to_display].to_json(orient='split', date_format='iso'),
-                    )['data']
-                ),
-                type=DataType.TABLE,
-                variable_uuid=variable_uuid,
+        elif is_dynamic:
+            output_pair: List[
+                Optional[Union[Dict, int, str, pd.DataFrame, Any]],
+            ] = await get_outputs_for_dynamic_block_async(
+                self,
+                execution_partition=execution_partition,
+                limit_parts=max_results,
+                sample=sample,
+                sample_count=sample_count_use,
             )
-            return data, True
-        return data, False
+            output_sets.append(output_pair)
+
+        # Limit the number of dynamic block children we display output for in the UI
+        output_sets = output_sets[:DATAFRAME_SAMPLE_COUNT_PREVIEW]
+        variable_sets = variable_sets[:DATAFRAME_SAMPLE_COUNT_PREVIEW]
+        child_data_sets = await asyncio.gather(*[
+            lazy_variable_set.read_data_async() for lazy_variable_set in variable_sets
+        ])
+
+        return get_outputs_for_display_dynamic_block(
+            self,
+            output_sets,
+            child_data_sets,
+            block_uuid=block_uuid,
+            csv_lines_only=csv_lines_only,
+            exclude_blank_variable_uuids=exclude_blank_variable_uuids,
+            execution_partition=execution_partition,
+            metadata=metadata,
+            sample=sample,
+            sample_count=sample_count_use,
+        )
+
+    def __format_output_data(self, *args, **kwargs) -> Tuple[Dict, bool]:
+        return format_output_data(self, *args, **kwargs)
 
     def __save_outputs_prepare(self, outputs, override_output_variable: bool = False) -> Dict:
         variable_mapping = dict()
@@ -2344,10 +2804,11 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             if not isinstance(o, dict):
                 continue
 
-            if all(k in o for k in ['variable_uuid', 'text_data']) and \
-                    (not is_output_variable(o['variable_uuid']) or
-                        BlockType.SCRATCHPAD == self.type or
-                        override_output_variable):
+            if all(k in o for k in ['variable_uuid', 'text_data']) and (
+                not is_output_variable(o['variable_uuid'])
+                or BlockType.SCRATCHPAD == self.type
+                or override_output_variable
+            ):
                 variable_mapping[o['variable_uuid']] = o['text_data']
 
         self._outputs = outputs
@@ -2375,9 +2836,26 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         self,
         outputs,
         override: bool = False,
+        override_conditionally: bool = False,
         override_outputs: bool = False,
     ) -> None:
         variable_mapping = self.__save_outputs_prepare(outputs)
+
+        if override_conditionally:
+            for variable_uuid, _ in variable_mapping.items():
+                variable = self.get_variable_object(
+                    self.uuid,
+                    variable_uuid,
+                )
+                if not variable or not variable.variable_type:
+                    continue
+
+                # if VariableType
+                # variable = self.get_variable_object(variable_uuid=variable_uuid)
+                # if variable.exists():
+                #     variable_mapping.pop(variable_uuid)
+                pass
+
         await self.store_variables_async(
             variable_mapping,
             override=override,
@@ -2386,18 +2864,29 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
 
     def get_executor_type(self) -> str:
         if self.executor_type:
-            return Template(self.executor_type).render(**get_template_vars())
-        return self.executor_type
+            block_executor_type = Template(self.executor_type).render(**get_template_vars())
+        else:
+            block_executor_type = None
+        if not block_executor_type or block_executor_type == ExecutorType.LOCAL_PYTHON:
+            # If block executor_type is not set, fall back to pipeline level executor_type
+            if self.pipeline:
+                pipeline_executor_type = self.pipeline.get_executor_type()
+            else:
+                pipeline_executor_type = None
+            block_executor_type = pipeline_executor_type or block_executor_type
+        return block_executor_type
 
     def get_pipelines_from_cache(self, block_cache: BlockCache = None) -> List[Dict]:
         if block_cache is None:
             block_cache = BlockCache()
-        arr = block_cache.get_pipelines(self)
+        arr = block_cache.get_pipelines(self, self.repo_path)
 
         return unique_by(
             arr,
-            lambda x: (f"{(x.get('pipeline') or {}).get('uuid')}_"
-                       f"{(x.get('pipeline') or {}).get('repo_path')}"),
+            lambda x: (
+                f"{(x.get('pipeline') or {}).get('uuid')}_"
+                f"{(x.get('pipeline') or {}).get('repo_path')}"
+            ),
         )
 
     def to_dict_base(
@@ -2417,7 +2906,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             configuration=self.configuration or {},
             downstream_blocks=self.downstream_block_uuids,
             executor_config=self.executor_config,
-            executor_type=format_enum(self.executor_type) if self.executor_type else None,
+            executor_type=(format_enum(self.executor_type) if self.executor_type else None),
             has_callback=self.has_callback,
             name=self.name,
             language=language,
@@ -2474,10 +2963,11 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             if include_outputs_use:
                 data['outputs'] = self.outputs
 
-                if check_if_file_exists and not \
-                        self.replicated_block and \
-                        BlockType.GLOBAL_DATA_PRODUCT != self.type:
-
+                if (
+                    check_if_file_exists
+                    and not self.replicated_block
+                    and BlockType.GLOBAL_DATA_PRODUCT != self.type
+                ):
                     file_path = self.file_path
                     if not os.path.isfile(file_path):
                         data['error'] = dict(
@@ -2501,9 +2991,12 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         include_content: bool = False,
         include_outputs: bool = False,
         include_outputs_spark: bool = False,
-        sample_count: int = None,
-        block_cache: BlockCache = None,
+        sample_count: Optional[int] = None,
+        block_cache: Optional[BlockCache] = None,
         check_if_file_exists: bool = False,
+        disable_output_preview: bool = False,
+        exclude_blank_variable_uuids: bool = False,
+        max_results: Optional[int] = None,
         **kwargs,
     ) -> Dict:
         data = self.to_dict_base(
@@ -2521,25 +3014,44 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
 
         if include_outputs:
             include_outputs_use = include_outputs
-            if self.is_using_spark() and self.compute_management_enabled():
+            if (
+                self.is_using_spark()
+                and self.compute_management_enabled()
+                and self.pipeline
+                and self.pipeline.type == PipelineType.PYSPARK
+            ):
                 include_outputs_use = include_outputs_use and include_outputs_spark
 
             if include_outputs_use:
-                data['outputs'] = await self.__outputs_async()
-
-                if check_if_file_exists and not \
-                        self.replicated_block and \
-                        BlockType.GLOBAL_DATA_PRODUCT != self.type:
+                if (
+                    disable_output_preview
+                    and self.configuration
+                    and self.configuration.get('disable_output_preview', False)
+                ):
+                    data['outputs'] = [
+                        'Output preview is disabled for this block. '
+                        'To enable it, go to block settings.',
+                    ]
+                else:
+                    data['outputs'] = await self.__outputs_async(
+                        exclude_blank_variable_uuids=exclude_blank_variable_uuids,
+                        max_results=max_results,
+                    )
 
-                    file_path = self.file.file_path
-                    if not os.path.isfile(file_path):
-                        data['error'] = dict(
-                            error='No such file or directory',
-                            message='You may have moved it or changed its filename. '
-                            'Delete the current block to remove it from the pipeline '
-                            'or write code and save the pipeline to create a new file at '
-                            f'{file_path}.',
-                        )
+                    if (
+                        check_if_file_exists
+                        and not self.replicated_block
+                        and BlockType.GLOBAL_DATA_PRODUCT != self.type
+                    ):
+                        file_path = self.file.file_path
+                        if not os.path.isfile(file_path):
+                            data['error'] = dict(
+                                error='No such file or directory',
+                                message='You may have moved it or changed its filename. '
+                                'Delete the current block to remove it from the pipeline '
+                                'or write code and save the pipeline to create a new file at '
+                                f'{file_path}.',
+                            )
 
         if include_block_metadata:
             data['metadata'] = await self.metadata_async()
@@ -2570,9 +3082,8 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
 
         check_upstream_block_order = kwargs.get('check_upstream_block_order', False)
         if 'upstream_blocks' in data and (
-            (check_upstream_block_order and
-                data['upstream_blocks'] != self.upstream_block_uuids) or
-            set(data['upstream_blocks']) != set(self.upstream_block_uuids)
+            (check_upstream_block_order and data['upstream_blocks'] != self.upstream_block_uuids)
+            or set(data['upstream_blocks']) != set(self.upstream_block_uuids)
         ):
             self.__update_upstream_blocks(
                 data['upstream_blocks'],
@@ -2607,7 +3118,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         if 'has_callback' in data and data['has_callback'] != self.has_callback:
             self.has_callback = data['has_callback']
             if self.has_callback:
-                CallbackBlock.create(self.uuid)
+                CallbackBlock.create(self.uuid, self.repo_path)
             self.__update_pipeline_block()
 
         if 'retry_config' in data and data['retry_config'] != self.retry_config:
@@ -2704,10 +3215,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         return list(visited)
 
     def run_upstream_blocks(
-        self,
-        from_notebook: bool = False,
-        incomplete_only: bool = False,
-        **kwargs
+        self, from_notebook: bool = False, incomplete_only: bool = False, **kwargs
     ) -> None:
         def process_upstream_block(
             block: 'Block',
@@ -2717,10 +3225,12 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 root_blocks.append(block)
             return block.uuid
 
-        upstream_blocks = list(filter(
-            lambda x: not incomplete_only or BlockStatus.EXECUTED != x.status,
-            self.get_all_upstream_blocks(),
-        ))
+        upstream_blocks = list(
+            filter(
+                lambda x: not incomplete_only or BlockStatus.EXECUTED != x.status,
+                self.get_all_upstream_blocks(),
+            )
+        )
         root_blocks = []
         upstream_block_uuids = list(
             map(lambda x: process_upstream_block(x, root_blocks), upstream_blocks),
@@ -2758,10 +3268,11 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
 
         self.dynamic_block_uuid = dynamic_block_uuid
 
-        if self.pipeline \
-            and PipelineType.INTEGRATION == self.pipeline.type \
-                and self.type in [BlockType.DATA_LOADER, BlockType.DATA_EXPORTER]:
-
+        if (
+            self.pipeline
+            and PipelineType.INTEGRATION == self.pipeline.type
+            and self.type in [BlockType.DATA_LOADER, BlockType.DATA_EXPORTER]
+        ):
             return
 
         test_functions = []
@@ -2804,7 +3315,9 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                         test_function = getattr(self.module, func.__name__)
                     try:
                         sig = signature(test_function)
-                        has_kwargs = any([p.kind == p.VAR_KEYWORD for p in sig.parameters.values()])
+                        has_kwargs = any([
+                            p.kind == p.VAR_KEYWORD for p in sig.parameters.values()
+                        ])
                         if has_kwargs and global_vars is not None and len(global_vars) != 0:
                             test_function(*outputs, **global_vars)
                         else:
@@ -2815,11 +3328,13 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                         stacktrace = traceback.format_exc()
 
                         if from_notebook:
-                            error_json = json.dumps(dict(
-                                error=str(err),
-                                message=error_message,
-                                stacktrace=stacktrace.split('\n'),
-                            ))
+                            error_json = json.dumps(
+                                dict(
+                                    error=str(err),
+                                    message=error_message,
+                                    stacktrace=stacktrace.split('\n'),
+                                )
+                            )
                             print(f'[__internal_test__]{error_json}')
                         else:
                             print('==============================================================')
@@ -2830,9 +3345,11 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 message = f'{tests_passed}/{len(test_functions)} tests passed.'
                 if from_notebook:
                     if len(test_functions) >= 1:
-                        success_json = json.dumps(dict(
-                            message=message,
-                        ))
+                        success_json = json.dumps(
+                            dict(
+                                message=message,
+                            )
+                        )
                         print(f'[__internal_test__]{success_json}')
                 else:
                     print('--------------------------------------------------------------')
@@ -2859,11 +3376,12 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
     ) -> None:
         if self.pipeline is None:
             return
+
         for uuid, data in variable_mapping.items():
-            if type(data) is pd.DataFrame:
+            if isinstance(data, pd.DataFrame):
                 if data.shape[1] > DATAFRAME_ANALYSIS_MAX_COLUMNS or shape_only:
-                    self.pipeline.variable_manager.add_variable(
-                        self.pipeline.uuid,
+                    self.variable_manager.add_variable(
+                        self.pipeline_uuid,
                         self.uuid,
                         uuid,
                         dict(
@@ -2874,6 +3392,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                         ),
                         partition=execution_partition,
                         variable_type=VariableType.DATAFRAME_ANALYSIS,
+                        disable_variable_type_inference=True,
                     )
                     continue
                 if data.shape[0] > DATAFRAME_ANALYSIS_MAX_ROWS:
@@ -2884,14 +3403,15 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                     data_for_analysis = data.reset_index(drop=True)
                 try:
                     from mage_ai.data_cleaner.data_cleaner import clean as clean_data
+
                     analysis = clean_data(
                         data_for_analysis,
                         df_original=data,
                         transform=False,
                         verbose=False,
                     )
-                    self.pipeline.variable_manager.add_variable(
-                        self.pipeline.uuid,
+                    self.variable_manager.add_variable(
+                        self.pipeline_uuid,
                         self.uuid,
                         uuid,
                         dict(
@@ -2903,14 +3423,15 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                         partition=execution_partition,
                         variable_type=VariableType.DATAFRAME_ANALYSIS,
                     )
-                except Exception:
-                    pass
+                except Exception as err:
+                    if is_debug():
+                        raise err
                     # TODO: we use to silently fail, but it looks bad when using BigQuery
                     # print('\nFailed to analyze dataframe:')
                     # print(traceback.format_exc())
-            elif type(data) is pl.DataFrame:
-                self.pipeline.variable_manager.add_variable(
-                    self.pipeline.uuid,
+            elif isinstance(data, pl.DataFrame):
+                self.variable_manager.add_variable(
+                    self.pipeline_uuid,
                     self.uuid,
                     uuid,
                     dict(
@@ -2921,6 +3442,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                     ),
                     partition=execution_partition,
                     variable_type=VariableType.DATAFRAME_ANALYSIS,
+                    disable_variable_type_inference=True,
                 )
 
     def set_global_vars(self, global_vars: Dict) -> None:
@@ -2930,9 +3452,15 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
 
     def __consolidate_variables(self, variable_mapping: Dict) -> Dict:
         # Consolidate print variables
-        output_variables = {k: v for k, v in variable_mapping.items() if is_output_variable(k)}
-        print_variables = {k: v for k, v in variable_mapping.items()
-                           if is_valid_print_variable(k, v, self.uuid)}
+        if BlockType.SCRATCHPAD == self.type:
+            output_variables = {}
+            print_variables = variable_mapping.copy()
+        else:
+            output_variables = {k: v for k, v in variable_mapping.items() if is_output_variable(k)}
+            print_variables = {
+                k: v for k, v in variable_mapping.items()
+                if is_valid_print_variable(k, v, self.uuid)
+            }
 
         print_variables_keys = sorted(print_variables.keys(), key=lambda k: int(k.split('_')[-1]))
 
@@ -2967,6 +3495,10 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 save_variable_and_reset_state()
                 continue
 
+            if json_value.get('msg_type') == 'status':
+                # Do not save status messages
+                continue
+
             if state['msg_key'] is not None and json_value['msg_type'] != state['msg_type']:
                 save_variable_and_reset_state()
 
@@ -2994,6 +3526,8 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         global_vars: Dict = None,
         dynamic_block_index: int = None,
     ) -> Dict:
+        from mage_ai.data_preparation.models.block.remote.models import RemoteBlock
+
         """
         Enriches the provided global variables dictionary with additional context, Spark session,
         environment, configuration, and an empty context dictionary.
@@ -3021,8 +3555,9 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         """
         if global_vars is None:
             global_vars = dict()
-        if ((self.pipeline is not None and self.pipeline.type == PipelineType.DATABRICKS) or
-                is_spark_env()):
+        if (
+            self.pipeline is not None and self.pipeline.type == PipelineType.DATABRICKS
+        ) or is_spark_env():
             if not global_vars.get('spark'):
                 spark = self.get_spark_session()
                 if spark is not None:
@@ -3034,12 +3569,21 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             global_vars['context'] = dict()
 
         # Add pipeline uuid and block uuid to global_vars
-        global_vars['pipeline_uuid'] = self.pipeline.uuid if self.pipeline else None
+        global_vars['pipeline_uuid'] = self.pipeline_uuid
         global_vars['block_uuid'] = self.uuid
 
         if dynamic_block_index is not None:
             global_vars['dynamic_block_index'] = dynamic_block_index
 
+        # Remote blocks
+        if global_vars.get('remote_blocks'):
+            global_vars['remote_blocks'] = [
+                RemoteBlock.load(
+                    **remote_block_dict,
+                ).get_outputs()
+                for remote_block_dict in global_vars['remote_blocks']
+            ]
+
         self.global_vars = global_vars
 
         return global_vars
@@ -3047,18 +3591,15 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
     def get_spark_session(self):
         if not SPARK_ENABLED:
             return None
-        if self.spark_init and (not self.pipeline or
-                                not self.pipeline.spark_config):
+        if self.spark_init and (not self.pipeline or not self.pipeline.spark_config):
             return self.spark
 
         try:
             if self.pipeline and self.pipeline.spark_config:
-                spark_config = SparkConfig.load(
-                    config=self.pipeline.spark_config)
+                spark_config = SparkConfig.load(config=self.pipeline.spark_config)
             else:
                 repo_config = RepoConfig(repo_path=self.repo_path)
-                spark_config = SparkConfig.load(
-                    config=repo_config.spark_config)
+                spark_config = SparkConfig.load(config=repo_config.spark_config)
             self.spark = get_spark_session(spark_config)
         except Exception:
             self.spark = None
@@ -3083,18 +3624,19 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         spark_config = SparkConfig.load(config=spark_config)
         if spark_config.use_custom_session:
             return global_vars.get('context', dict()).get(
-                spark_config.custom_session_var_name, spark)
+                spark_config.custom_session_var_name, spark
+            )
         return spark
 
-    def __store_variables_prepare(
+    def __get_variable_uuids(
         self,
-        variable_mapping: Dict,
-        execution_partition: str = None,
-        override: bool = False,
-        override_outputs: bool = False,
-        dynamic_block_index: int = None,
-        dynamic_block_uuid: str = None,
-    ) -> Dict:
+        dynamic_block_index: Optional[int] = None,
+        dynamic_block_uuid: Optional[str] = None,
+        execution_partition: Optional[str] = None,
+    ) -> Tuple[List[str], str, bool]:
+        if self.pipeline is None:
+            return []
+
         self.dynamic_block_uuid = dynamic_block_uuid
 
         block_uuid, changed = uuid_for_output_variables(
@@ -3104,21 +3646,37 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             dynamic_block_uuid=dynamic_block_uuid,
         )
 
-        if self.pipeline is None:
-            return
+        return (
+            self.variable_manager.get_variables_by_block(
+                self.pipeline_uuid,
+                block_uuid=block_uuid,
+                partition=execution_partition,
+                clean_block_uuid=not changed,
+            ),
+            block_uuid,
+            changed,
+        )
 
-        all_variables = self.pipeline.variable_manager.get_variables_by_block(
-            self.pipeline.uuid,
-            block_uuid=block_uuid,
-            partition=execution_partition,
-            clean_block_uuid=not changed,
+    def __store_variables_prepare(
+        self,
+        variable_mapping: Dict,
+        execution_partition: Optional[str] = None,
+        override: bool = False,
+        override_outputs: bool = False,
+        dynamic_block_index: Optional[int] = None,
+        dynamic_block_uuid: Optional[str] = None,
+    ) -> Dict:
+        variable_uuids, _block_uuid, _changed = self.__get_variable_uuids(
+            dynamic_block_index=dynamic_block_index,
+            dynamic_block_uuid=dynamic_block_uuid,
+            execution_partition=execution_partition,
         )
 
         variable_mapping = self.__consolidate_variables(variable_mapping)
 
         variable_names = [clean_name_orig(v) for v in variable_mapping]
         removed_variables = []
-        for v in all_variables:
+        for v in variable_uuids:
             if v in variable_names:
                 continue
 
@@ -3131,16 +3689,97 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             variable_mapping=variable_mapping,
         )
 
+    def delete_variables(
+        self,
+        dynamic_block_index: Optional[int] = None,
+        dynamic_block_uuid: Optional[str] = None,
+        execution_partition: Optional[str] = None,
+        variable_uuids: Optional[List[str]] = None,
+    ) -> None:
+        if self.pipeline is None:
+            return
+
+        variable_uuids_all, block_uuid, _changed = self.__get_variable_uuids(
+            dynamic_block_index=dynamic_block_index,
+            dynamic_block_uuid=dynamic_block_uuid,
+            execution_partition=execution_partition,
+        )
+
+        for variable_uuid in variable_uuids or variable_uuids_all:
+            variable_object = self.variable_manager.get_variable_object(
+                self.pipeline_uuid,
+                block_uuid,
+                variable_uuid,
+                partition=execution_partition,
+            )
+            write_policy = (
+                self.write_settings.batch_settings.mode
+                if self.write_settings and self.write_settings.batch_settings
+                else None
+            )
+            if write_policy and variable_object.data_exists():
+                if ExportWritePolicy.FAIL == write_policy:
+                    raise Exception(f'Write policy for block {self.uuid} is {write_policy}.')
+                elif ExportWritePolicy.APPEND == write_policy:
+                    return
+
+            variable_object.delete()
+
+    def aggregate_summary_info(self, execution_partition: Optional[str] = None):
+        """
+        Run this only after executing blocks in a notebook so that reading pipelines
+        don’t take forever to load while waiting for all the nested variable folders
+        to be read.
+        """
+        if not VARIABLE_DATA_OUTPUT_META_CACHE or not self.variable_manager:
+            return
+
+        aggregate_summary_info_for_all_variables(
+            self.variable_manager,
+            self.pipeline_uuid,
+            self.uuid,
+            partition=execution_partition,
+        )
+
     def store_variables(
         self,
         variable_mapping: Dict,
-        execution_partition: str = None,
+        clean_variable_uuid: Optional[bool] = True,
+        dynamic_block_index: Optional[int] = None,
+        dynamic_block_uuid: Optional[str] = None,
+        execution_partition: Optional[str] = None,
         override: bool = False,
         override_outputs: bool = False,
-        spark=None,
-        dynamic_block_index: int = None,
-        dynamic_block_uuid: str = None,
-    ) -> None:
+        skip_delete: Optional[bool] = None,
+        spark: Optional[Any] = None,
+        save_variable_types_only: Optional[Any] = None,
+    ) -> Optional[List[Variable]]:
+        block_uuid, changed = uuid_for_output_variables(
+            self,
+            block_uuid=self.uuid,
+            dynamic_block_index=dynamic_block_index,
+        )
+
+        is_dynamic = is_dynamic_block(self)
+        is_dynamic_child = is_dynamic_block_child(self)
+
+        shared_args = dict(
+            clean_block_uuid=not changed,
+            clean_variable_uuid=not is_dynamic and not is_dynamic_child and clean_variable_uuid,
+            partition=execution_partition,
+        )
+
+        if save_variable_types_only:
+            for variable_uuid, variable_types in variable_mapping.items():
+                self.variable_manager.add_variable_types(
+                    self.pipeline_uuid,
+                    block_uuid,
+                    variable_uuid,
+                    variable_types,
+                    **shared_args,
+                )
+            return []
+
         variables_data = self.__store_variables_prepare(
             variable_mapping,
             execution_partition,
@@ -3149,40 +3788,45 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             dynamic_block_index=dynamic_block_index,
         )
 
-        block_uuid, changed = uuid_for_output_variables(
-            self,
-            block_uuid=self.uuid,
-            dynamic_block_index=dynamic_block_index,
-        )
-
-        is_dynamic_child = is_dynamic_block_child(self)
-        if is_dynamic_child:
+        if not skip_delete and is_dynamic_child:
+            # execute_block_function will take care of this if decorated function is a generator
             delete_variable_objects_for_dynamic_child(
                 self,
                 dynamic_block_index=dynamic_block_index,
                 execution_partition=execution_partition,
             )
 
+        variables = []
         for uuid, data in variables_data['variable_mapping'].items():
-            if spark is not None and self.pipeline.type == PipelineType.PYSPARK \
-                    and type(data) is pd.DataFrame:
+            if (
+                spark is not None
+                and self.pipeline is not None
+                and self.pipeline.type == PipelineType.PYSPARK
+                and isinstance(data, pd.DataFrame)
+            ):
                 data = spark.createDataFrame(data)
-            self.pipeline.variable_manager.add_variable(
-                self.pipeline.uuid,
-                block_uuid,
-                uuid,
-                data,
-                partition=execution_partition,
-                clean_block_uuid=not changed,
-            )
 
-        if not is_dynamic_child:
-            for uuid in variables_data['removed_variables']:
-                self.pipeline.variable_manager.delete_variable(
-                    self.pipeline.uuid,
+            variables.append(
+                self.variable_manager.add_variable(
+                    self.pipeline_uuid,
                     block_uuid,
                     uuid,
+                    data,
+                    resource_usage=self.resource_usage,
+                    write_batch_settings=self.write_batch_settings,
+                    write_chunks=self.write_chunks,
+                    **shared_args,
                 )
+            )
+        if not skip_delete and not is_dynamic_child and variables_data.get('removed_variables'):
+            self.delete_variables(
+                dynamic_block_index=dynamic_block_index,
+                dynamic_block_uuid=dynamic_block_uuid,
+                execution_partition=execution_partition,
+                variable_uuids=variables_data['removed_variables'],
+            )
+
+        return variables
 
     async def store_variables_async(
         self,
@@ -3220,19 +3864,22 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             if spark is not None and type(data) is pd.DataFrame:
                 data = spark.createDataFrame(data)
 
-            await self.pipeline.variable_manager.add_variable_async(
-                self.pipeline.uuid,
+            await self.variable_manager.add_variable_async(
+                self.pipeline_uuid,
                 block_uuid,
                 uuid,
                 data,
                 partition=execution_partition,
                 clean_block_uuid=not changed,
+                write_batch_settings=self.write_batch_settings,
+                write_chunks=self.write_chunks,
+                resource_usage=self.resource_usage,
             )
 
         if not is_dynamic_child:
             for uuid in variables_data['removed_variables']:
-                self.pipeline.variable_manager.delete_variable(
-                    self.pipeline.uuid,
+                self.variable_manager.delete_variable(
+                    self.pipeline_uuid,
                     block_uuid,
                     uuid,
                     clean_block_uuid=not changed,
@@ -3261,8 +3908,8 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         for b in self.upstream_blocks:
             for v in b.output_variables(execution_partition=execution_partition):
                 objs.append(
-                    self.pipeline.variable_manager.get_variable_object(
-                        self.pipeline.uuid,
+                    self.get_variable_object(
+                        self.pipeline_uuid,
                         b.uuid,
                         v,
                         partition=execution_partition,
@@ -3278,6 +3925,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         from_notebook: bool = False,
         global_vars: Dict = None,
         input_args: List[Any] = None,
+        max_results: Optional[int] = None,
         block_uuid: str = None,
     ) -> List[str]:
         return output_variables(
@@ -3289,6 +3937,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             from_notebook=from_notebook,
             global_vars=global_vars,
             input_args=input_args,
+            max_results=max_results,
         )
 
     def output_variable_objects(
@@ -3312,12 +3961,15 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         if len(output_variables) == 0:
             return []
 
-        variable_objects = [self.pipeline.variable_manager.get_variable_object(
-            self.pipeline.uuid,
-            self.uuid,
-            v,
-            partition=execution_partition,
-        ) for v in output_variables]
+        variable_objects = [
+            self.variable_manager.get_variable_object(
+                self.pipeline_uuid,
+                self.uuid,
+                v,
+                partition=execution_partition,
+            )
+            for v in output_variables
+        ]
         if variable_type is not None:
             variable_objects = [v for v in variable_objects if v.variable_type == variable_type]
         return variable_objects
@@ -3353,8 +4005,8 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
     ) -> Any:
         if self.pipeline is None:
             return []
-        return self.pipeline.variable_manager.get_variable_object(
-            self.pipeline.uuid,
+        return self.variable_manager.get_variable_object(
+            self.pipeline_uuid,
             self.uuid,
             variable_uuid,
             partition=execution_partition,
@@ -3391,32 +4043,41 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         self.uuid = new_uuid
 
         # This file has a path in its file_source that must be updated.
-        if project_platform_activated() and \
-                self.file_source_path() and \
-                add_absolute_path(self.file_source_path()) == self.file_path:
-
-            #  /home/src/data-vault/perftools/mage/data_loaders/team/illusory_glitter
+        if (
+            project_platform_activated()
+            and self.file_source_path()
+            and add_absolute_path(self.file_source_path()) == self.file_path
+        ):
+            # /home/src/data-vault/perftools/mage/data_loaders/team/illusory_glitter
             old_file_path_without_extension = str(Path(old_file_path).with_suffix(''))
             #  /home/src/data-vault/perftools/mage/data_loaders/team
-            old_file_path_without_uuid = str(Path(old_file_path_without_extension.replace(
-                str(Path(old_uuid)),
-                '',
-            )))
+            old_file_path_without_uuid = str(
+                Path(
+                    old_file_path_without_extension.replace(
+                        str(Path(old_uuid)),
+                        '',
+                    )
+                )
+            )
 
-            #  perftools/mage/data_loaders/team
+            # perftools/mage/data_loaders/team
             old_file_path_without_repo_path = remove_base_repo_path(old_file_path_without_uuid)
-            #  perftools/mage
+            # perftools/mage
             path_without_block_directory = str(old_file_path_without_repo_path).split(
                 directory_name,
             )[0]
 
             file_extension_new = Path(self.uuid).suffix or file_extension
             # perftools/mage/data_loaders/load_titanic.py
-            new_path = str(Path(os.path.join(
-                path_without_block_directory,
-                directory_name,
-                str(Path(self.uuid).with_suffix('')),
-            )).with_suffix(file_extension_new))
+            new_path = str(
+                Path(
+                    os.path.join(
+                        path_without_block_directory,
+                        directory_name,
+                        str(Path(self.uuid).with_suffix('')),
+                    )
+                ).with_suffix(file_extension_new)
+            )
 
             configuration = self.configuration or {}
             if not configuration.get('file_source'):
@@ -3425,7 +4086,16 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             self.configuration = configuration
 
         # This has to be here
-        new_file_path = self.file_path
+        new_file_path, new_file_path_relative = self.build_file_path_directory(
+            block_uuid=new_uuid,
+        )
+
+        configuration = self.configuration or {}
+        if not configuration.get('file_source'):
+            configuration['file_source'] = {}
+        configuration['file_path'] = new_file_path_relative
+        configuration['file_source']['path'] = new_file_path_relative
+        self.configuration = configuration
 
         if self.pipeline is not None:
             DX_PRINTER.critical(
@@ -3437,7 +4107,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 name=self.name,
                 uuid=self.uuid,
                 file_path=new_file_path,
-                pipeline=self.pipeline.uuid,
+                pipeline=self.pipeline_uuid,
                 repo_path=self.pipeline.repo_path,
                 configuration=self.configuration,
                 __uuid='__update_name',
@@ -3453,20 +4123,23 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 )
 
         if not self.replicated_block and BlockType.GLOBAL_DATA_PRODUCT != self.type:
-            if os.path.exists(new_file_path):
-                raise Exception(f'Block {new_uuid} already exists. Please use a different name.')
+            if new_file_path and os.path.exists(new_file_path):
+                raise Exception(
+                    f'Block {new_uuid} already exists at {new_file_path}. '
+                    'Please use a different name.'
+                )
 
             parent_dir = os.path.dirname(new_file_path)
             os.makedirs(parent_dir, exist_ok=True)
 
             if detach:
-                """"
+                """ "
                 Detaching a block creates a copy of the block file while keeping the existing block
                 file the same. Without detaching a block, the existing block file is simply renamed.
                 """
                 with open(new_file_path, 'w') as f:
                     f.write(block_content)
-            else:
+            elif os.path.exists(old_file_path):
                 os.rename(old_file_path, new_file_path)
 
         if self.pipeline is not None:
@@ -3477,11 +4150,12 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                 from mage_ai.data_preparation.models.block.block_factory import (
                     BlockFactory,
                 )
+
                 """"
                 New block added to pipeline, so it must be added to the block cache.
                 Old block no longer in pipeline, so it must be removed from block cache.
                 """
-                cache.add_pipeline(self, self.pipeline)
+                cache.add_pipeline(self, self.pipeline, self.pipeline.repo_path)
                 old_block = BlockFactory.get_block(
                     old_uuid,
                     old_uuid,
@@ -3489,12 +4163,16 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
                     language=self.language,
                     pipeline=self.pipeline,
                 )
-                cache.remove_pipeline(old_block, self.pipeline.uuid, self.pipeline.repo_path)
+                cache.remove_pipeline(old_block, self.pipeline_uuid, self.pipeline.repo_path)
             else:
-                cache.move_pipelines(self, dict(
-                    type=self.type,
-                    uuid=old_uuid,
-                ))
+                cache.move_pipelines(
+                    self,
+                    dict(
+                        type=self.type,
+                        uuid=old_uuid,
+                    ),
+                    self.pipeline.repo_path,
+                )
 
     def __update_pipeline_block(self, widget=False) -> None:
         if self.pipeline is None:
@@ -3568,7 +4246,9 @@ class SensorBlock(Block):
         block_function: Callable,
         input_vars: List,
         from_notebook: bool = False,
-        global_vars: Dict = None,
+        global_vars: Optional[Dict] = None,
+        *args,
+        **kwargs,
     ) -> List:
         if from_notebook:
             return super().execute_block_function(
@@ -3584,8 +4264,9 @@ class SensorBlock(Block):
             use_global_vars = has_kwargs and global_vars is not None and len(global_vars) != 0
             args = input_vars if has_args else []
             while True:
-                condition = block_function(*args, **global_vars) \
-                            if use_global_vars else block_function()
+                condition = (
+                    block_function(*args, **global_vars) if use_global_vars else block_function()
+                )
                 if condition:
                     break
                 print('Sensor sleeping for 1 minute...')
@@ -3605,7 +4286,7 @@ class AddonBlock(Block):
         global_vars = merge_dict(
             global_vars or dict(),
             dict(
-                pipeline_uuid=self.pipeline.uuid,
+                pipeline_uuid=self.pipeline_uuid,
                 block_uuid=self.uuid,
                 pipeline_run=pipeline_run,
             ),
@@ -3624,9 +4305,7 @@ class AddonBlock(Block):
             global_vars = merge_dict(parent_block.global_vars, global_vars)
             global_vars['parent_block_uuid'] = parent_block.uuid
 
-            if parent_block.pipeline and \
-                    PipelineType.INTEGRATION == parent_block.pipeline.type:
-
+            if parent_block.pipeline and PipelineType.INTEGRATION == parent_block.pipeline.type:
                 template_runtime_configuration = parent_block.template_runtime_configuration
                 index = template_runtime_configuration.get('index', None)
                 is_last_block_run = template_runtime_configuration.get('is_last_block_run', False)
@@ -3652,7 +4331,7 @@ class ConditionalBlock(AddonBlock):
         global_vars: Dict = None,
         logger: Logger = None,
         logging_tags: Dict = None,
-        **kwargs
+        **kwargs,
     ) -> bool:
         with self._redirect_streams(
             logger=logger,
@@ -3696,11 +4375,11 @@ class ConditionalBlock(AddonBlock):
 
 class CallbackBlock(AddonBlock):
     @classmethod
-    def create(cls, orig_block_name) -> 'CallbackBlock':
+    def create(cls, orig_block_name, repo_path: str) -> 'CallbackBlock':
         return Block.create(
             f'{clean_name_orig(orig_block_name)}_callback',
             BlockType.CALLBACK,
-            get_repo_path(),
+            repo_path,
             language=BlockLanguage.PYTHON,
         )
 
@@ -3719,7 +4398,7 @@ class CallbackBlock(AddonBlock):
         from_notebook: bool = False,
         metadata: Dict = None,
         upstream_block_uuids_override: List[str] = None,
-        **kwargs
+        **kwargs,
     ) -> None:
         with self._redirect_streams(
             logger=logger,
@@ -3731,7 +4410,7 @@ class CallbackBlock(AddonBlock):
                 global_vars,
                 parent_block,
                 dynamic_block_index=dynamic_block_index,
-                **kwargs
+                **kwargs,
             )
 
             callback_functions = []
@@ -3813,12 +4492,10 @@ class CallbackBlock(AddonBlock):
                     # As of version 0.8.81, callback functions have access to the parent block’s
                     # data output. If the callback function has any positional arguments, we will
                     # pass in the input variables as positional arguments.
-                    if not input_vars or any(
-                        [
-                            p.kind not in [p.KEYWORD_ONLY, p.VAR_KEYWORD]
-                            for p in sig.parameters.values()
-                        ]
-                    ):
+                    if not input_vars or any([
+                        p.kind not in [p.KEYWORD_ONLY, p.VAR_KEYWORD]
+                        for p in sig.parameters.values()
+                    ]):
                         inner_function(
                             *input_vars,
                             **callback_kwargs,
@@ -3827,9 +4504,12 @@ class CallbackBlock(AddonBlock):
                     # a user has already written callback functions with only keyword arguments.
                     else:
                         inner_function(
-                            **merge_dict(callback_kwargs, dict(
-                                __input=outputs_from_input_vars,
-                            )),
+                            **merge_dict(
+                                callback_kwargs,
+                                dict(
+                                    __input=outputs_from_input_vars,
+                                ),
+                            ),
                         )
 
     def update_content(self, content, widget=False) -> 'CallbackBlock':
@@ -3852,9 +4532,11 @@ class CallbackBlock(AddonBlock):
         def custom_code(callback_status: CallbackStatus = CallbackStatus.SUCCESS, *args, **kwargs):
             # If the decorator is just @callback with no arguments, default to success callback
             if isfunction(callback_status):
+
                 def func(callback_status_inner, *args, **kwargs):
                     if callback_status_inner == CallbackStatus.SUCCESS:
                         return callback_status(*args, **kwargs)
+
                 decorated_functions.append(func)
                 return func
 
@@ -3868,6 +4550,7 @@ class CallbackBlock(AddonBlock):
                 def func(callback_status_inner):
                     if callback_status_inner == callback_status:
                         return function
+
                 decorated_functions.append(func)
 
             return inner