mage-ai 0.9.66__py3-none-any.whl → 0.9.68__py3-none-any.whl

mage_ai/data_preparation/models/block/__init__.py

@@ -90,7 +90,7 @@ from mage_ai.data_preparation.templates.data_integrations.utils import get_templ
 from mage_ai.data_preparation.templates.template import load_template
 from mage_ai.server.kernel_output_parser import DataType
 from mage_ai.services.spark.config import SparkConfig
-from mage_ai.services.spark.spark import get_spark_session
+from mage_ai.services.spark.spark import SPARK_ENABLED, get_spark_session
 from mage_ai.settings.platform.constants import project_platform_activated
 from mage_ai.settings.repo import get_repo_path
 from mage_ai.shared.array import unique_by
@@ -339,7 +339,6 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         self.upstream_blocks = []
         self.downstream_blocks = []
         self.test_functions = []
-        self.global_vars = {}
         self.template_runtime_configuration = {}
 
         self.dynamic_block_index = None
@@ -724,41 +723,6 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 widget=widget,
             )
 
-    @classmethod
-    def block_class_from_type(self, block_type: str, language=None, pipeline=None) -> 'Block':
-        from mage_ai.data_preparation.models.block.constants import BLOCK_TYPE_TO_CLASS
-        from mage_ai.data_preparation.models.block.integration import (
-            DestinationBlock,
-            SourceBlock,
-            TransformerBlock,
-        )
-        from mage_ai.data_preparation.models.block.r import RBlock
-        from mage_ai.data_preparation.models.block.sql import SQLBlock
-        from mage_ai.data_preparation.models.widget import Widget
-
-        if BlockType.CHART == block_type:
-            return Widget
-        elif BlockType.DBT == block_type:
-            from mage_ai.data_preparation.models.block.dbt import DBTBlock
-
-            return DBTBlock
-        elif pipeline and PipelineType.INTEGRATION == pipeline.type:
-            if BlockType.CALLBACK == block_type:
-                return CallbackBlock
-            elif BlockType.CONDITIONAL == block_type:
-                return ConditionalBlock
-            elif BlockType.DATA_LOADER == block_type:
-                return SourceBlock
-            elif BlockType.DATA_EXPORTER == block_type:
-                return DestinationBlock
-            else:
-                return TransformerBlock
-        elif BlockLanguage.SQL == language:
-            return SQLBlock
-        elif BlockLanguage.R == language:
-            return RBlock
-        return BLOCK_TYPE_TO_CLASS.get(block_type)
-
     @classmethod
     def create(
         self,
@@ -778,6 +742,8 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
         widget: bool = False,
         downstream_block_uuids: List[str] = None,
     ) -> 'Block':
+        from mage_ai.data_preparation.models.block.block_factory import BlockFactory
+
         """
         1. Create a new folder for block_type if not exist
         2. Create a new python file with code template
@@ -871,7 +837,11 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                     language,
                 )
 
-        block = self.block_class_from_type(block_type, pipeline=pipeline)(
+        block = BlockFactory.block_class_from_type(
+            block_type,
+            language=language,
+            pipeline=pipeline,
+        )(
             name,
             uuid,
             block_type,
@@ -932,34 +902,6 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                     block_uuids[t.value].append(f.split('.')[0])
         return block_uuids
 
-    @classmethod
-    def get_block(
-        self,
-        name,
-        uuid,
-        block_type,
-        configuration=None,
-        content=None,
-        language=None,
-        pipeline=None,
-        status=BlockStatus.NOT_EXECUTED,
-    ) -> 'Block':
-        block_class = self.block_class_from_type(
-            block_type,
-            language=language,
-            pipeline=pipeline,
-        ) or Block
-        return block_class(
-            name,
-            uuid,
-            block_type,
-            configuration=configuration,
-            content=content,
-            language=language,
-            pipeline=pipeline,
-            status=status,
-        )
-
     @classmethod
     def get_block_from_file_path(self, file_path: str) -> 'Block':
         parts = get_path_parts(file_path)
@@ -1198,7 +1140,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                         f'Please run upstream blocks {upstream_block_uuids} '
                         'before running the current block.'
                     )
-            global_vars = self.__enrich_global_vars(
+            global_vars = self.enrich_global_vars(
                 global_vars,
                 dynamic_block_index=dynamic_block_index,
             )
@@ -2204,6 +2146,7 @@ class Block(DataIntegrationMixin, SparkBlock, ProjectPlatformAccessible):
                 self.pipeline.uuid,
                 block_uuid,
                 partition=execution_partition,
+                max_results=DATAFRAME_SAMPLE_COUNT_PREVIEW,
             )
 
             if not include_print_outputs:
@@ -2445,8 +2388,10 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             return Template(self.executor_type).render(**get_template_vars())
         return self.executor_type
 
-    def get_pipelines_from_cache(self) -> List[Dict]:
-        arr = BlockCache().get_pipelines(self)
+    def get_pipelines_from_cache(self, block_cache: BlockCache = None) -> List[Dict]:
+        if block_cache is None:
+            block_cache = BlockCache()
+        arr = block_cache.get_pipelines(self)
 
         return unique_by(
             arr,
@@ -2497,11 +2442,13 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
     def to_dict(
         self,
         include_block_catalog: bool = False,
+        include_block_pipelines: bool = False,
         include_callback_blocks: bool = False,
         include_content: bool = False,
         include_outputs: bool = False,
         include_outputs_spark: bool = False,
         sample_count: int = None,
+        block_cache: BlockCache = None,
         check_if_file_exists: bool = False,
         **kwargs,
     ) -> Dict:
@@ -2515,6 +2462,9 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         if include_block_catalog and self.is_data_integration() and self.pipeline:
             data['catalog'] = self.get_catalog_from_file()
 
+        if include_block_pipelines:
+            data['pipelines'] = self.get_pipelines_from_cache(block_cache=block_cache)
+
         if include_outputs:
             include_outputs_use = include_outputs
             if self.is_using_spark() and self.compute_management_enabled():
@@ -2551,6 +2501,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         include_outputs: bool = False,
         include_outputs_spark: bool = False,
         sample_count: int = None,
+        block_cache: BlockCache = None,
         check_if_file_exists: bool = False,
         **kwargs,
     ) -> Dict:
@@ -2596,7 +2547,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
             data['tags'] = self.tags()
 
         if include_block_pipelines:
-            data['pipelines'] = self.get_pipelines_from_cache()
+            data['pipelines'] = self.get_pipelines_from_cache(block_cache=block_cache)
 
         return data
 
@@ -3037,7 +2988,7 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         variable_mapping = merge_dict(output_variables, consolidated_print_variables)
         return variable_mapping
 
-    def __enrich_global_vars(
+    def enrich_global_vars(
         self,
         global_vars: Dict = None,
         dynamic_block_index: int = None,
@@ -3093,6 +3044,8 @@ df = get_variable('{self.pipeline.uuid}', '{self.uuid}', 'df')
         return global_vars
 
     def get_spark_session(self):
+        if not SPARK_ENABLED:
+            return None
         if self.spark_init and (not self.pipeline or
                                 not self.pipeline.spark_config):
             return self.spark
@@ -3658,23 +3611,29 @@ class AddonBlock(Block):
             global_vars['dynamic_block_index'] = dynamic_block_index
 
         if parent_block:
+            if parent_block.global_vars is None:
+                global_vars_copy = global_vars.copy()
+                parent_block.enrich_global_vars(
+                    global_vars=global_vars_copy,
+                    dynamic_block_index=dynamic_block_index,
+                )
+            global_vars = merge_dict(parent_block.global_vars, global_vars)
             global_vars['parent_block_uuid'] = parent_block.uuid
 
-        if parent_block and \
-                parent_block.pipeline and \
-                PipelineType.INTEGRATION == parent_block.pipeline.type:
-
-            template_runtime_configuration = parent_block.template_runtime_configuration
-            index = template_runtime_configuration.get('index', None)
-            is_last_block_run = template_runtime_configuration.get('is_last_block_run', False)
-            selected_streams = template_runtime_configuration.get('selected_streams', [])
-            stream = selected_streams[0] if len(selected_streams) >= 1 else None
-            destination_table = template_runtime_configuration.get('destination_table', stream)
-
-            global_vars['index'] = index
-            global_vars['is_last_block_run'] = is_last_block_run
-            global_vars['stream'] = stream
-            global_vars['destination_table'] = destination_table
+            if parent_block.pipeline and \
+                    PipelineType.INTEGRATION == parent_block.pipeline.type:
+
+                template_runtime_configuration = parent_block.template_runtime_configuration
+                index = template_runtime_configuration.get('index', None)
+                is_last_block_run = template_runtime_configuration.get('is_last_block_run', False)
+                selected_streams = template_runtime_configuration.get('selected_streams', [])
+                stream = selected_streams[0] if len(selected_streams) >= 1 else None
+                destination_table = template_runtime_configuration.get('destination_table', stream)
+
+                global_vars['index'] = index
+                global_vars['is_last_block_run'] = is_last_block_run
+                global_vars['stream'] = stream
+                global_vars['destination_table'] = destination_table
 
         return global_vars
 

mage_ai/data_preparation/models/block/block_factory.py

@@ -0,0 +1,77 @@
+from mage_ai.data_preparation.models.block import Block, CallbackBlock, ConditionalBlock
+from mage_ai.data_preparation.models.block.constants import BLOCK_TYPE_TO_CLASS
+from mage_ai.data_preparation.models.block.integration import (
+    DestinationBlock,
+    SourceBlock,
+    TransformerBlock,
+)
+from mage_ai.data_preparation.models.block.r import RBlock
+from mage_ai.data_preparation.models.block.sql import SQLBlock
+from mage_ai.data_preparation.models.constants import (
+    BlockLanguage,
+    BlockStatus,
+    BlockType,
+    PipelineType,
+)
+from mage_ai.data_preparation.models.widget import Widget
+
+try:
+    from mage_ai.data_preparation.models.block.dbt.block_sql import DBTBlockSQL
+    from mage_ai.data_preparation.models.block.dbt.block_yaml import DBTBlockYAML
+except Exception:
+    print('DBT library not installed.')
+
+
+class BlockFactory:
+    @classmethod
+    def block_class_from_type(self, block_type: str, language=None, pipeline=None) -> 'Block':
+        if BlockType.CHART == block_type:
+            return Widget
+        elif BlockType.DBT == block_type:
+            if language == BlockLanguage.YAML:
+                return DBTBlockYAML
+            return DBTBlockSQL
+        elif pipeline and PipelineType.INTEGRATION == pipeline.type:
+            if BlockType.CALLBACK == block_type:
+                return CallbackBlock
+            elif BlockType.CONDITIONAL == block_type:
+                return ConditionalBlock
+            elif BlockType.DATA_LOADER == block_type:
+                return SourceBlock
+            elif BlockType.DATA_EXPORTER == block_type:
+                return DestinationBlock
+            else:
+                return TransformerBlock
+        elif BlockLanguage.SQL == language:
+            return SQLBlock
+        elif BlockLanguage.R == language:
+            return RBlock
+        return BLOCK_TYPE_TO_CLASS.get(block_type)
+
+    @classmethod
+    def get_block(
+        self,
+        name,
+        uuid,
+        block_type,
+        configuration=None,
+        content=None,
+        language=None,
+        pipeline=None,
+        status=BlockStatus.NOT_EXECUTED,
+    ) -> 'Block':
+        block_class = BlockFactory.block_class_from_type(
+            block_type,
+            language=language,
+            pipeline=pipeline,
+        ) or Block
+        return block_class(
+            name,
+            uuid,
+            block_type,
+            configuration=configuration,
+            content=content,
+            language=language,
+            pipeline=pipeline,
+            status=status,
+        )

mage_ai/data_preparation/models/block/dbt/block.py

@@ -26,19 +26,17 @@ from mage_ai.shared.parsers import encode_complex
 
 
 class DBTBlock(Block):
-    def __new__(cls, *args, **kwargs) -> 'DBTBlock':
+    @classmethod
+    def create(cls, *args, **kwargs) -> 'DBTBlock':
         """
         Factory for the child blocks
         """
         # Import Child blocks here to prevent cycle import
         from mage_ai.data_preparation.models.block.dbt.block_sql import DBTBlockSQL
         from mage_ai.data_preparation.models.block.dbt.block_yaml import DBTBlockYAML
-        if cls is DBTBlock:
-            if kwargs.get('language', BlockLanguage.SQL) == BlockLanguage.YAML:
-                return super(DBTBlock, cls).__new__(DBTBlockYAML)
-            return super(DBTBlock, cls).__new__(DBTBlockSQL)
-        else:
-            return super(DBTBlock, cls).__new__(cls, *args, **kwargs)
+        if kwargs.get('language', BlockLanguage.SQL) == BlockLanguage.YAML:
+            return DBTBlockYAML(*args, **kwargs)
+        return DBTBlockSQL(*args, **kwargs)
 
     @property
     def base_project_path(self) -> Union[str, os.PathLike]:

mage_ai/data_preparation/models/block/global_data_product/__init__.py

@@ -2,12 +2,15 @@ from logging import Logger
 from typing import Dict, List
 
 from mage_ai.data_preparation.models.block import Block
-from mage_ai.data_preparation.models.global_data_product import GlobalDataProduct
-from mage_ai.orchestration.triggers import global_data_product as trigger
 
 
 class GlobalDataProductBlock(Block):
-    def get_global_data_product(self) -> GlobalDataProduct:
+    def get_global_data_product(self):
+        # Avoid circular dependency of Pipeline class
+        from mage_ai.data_preparation.models.global_data_product import (
+            GlobalDataProduct,
+        )
+
         override_configuration = (self.configuration or {}).get('global_data_product', {})
         global_data_product = GlobalDataProduct.get(override_configuration.get('uuid'))
 
@@ -31,6 +34,9 @@ class GlobalDataProductBlock(Block):
         logging_tags: Dict = None,
         **kwargs,
     ) -> List:
+        # Avoid circular dependency of Pipeline class
+        from mage_ai.orchestration.triggers import global_data_product as trigger
+
         trigger.trigger_and_check_status(
             self.get_global_data_product(),
             block=self,

mage_ai/data_preparation/models/block/integration/__init__.py

@@ -416,9 +416,10 @@ class SourceBlock(IntegrationBlock):
 class DestinationBlock(IntegrationBlock):
     def to_dict(
         self,
-        include_content=False,
-        include_outputs=False,
-        sample_count=None,
+        include_content: bool = False,
+        include_outputs: bool = False,
+        include_block_pipelines: bool = False,
+        sample_count: int = None,
         check_if_file_exists: bool = False,
         destination_table: str = None,
         state_stream: str = None,
@@ -444,6 +445,7 @@ class DestinationBlock(IntegrationBlock):
             super().to_dict(
                 include_content=include_content,
                 include_outputs=include_outputs,
+                include_block_pipelines=include_block_pipelines,
                 sample_count=sample_count,
                 check_if_file_exists=check_if_file_exists,
             ),
@@ -452,9 +454,10 @@ class DestinationBlock(IntegrationBlock):
 
     async def to_dict_async(
         self,
-        include_content=False,
-        include_outputs=False,
-        sample_count=None,
+        include_content: bool = False,
+        include_outputs: bool = False,
+        include_block_pipelines: bool = False,
+        sample_count: int = None,
         check_if_file_exists: bool = False,
         destination_table: str = None,
         state_stream: str = None,
@@ -463,13 +466,14 @@ class DestinationBlock(IntegrationBlock):
         return self.to_dict(
             include_content=include_content,
             include_outputs=include_outputs,
+            include_block_pipelines=include_block_pipelines,
             sample_count=sample_count,
             check_if_file_exists=check_if_file_exists,
             destination_table=destination_table,
             state_stream=state_stream,
         )
 
-    def update(self, data, update_state=False):
+    def update(self, data, update_state=False, **kwargs):
         if update_state:
             from mage_ai.data_preparation.models.pipelines.integration_pipeline import (
                 IntegrationPipeline,
@@ -493,7 +497,7 @@ class DestinationBlock(IntegrationBlock):
                     bookmark_values=bookmark_values
                 )
 
-        return super().update(data)
+        return super().update(data, **kwargs)
 
     def output_variables(self, execution_partition: str = None) -> List[str]:
         return []

mage_ai/data_preparation/models/block/platform/mixins.py

@@ -233,7 +233,7 @@ class ProjectPlatformAccessible:
                 ),
             )
 
-        return block_class(
+        return block_class.create(
             name,
             uuid,
             block_type,

mage_ai/data_preparation/models/pipeline.py

@@ -15,8 +15,10 @@ import yaml
 from jinja2 import Template
 
 from mage_ai.authentication.permissions.constants import EntityName
+from mage_ai.cache.block import BlockCache
 from mage_ai.cache.pipeline import PipelineCache
 from mage_ai.data_preparation.models.block import Block, run_blocks, run_blocks_sync
+from mage_ai.data_preparation.models.block.block_factory import BlockFactory
 from mage_ai.data_preparation.models.block.data_integration.utils import (
     convert_outputs_to_data,
 )
@@ -82,6 +84,8 @@ class Pipeline:
         repo_config=None,
         catalog=None,
         use_repo_path: bool = False,
+        description: str = None,
+        tags: List[str] = None,
     ):
         self.block_configs = []
         self.blocks_by_uuid = {}
@@ -90,7 +94,7 @@ class Pipeline:
         self.concurrency_config = dict()
         self.created_at = None
         self.data_integration = None
-        self.description = None
+        self.description = description
         self.executor_config = dict()
         self.executor_type = None
         self.extensions = {}
@@ -101,7 +105,7 @@ class Pipeline:
         self.run_pipeline_in_one_process = False
         self.schedules = []
         self.settings = {}
-        self.tags = []
+        self.tags = tags or []
         self.type = PipelineType.PYTHON
         self.use_repo_path = use_repo_path
         self.uuid = uuid
@@ -215,7 +219,14 @@ class Pipeline:
             self.widget_configs
 
     @classmethod
-    def create(self, name, pipeline_type=PipelineType.PYTHON, repo_path=None):
+    def create(
+        self,
+        name: str,
+        description: str = None,
+        pipeline_type: PipelineType = PipelineType.PYTHON,
+        repo_path: str = None,
+        tags: List[str] = None,
+    ):
         """
         1. Create a new folder for pipeline
         2. Create a new yaml file to store pipeline config
@@ -231,14 +242,20 @@ class Pipeline:
         with open(os.path.join(pipeline_path, PIPELINE_CONFIG_FILE), 'w') as fp:
             yaml.dump(dict(
                 created_at=str(datetime.now(tz=pytz.UTC)),
+                description=description,
                 name=name,
+                tags=tags or [],
                 uuid=uuid,
                 type=format_enum(pipeline_type or PipelineType.PYTHON),
             ), fp)
+
         pipeline = Pipeline(
             uuid,
+            description=description,
             repo_path=repo_path,
+            tags=tags or [],
         )
+
         return pipeline
 
     @classmethod
@@ -368,6 +385,54 @@ class Pipeline:
             IntegrationPipeline,
         )
 
+        config_path, repo_path = self._get_config_path(
+            uuid,
+            repo_path=repo_path,
+            all_projects=all_projects,
+            use_repo_path=use_repo_path,
+        )
+
+        if check_if_exists and not os.path.exists(config_path):
+            return None
+
+        pipeline = self(uuid, repo_path=repo_path, use_repo_path=use_repo_path)
+        if PipelineType.INTEGRATION == pipeline.type:
+            pipeline = IntegrationPipeline(uuid, repo_path=repo_path)
+
+        return pipeline
+
+    @classmethod
+    def get_config(
+        self,
+        uuid,
+        repo_path: str = None,
+        all_projects: bool = False,
+        use_repo_path: bool = False,
+    ):
+        config_path, _ = self._get_config_path(
+            uuid,
+            repo_path=repo_path,
+            all_projects=all_projects,
+            use_repo_path=use_repo_path,
+        )
+
+        metadata_path = os.path.join(config_path, PIPELINE_CONFIG_FILE)
+
+        if not os.path.exists(metadata_path):
+            return None
+
+        with open(metadata_path) as fp:
+            config = yaml.full_load(fp) or {}
+        return config
+
+    @classmethod
+    def _get_config_path(
+        self,
+        uuid,
+        repo_path: str = None,
+        all_projects: bool = False,
+        use_repo_path: bool = False,
+    ) -> Tuple[str, str]:
         if all_projects and not use_repo_path and project_platform_activated():
             from mage_ai.settings.platform.utils import get_pipeline_config_path
 
@@ -379,15 +444,7 @@ class Pipeline:
                 PIPELINES_FOLDER,
                 uuid,
             )
-
-        if check_if_exists and not os.path.exists(config_path):
-            return None
-
-        pipeline = self(uuid, repo_path=repo_path, use_repo_path=use_repo_path)
-        if PipelineType.INTEGRATION == pipeline.type:
-            pipeline = IntegrationPipeline(uuid, repo_path=repo_path)
-
-        return pipeline
+        return config_path, repo_path
 
     @classmethod
     async def load_metadata(
@@ -717,7 +774,8 @@ class Pipeline:
                 )
 
             language = c.get('language')
-            return Block.block_class_from_type(block_type, language=language, pipeline=self)(
+
+            return BlockFactory.block_class_from_type(block_type, language=language, pipeline=self)(
                 c.get('name'),
                 c.get('uuid'),
                 block_type,
@@ -931,6 +989,8 @@ class Pipeline:
             include_outputs=include_outputs,
             sample_count=sample_count,
         )
+        if include_block_pipelines:
+            shared_kwargs['block_cache'] = BlockCache()
         blocks_data = await asyncio.gather(
             *[b.to_dict_async(**merge_dict(shared_kwargs, dict(
                 include_block_catalog=include_block_catalog,
@@ -991,6 +1051,7 @@ class Pipeline:
         old_uuid = None
         blocks_to_remove_from_cache = []
         block_uuids_to_add_to_cache = []
+        tags_to_remove_from_cache = []
         should_update_block_cache = False
         should_update_tag_cache = False
 
@@ -1043,6 +1104,10 @@ class Pipeline:
             old_tags = self.tags or []
 
             if sorted(new_tags) != sorted(old_tags):
+                tags_diff = set(old_tags).symmetric_difference(set(new_tags))
+                for tag in tags_diff:
+                    if tag in old_tags:
+                        tags_to_remove_from_cache.append(tag)
                 self.tags = new_tags
                 should_save = True
                 should_update_tag_cache = True
@@ -1296,6 +1361,8 @@ class Pipeline:
 
             cache = await TagCache.initialize_cache()
 
+            for tag_uuid in tags_to_remove_from_cache:
+                cache.remove_pipeline(tag_uuid, self.uuid, self.repo_path)
             for tag_uuid in self.tags:
                 if old_uuid:
                     cache.remove_pipeline(tag_uuid, old_uuid, self.repo_path)

mage_ai/data_preparation/storage/gcs_storage.py

@@ -29,11 +29,36 @@ class GCSStorage(BaseStorage):
             path += '/'
         return self.path_exists(path)
 
-    def listdir(self, path: str, suffix: str = None) -> List[str]:
+    def listdir(
+        self,
+        path: str,
+        suffix: str = None,
+        max_results: int = None,
+    ) -> List[str]:
         if not path.endswith('/'):
             path += '/'
         path = gcs_url_path(path)
-        blobs = self.bucket.list_blobs(prefix=path)
+
+        """
+        NOTE: The number of results returned by the GCS bucket list_blobs method may
+        be lower than the number entered for the max_results argument, since this
+        method recursively fetches files inside of subdirectories at the path
+        specified, not just the directories themselves at the top level of the path.
+
+        As a result, when fetching block output folders in a remote variables directory,
+        the json or parquet files inside of the block output folders (e.g. the "type.json"
+        file in folder "output_0") are counted towards the "max_results" limit. If max_results
+        is set to 10, less than 10 output folders may be returned due to there being multiple
+        json or other files in each output folder. We do not increase the max_results or
+        include a "delimiter" argument because it would increase load times to unacceptable levels.
+
+        Example block output path in GCS:
+            gs://mage_demo/pipelines/example_pipeline/.variables/example_block/output_0/
+        """
+        blobs = self.bucket.list_blobs(
+            prefix=path,
+            max_results=max_results,
+        )
         keys = []
         for blob in blobs:
             # Avoid finding files recursevively in the dir path.
@@ -123,7 +148,7 @@ class GCSStorage(BaseStorage):
         df.write_parquet(buffer)
         buffer.seek(0)
         blob = self.bucket.blob(gcs_url_path(file_path))
-        blob.upload_from_string(buffer)
+        blob.upload_from_string(buffer.getvalue())
 
     @contextmanager
     def open_to_write(self, file_path: str) -> None: