mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl

mage_ai/data_preparation/models/block/dynamic/utils.py

@@ -2,14 +2,39 @@ import json
 import os
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import pandas as pd
+import polars as pl
+from scipy.sparse import csr_matrix
 
+from mage_ai.data_preparation.models.block.dynamic.shared import (
+    has_dynamic_block_upstream_parent as has_dynamic_block_upstream_parent_v1,
+)
+from mage_ai.data_preparation.models.block.dynamic.shared import (
+    has_reduce_output_from_upstreams as has_reduce_output_from_upstreams_v1,
+)
+from mage_ai.data_preparation.models.block.dynamic.shared import (
+    is_dynamic_block as is_dynamic_block_v1,
+)
+from mage_ai.data_preparation.models.block.dynamic.shared import (
+    is_dynamic_block_child as is_dynamic_block_child_v1,
+)
+from mage_ai.data_preparation.models.block.dynamic.shared import (
+    should_reduce_output as should_reduce_output_v1,
+)
 from mage_ai.data_preparation.models.block.dynamic.variables import (
     get_outputs_for_dynamic_block,
 )
-from mage_ai.data_preparation.models.constants import DATAFRAME_ANALYSIS_MAX_COLUMNS
+from mage_ai.data_preparation.models.constants import (
+    DATAFRAME_ANALYSIS_MAX_COLUMNS,
+    DATAFRAME_SAMPLE_COUNT_PREVIEW,
+)
+from mage_ai.data_preparation.models.utils import (
+    is_basic_iterable,
+    is_dataframe_or_series,
+    prepare_data_for_output,
+)
 from mage_ai.server.kernel_output_parser import DataType
 from mage_ai.shared.array import find
 from mage_ai.shared.custom_logger import DX_PRINTER
@@ -73,66 +98,23 @@ def extract_dynamic_block_index(block_run_block_uuid: str) -> int:
 
 
 def is_dynamic_block(block) -> bool:
-    """
-    Checks if the given block is a dynamic block.
-
-    Args:
-        block: The block.
-
-    Returns:
-        bool: True if the block is a dynamic block, False otherwise.
-    """
-    return block and block.configuration and block.configuration.get('dynamic', False)
+    return is_dynamic_block_v1(block)
 
 
 def should_reduce_output(block) -> bool:
-    """
-    Checks if the given block should reduce its output.
-
-    Args:
-        block: The block.
-
-    Returns:
-        bool: True if the block should reduce its output, False otherwise.
-    """
-    if not block or not block.configuration or not block.configuration.get('reduce_output', False):
-        return False
-    return True
+    return should_reduce_output_v1(block)
 
 
 def has_reduce_output_from_upstreams(block) -> bool:
-    return any([should_reduce_output(upstream_block) for upstream_block in block.upstream_blocks])
+    return has_reduce_output_from_upstreams_v1(block)
 
 
 def has_dynamic_block_upstream_parent(block) -> bool:
-    return block.upstream_blocks and any([is_dynamic_block(b) for b in block.upstream_blocks])
-
-
-def is_dynamic_block_child(block) -> bool:
-    """
-    Checks if the given block is a dynamic block child.
+    return has_dynamic_block_upstream_parent_v1(block)
 
-    Args:
-        block: The block.
-
-    Returns:
-        bool: True if the block is a dynamic block child, False otherwise.
-    """
-    if not block:
-        return False
-
-    dynamic_or_child = []
 
-    for upstream_block in block.upstream_blocks:
-        if is_dynamic_block(upstream_block) or is_dynamic_block_child(upstream_block):
-            dynamic_or_child.append(upstream_block)
-
-    if len(dynamic_or_child) == 0:
-        return False
-
-    dynamic_or_child_with_reduce = list(filter(lambda x: should_reduce_output(x), dynamic_or_child))
-
-    return len(dynamic_or_child) > len(dynamic_or_child_with_reduce)
+def is_dynamic_block_child(block, include_reduce_output: Optional[bool] = None) -> bool:
+    return is_dynamic_block_child_v1(block, include_reduce_output=include_reduce_output)
 
 
 def is_replicated_block(block) -> bool:
@@ -149,6 +131,7 @@ def is_original_dynamic_child_block(
 
     def __get_block_run(block_run_id=block_run_id):
         from mage_ai.orchestration.db.models.schedules import BlockRun
+
         return BlockRun.query.get(block_run_id)
 
     if block:
@@ -176,10 +159,10 @@ def is_original_dynamic_child_block(
 
 def uuid_for_output_variables(
     block,
-    block_uuid: str = None,
-    dynamic_block_index: int = None,
-    dynamic_block_uuid: str = None,
-    join_character: str = None,
+    block_uuid: Optional[str] = None,
+    dynamic_block_index: Optional[int] = None,
+    dynamic_block_uuid: Optional[str] = None,
+    join_character: Optional[str] = None,
     **kwargs,
 ) -> Tuple[str, bool]:
     changed = False
@@ -196,9 +179,9 @@ def uuid_for_output_variables(
         return os.path.join(block.uuid, str(dynamic_block_index)), True
 
     is_dynamic = is_dynamic_block(block)
-    if (not is_dynamic and (dynamic_block_index is None or is_dynamic_child)) or \
-            (is_dynamic and is_dynamic_child):
-
+    if (not is_dynamic and (dynamic_block_index is None or is_dynamic_child)) or (
+        is_dynamic and is_dynamic_child
+    ):
         parts = block_uuid.split(':')
 
         if len(parts) >= 2:
@@ -236,26 +219,38 @@ def uuid_for_output_variables(
     return (block_uuid, changed)
 
 
-def transform_dataframe_for_display(dataframe: pd.DataFrame) -> Dict:
+def transform_dataframe_for_display(
+    dataframe: pd.DataFrame,
+    is_dynamic: bool = False,
+    is_dynamic_child: bool = False,
+    sample_columns: int = DATAFRAME_ANALYSIS_MAX_COLUMNS,
+    sample_count: int = DATAFRAME_SAMPLE_COUNT_PREVIEW,
+    shape: Optional[Tuple[int, int]] = None,
+) -> Dict:
     data = None
+
+    column_name = '-'
+    if is_dynamic:
+        column_name = 'dynamic child blocks'
+    elif is_dynamic_child:
+        column_name = 'output'
+
     if isinstance(dataframe, pd.DataFrame):
-        columns_to_display = dataframe.columns.tolist()[:DATAFRAME_ANALYSIS_MAX_COLUMNS]
-        row_count, column_count = dataframe.shape
+        columns_to_display = dataframe.columns.tolist()[:sample_columns]
+        row_count, column_count = shape or dataframe.shape
+
+        df = dataframe.iloc[:sample_count][columns_to_display]
 
         data = dict(
             columns=columns_to_display,
-            rows=json.loads(
-                dataframe[columns_to_display].to_json(orient='split')
-            )['data'],
-            index=list(dataframe.index),
+            rows=json.loads(df.to_json(orient='split'))['data'],
             shape=[row_count, column_count],
         )
     else:
         data = dict(
-            columns=['col0'],
-            rows=[[dataframe]],
-            index=[0],
-            shape=[1, 1],
+            columns=[column_name],
+            rows=[[dataframe[:sample_count]]],
+            shape=list(shape) if shape else [1, 1],
         )
 
     return dict(
@@ -264,10 +259,26 @@ def transform_dataframe_for_display(dataframe: pd.DataFrame) -> Dict:
     )
 
 
-def coerce_into_dataframe(child_data: Union[
-    List[Union[Dict, int, str, pd.DataFrame]],
-    pd.DataFrame
-]) -> Dict:
+def coerce_into_dataframe(
+    child_data: Union[
+        List[Union[Dict, int, str, pd.DataFrame]],
+        Dict,
+        int,
+        str,
+        pd.DataFrame,
+    ],
+    is_dynamic: bool = False,
+    is_dynamic_child: bool = False,
+    single_item_only: bool = False,
+) -> pd.DataFrame:
+    child_data, _ = prepare_data_for_output(child_data, single_item_only=single_item_only)
+
+    column_name = '-'
+    if is_dynamic:
+        column_name = 'dynamic child blocks'
+    elif is_dynamic_child:
+        column_name = 'output'
+
     if isinstance(child_data, list) and len(child_data) >= 1:
         item = child_data[0]
         if isinstance(item, pd.DataFrame):
@@ -276,34 +287,51 @@ def coerce_into_dataframe(child_data: Union[
             child_data = pd.DataFrame(child_data)
         else:
             child_data = pd.DataFrame(
-                [dict(col=value) for value in child_data],
+                [
+                    {
+                        column_name: value,
+                    }
+                    for value in child_data
+                ],
             )
-    elif isinstance(child_data, pd.DataFrame):
+    elif isinstance(child_data, (pd.DataFrame, pl.DataFrame)):
         return child_data
     else:
-        child_data = pd.DataFrame([dict(col=child_data)])
+        child_data = pd.DataFrame([
+            {
+                column_name: child_data,
+            }
+        ])
 
     return child_data
 
 
-def limit_output(output: Union[List, pd.DataFrame], sample_count: int) -> Union[List, pd.DataFrame]:
-    if sample_count is not None:
-        if output is not None:
-            if isinstance(output, list):
-                output = output[:sample_count]
-            elif isinstance(output, pd.DataFrame):
+def limit_output(
+    output: Union[List, pd.DataFrame, pd.Series, csr_matrix],
+    sample_count: int = DATAFRAME_SAMPLE_COUNT_PREVIEW,
+    sample_columns: Optional[int] = DATAFRAME_ANALYSIS_MAX_COLUMNS,
+) -> Union[List, pd.DataFrame, pd.Series, csr_matrix]:
+    if sample_count is not None and output is not None:
+        if isinstance(output, list):
+            output = output[:sample_count]
+        elif isinstance(output, pd.Series):
+            output = output.iloc[:sample_count]
+        elif isinstance(output, pd.DataFrame):
+            if sample_columns is not None:
+                output = output.iloc[:sample_count, :sample_columns]
+            else:
                 output = output.iloc[:sample_count]
+        elif isinstance(output, csr_matrix):
+            output = output[:sample_count]  # For csr_matrix, this slices rows
+            if sample_columns is not None:
+                output = output[:, :sample_columns]
     return output
 
 
 def transform_output(
-    output: Tuple[
-        Union[
-            List[Union[Dict, int, str, pd.DataFrame]],
-            pd.DataFrame
-        ],
-        List[Dict]
-    ],
+    output: Tuple[Union[List[Union[Dict, int, str, pd.DataFrame]], pd.DataFrame], List[Dict]],
+    is_dynamic: bool = False,
+    is_dynamic_child: bool = False,
 ):
     child_data = None
     metadata = None
@@ -316,39 +344,66 @@ def transform_output(
     if child_data is None:
         return []
 
-    child_data = coerce_into_dataframe(child_data)
+    child_data = coerce_into_dataframe(
+        child_data,
+        is_dynamic=is_dynamic,
+        is_dynamic_child=is_dynamic_child,
+    )
 
     if isinstance(child_data, tuple):
-        return transform_output(child_data)
+        return transform_output(
+            child_data,
+            is_dynamic=is_dynamic,
+            is_dynamic_child=is_dynamic_child,
+        )
     elif isinstance(child_data, list):
-        child_data = [transform_dataframe_for_display(data) for data in child_data]
+        child_data = [
+            transform_dataframe_for_display(
+                data,
+                is_dynamic=is_dynamic,
+                is_dynamic_child=is_dynamic_child,
+            )
+            for data in child_data
+        ]
     else:
-        child_data = transform_dataframe_for_display(child_data)
+        child_data = transform_dataframe_for_display(
+            child_data,
+            is_dynamic=is_dynamic,
+            is_dynamic_child=is_dynamic_child,
+        )
 
     if metadata is not None:
-        metadata = transform_dataframe_for_display(coerce_into_dataframe(metadata))
+        metadata = transform_dataframe_for_display(
+            coerce_into_dataframe(
+                metadata,
+                is_dynamic=is_dynamic,
+                is_dynamic_child=is_dynamic_child,
+            ),
+            is_dynamic=is_dynamic,
+            is_dynamic_child=is_dynamic_child,
+        )
 
     return child_data, metadata
 
 
 def transform_output_for_display(
-    output: Tuple[
-        Union[
-            List[Union[Dict, int, str, pd.DataFrame]],
-            pd.DataFrame
-        ],
-        List[Dict]
-    ],
-    sample_count: int = None,
+    output: Tuple[Union[List[Union[Dict, int, str, pd.DataFrame]], pd.DataFrame], List[Dict]],
+    is_dynamic: bool = False,
+    is_dynamic_child: bool = False,
+    sample_count: Optional[int] = None,
+    sample_columns: Optional[int] = None,
 ) -> List[Dict]:
-    child_data, metadata = transform_output(output)
-    child_data = limit_output(child_data, sample_count)
-    metadata = limit_output(metadata, sample_count)
+    child_data, metadata = transform_output(
+        output,
+        is_dynamic=is_dynamic,
+        is_dynamic_child=is_dynamic_child,
+    )
+    child_data = limit_output(child_data, sample_count, sample_columns=sample_columns)
+    metadata = limit_output(metadata, sample_count, sample_columns=sample_columns)
 
     return dict(
         data=dict(
-            columns=['child_data', 'metadata'],
-            index=[0, 1],
+            columns=['dynamic children data', 'metadata'],
             rows=[child_data, metadata],
             shape=[2, 2],
         ),
@@ -359,34 +414,54 @@ def transform_output_for_display(
 
 def transform_output_for_display_reduce_output(
     output: List[Any],
-    sample_count: int = None,
+    is_dynamic: bool = False,
+    is_dynamic_child: bool = False,
+    sample_count: Optional[int] = None,
+    sample_columns: Optional[int] = None,
 ) -> List[Dict]:
-    output = limit_output(output, sample_count)
+    output = [
+        limit_output(
+            values,
+            sample_count=sample_count,
+            sample_columns=sample_columns,
+        )
+        for values in output
+    ]
 
-    arr = [dict(
-        text_data=data,
-        type=DataType.TEXT,
-        variable_uuid=f'output_{idx}',
-    ) for idx, data in enumerate(output)]
+    output = limit_output(output, sample_count=sample_count, sample_columns=sample_columns)
+
+    arr = [
+        dict(
+            text_data=data,
+            type=DataType.TEXT,
+            variable_uuid=f'reduced output {idx}',
+        )
+        for idx, data in enumerate(output)
+    ]
 
     return arr
 
 
-def combine_transformed_output_for_multi_output(transform_outputs: List[Dict]):
-    columns = []
-    index = []
+def combine_transformed_output_for_multi_output(
+    transform_outputs: List[Dict],
+    columns: Optional[List[str]] = None,
+    is_grouping: bool = False,
+) -> Dict[str, Union[DataType, Dict, bool, str]]:
+    columns_use = columns or []
     for i in range(len(transform_outputs)):
-        columns.append(f'child_{i}')
-        index.append(i)
+        if not columns:
+            columns_use.append(f'output {i}')
 
     return dict(
         data=dict(
-            columns=columns,
-            index=index,
+            columns=columns_use,
             rows=transform_outputs,
-            shape=[len(transform_outputs), len(columns)],
+            shape=[
+                len(transform_outputs),
+                len(columns_use),
+            ],
         ),
-        type=DataType.TABLE,
+        type=DataType.GROUP if is_grouping else DataType.TABLE,
         multi_output=True,
     )
 
@@ -398,25 +473,41 @@ def transform_output_for_display_dynamic_child(
             Dict,
             int,
             str,
-            pd.DataFrame
-        ]
+            pd.DataFrame,
+        ],
     ],
     is_dynamic: bool = False,
-    sample_count: int = None,
+    is_dynamic_child: bool = False,
+    single_item_only: bool = False,
 ) -> List[Dict]:
     df = None
     for output_from_variable_object in output:
-        df_inner = coerce_into_dataframe(output_from_variable_object)
+        df_inner = coerce_into_dataframe(
+            output_from_variable_object,
+            is_dynamic=is_dynamic,
+            is_dynamic_child=is_dynamic_child,
+            single_item_only=single_item_only,
+        )
         if df is None:
             df = df_inner
         else:
-            df = pd.concat([df, df_inner], axis=1)
-
-    df = limit_output(df, sample_count)
-    if 1 == len(set(df.columns)):
-        df.columns = [f'{col}_{idx}' for idx, col in enumerate(df.columns)]
-
-    return transform_dataframe_for_display(df)
+            df.reset_index(drop=True, inplace=True)
+            df_inner.reset_index(drop=True, inplace=True)
+            df = pd.concat([df, df_inner], axis=1, ignore_index=True)
+
+    shape = None
+    if hasattr(df, 'shape'):
+        shape = df.shape
+
+    if isinstance(df, pd.DataFrame) and len(set(df.columns)) == 1:
+        df.columns = [str(idx) for idx, col in enumerate(df.columns)]
+
+    return transform_dataframe_for_display(
+        df,
+        is_dynamic=is_dynamic,
+        is_dynamic_child=is_dynamic_child,
+        shape=shape,
+    )
 
 
 def create_combinations(combinations: List[Any]) -> List[Any]:
@@ -425,7 +516,7 @@ def create_combinations(combinations: List[Any]) -> List[Any]:
 
         for idx, arr in enumerate(combinations_inner):
             for value in arr:
-                combinations_next = combinations_inner[(idx + 1):]
+                combinations_next = combinations_inner[(idx + 1) :]
                 if len(combinations_next) >= 1:
                     for combos_down in __create_combinations(combinations_next):
                         combos.append([value] + combos_down)
@@ -442,6 +533,7 @@ def create_combinations(combinations: List[Any]) -> List[Any]:
 def build_combinations_for_dynamic_child(
     block,
     execution_partition: str = None,
+    origin_block: Optional[Any] = None,
     **kwargs,
 ):
     """
@@ -464,6 +556,9 @@ def build_combinations_for_dynamic_child(
         [0, 1],
     ]
     """
+    if origin_block is None:
+        origin_block = block
+
     dynamic_counts = []
 
     for upstream_block in block.upstream_blocks:
@@ -488,16 +583,22 @@ def build_combinations_for_dynamic_child(
                 children_created = build_combinations_for_dynamic_child(
                     upstream_block,
                     execution_partition=execution_partition,
+                    origin_block=origin_block,
                 )
                 if is_dynamic:
                     for dynamic_block_index in range(len(children_created)):
                         values, _metadata = get_outputs_for_dynamic_block(
                             upstream_block,
                             execution_partition=execution_partition,
-                            dynamic_block_index=dynamic_block_index
+                            dynamic_block_index=dynamic_block_index,
+                            origin_block=origin_block,
                         )
                         if values is not None:
-                            arr.extend([idx for idx in range(len(values))])
+                            if is_basic_iterable(values) or is_dataframe_or_series(values):
+                                count = len(values)
+                            else:
+                                count = 1
+                            arr.extend([idx for idx in range(count)])
                         else:
                             arr.append(0)
                 else:
@@ -506,8 +607,12 @@ def build_combinations_for_dynamic_child(
                 arr, _metadata = get_outputs_for_dynamic_block(
                     upstream_block,
                     execution_partition=execution_partition,
+                    origin_block=origin_block,
                 )
-            if arr is not None:
+                if arr is not None:
+                    if not is_basic_iterable(arr) and not is_dataframe_or_series(arr):
+                        arr = [0]
+            if arr is not None and hasattr(arr, '__len__') and len(arr) > 0:
                 dynamic_counts.append([idx for idx in range(len(arr))])
             else:
                 dynamic_counts.append([0])
@@ -537,10 +642,12 @@ def build_combinations_for_dynamic_child(
             if is_dynamic_child or is_dynamic:
                 dynamic_block_indexes[upstream_block.uuid] = parent_index
 
-        settings.append(dict(
-            dynamic_block_index=dynamic_block_index,
-            dynamic_block_indexes=dynamic_block_indexes,
-        ))
+        settings.append(
+            dict(
+                dynamic_block_index=dynamic_block_index,
+                dynamic_block_indexes=dynamic_block_indexes,
+            )
+        )
 
     return settings
 
@@ -591,7 +698,9 @@ class DynamicBlockWrapper(BaseDataClass):
     # Upstream blocks that are dynamic blocks.
     upstream_dynamic_blocks: List[DynamicBlockWrapperBase] = field(default_factory=lambda: [])
     # Upstream blocks that are dynamic child blocks.
-    upstream_dynamic_child_blocks: List[DynamicBlockWrapperBase] = field(default_factory=lambda: [])
+    upstream_dynamic_child_blocks: List[DynamicBlockWrapperBase] = field(
+        default_factory=lambda: []
+    )
     # Unique identifier used as a suffix in the block run.
     uuid: str = None
 
@@ -645,8 +754,9 @@ class DynamicBlockWrapper(BaseDataClass):
         if not self.block:
             return False
 
-        return self.block.uuid == self.block_run_block_uuid or \
-            (self.is_replicated() and self.block.uuid_replicated == self.block_run_block_uuid)
+        return self.block.uuid == self.block_run_block_uuid or (
+            self.is_replicated() and self.block.uuid_replicated == self.block_run_block_uuid
+        )
 
     def is_replicated(self) -> bool:
         return DynamicBlockFlag.REPLICATED in (self.flags or []) or is_replicated_block(self.block)
@@ -671,11 +781,15 @@ class DynamicBlockWrapper(BaseDataClass):
         if DynamicBlockFlag.SPAWN_OF_DYNAMIC_CHILD in (self.flags or []):
             return True
 
-        return (self.block or self.block_uuid) and \
-            self.block_run_block_uuid and \
-            ((self.block and self.block.uuid != self.block_run_block_uuid) or
-                (self.block_uuid and self.block_uuid != self.block_run_block_uuid)) and \
-            not self.is_original(include_clone=True)
+        return (
+            (self.block or self.block_uuid)
+            and self.block_run_block_uuid
+            and (
+                (self.block and self.block.uuid != self.block_run_block_uuid)
+                or (self.block_uuid and self.block_uuid != self.block_run_block_uuid)
+            )
+            and not self.is_original(include_clone=True)
+        )
 
     def should_reduce_output(self) -> bool:
         if DynamicBlockFlag.REDUCE_OUTPUT in (self.flags or []):
@@ -764,11 +878,13 @@ class DynamicBlockWrapper(BaseDataClass):
         **kwargs,
     ) -> dict:
         if use_metadata_instructions:
-            return ignore_keys_with_blank_values(dict(
-                clone_original=self.metadata_instructions.clone_original,
-                original=self.metadata_instructions.original.to_dict_base(),
-                upstream=self.metadata_instructions.upstream.to_dict_base(),
-            ))
+            return ignore_keys_with_blank_values(
+                dict(
+                    clone_original=self.metadata_instructions.clone_original,
+                    original=self.metadata_instructions.original.to_dict_base(),
+                    upstream=self.metadata_instructions.upstream.to_dict_base(),
+                )
+            )
 
         data = self.to_dict_base(**kwargs)
         if include_all:
@@ -901,11 +1017,14 @@ def check_all_dynamic_upstreams_completed(
 
         from mage_ai.orchestration.db.models.schedules import BlockRun
 
-        upstreams_done = all([check_all_dynamic_upstreams_completed(
-            b,
-            block_runs=block_runs,
-            execution_partition=execution_partition,
-        ) for b in upstream_block.upstream_blocks])
+        upstreams_done = all([
+            check_all_dynamic_upstreams_completed(
+                b,
+                block_runs=block_runs,
+                execution_partition=execution_partition,
+            )
+            for b in upstream_block.upstream_blocks
+        ])
 
         if not upstreams_done:
             return False
@@ -917,17 +1036,26 @@ def check_all_dynamic_upstreams_completed(
                 execution_partition=execution_partition,
             )
 
-            selected = [br for br in block_runs if pipeline.get_block(
-                br.block_uuid,
-            ).uuid == upstream_block.uuid and br.status == BlockRun.BlockRunStatus.COMPLETED]
+            selected = [
+                br
+                for br in block_runs
+                if pipeline.get_block(
+                    br.block_uuid,
+                ).uuid
+                == upstream_block.uuid
+                and br.status == BlockRun.BlockRunStatus.COMPLETED
+            ]
 
             if len(list(selected)) < len(combos) + 1:
                 return False
 
-        return all([br.status == BlockRun.BlockRunStatus.COMPLETED for br in filter(
-            lambda br: pipeline.get_block(br.block_uuid).uuid == upstream_block.uuid,
-            block_runs,
-        )])
+        return all([
+            br.status == BlockRun.BlockRunStatus.COMPLETED
+            for br in filter(
+                lambda br: pipeline.get_block(br.block_uuid).uuid == upstream_block.uuid,
+                block_runs,
+            )
+        ])
 
     # 1. Are all the upstream blocks for my upstream blocks completed?
     # 2. Then, are my immediate upstream blocks completed?