mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl

mage_ai/orchestration/db/models/schedules.py

@@ -61,6 +61,7 @@ from mage_ai.data_preparation.variable_manager import get_global_variables
 from mage_ai.orchestration.db import db_connection, safe_db_query
 from mage_ai.orchestration.db.errors import ValidationError
 from mage_ai.orchestration.db.models.base import Base, BaseModel, classproperty
+from mage_ai.orchestration.db.models.dynamic.controller import is_ready_to_process_data
 from mage_ai.orchestration.db.models.schedules_project_platform import (
     BlockRunProjectPlatformMixin,
     PipelineRunProjectPlatformMixin,
@@ -105,7 +106,7 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
     event_matchers = relationship(
         'EventMatcher',
         secondary=pipeline_schedule_event_matcher_association_table,
-        back_populates='pipeline_schedules'
+        back_populates='pipeline_schedules',
     )
 
     @classproperty
@@ -123,10 +124,13 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
     @validates('name')
     def validate_name(self, key, value):
         if not value or len(value) == 0:
-            raise ValidationError(f'{key} cannot be empty.', metadata=dict(
-                key=key,
-                value=value,
-            ))
+            raise ValidationError(
+                f'{key} cannot be empty.',
+                metadata=dict(
+                    key=key,
+                    value=value,
+                ),
+            )
 
         return value
 
@@ -140,33 +144,26 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
     @classmethod
     def fetch_latest_pipeline_runs_without_retries(self, ids: List[int]) -> List:
         sub_query = (
-            PipelineRun.
-            select(
+            PipelineRun.select(
                 PipelineRun.id,
                 PipelineRun.execution_date,
                 PipelineRun.started_at,
                 PipelineRun.status,
                 (
-                    func.
-                    row_number().
-                    over(
+                    func.row_number()
+                    .over(
                         partition_by=PipelineRun.execution_date,
-                        order_by=(PipelineRun.started_at.desc(), PipelineRun.id.desc())
-                    ).
-                    label('n_row_number')
-                )
-            ).
-            filter(
+                        order_by=(PipelineRun.started_at.desc(), PipelineRun.id.desc()),
+                    )
+                    .label('n_row_number')
+                ),
+            ).filter(
                 PipelineRun.pipeline_schedule_id.in_(ids),
             )
         ).subquery()
 
-        query = (
-            PipelineRun.
-            select(sub_query).
-            where(
-                text('n_row_number = 1'),
-            )
+        query = PipelineRun.select(sub_query).where(
+            text('n_row_number = 1'),
         )
 
         query.cache = True
@@ -182,7 +179,7 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
         if project_platform_activated():
             return self.pipeline_project_platform
 
-        return Pipeline.get(self.pipeline_uuid)
+        return Pipeline.get(self.pipeline_uuid, repo_path=get_repo_path())
 
     @property
     def pipeline_in_progress_runs_count(self) -> int:
@@ -193,12 +190,10 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
             PipelineRun.select(func.count(PipelineRun.id))
             .filter(
                 PipelineRun.pipeline_schedule_id == self.id,
-                PipelineRun.status.in_(
-                    [
-                        PipelineRun.PipelineRunStatus.INITIAL,
-                        PipelineRun.PipelineRunStatus.RUNNING,
-                    ]
-                ),
+                PipelineRun.status.in_([
+                    PipelineRun.PipelineRunStatus.INITIAL,
+                    PipelineRun.PipelineRunStatus.RUNNING,
+                ]),
                 (coalesce(PipelineRun.passed_sla, False).is_(False)),
             )
             .scalar()
@@ -227,8 +222,7 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
 
     @validates('schedule_interval')
     def validate_schedule_interval(self, key, schedule_interval):
-        if schedule_interval and schedule_interval not in \
-                [e.value for e in ScheduleInterval]:
+        if schedule_interval and schedule_interval not in [e.value for e in ScheduleInterval]:
             if not croniter.is_valid(schedule_interval):
                 raise ValueError('Cron expression is invalid.')
 
@@ -258,23 +252,22 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
     @property
     def tag_associations(self):
         return (
-            TagAssociation.
-            select(
+            TagAssociation.select(
                 TagAssociation.id,
                 TagAssociation.tag_id,
                 TagAssociation.taggable_id,
                 TagAssociation.taggable_type,
                 Tag.name,
-            ).
-            join(
+            )
+            .join(
                 Tag,
                 Tag.id == TagAssociation.tag_id,
-            ).
-            filter(
+            )
+            .filter(
                 TagAssociation.taggable_id == self.id,
                 TagAssociation.taggable_type == self.__class__.__name__,
-            ).
-            all()
+            )
+            .all()
         )
 
     @classmethod
@@ -325,8 +318,10 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
                 continue
 
             last_enabled_at = trigger_config.last_enabled_at
-            if trigger_config.status == ScheduleStatus.ACTIVE and \
-                    trigger_config.last_enabled_at is None:
+            if (
+                trigger_config.status == ScheduleStatus.ACTIVE
+                and trigger_config.last_enabled_at is None
+            ):
                 last_enabled_at = datetime.now(tz=pytz.UTC)
                 trigger_config.last_enabled_at = last_enabled_at
                 add_or_update_trigger_for_pipeline_and_persist(
@@ -335,6 +330,7 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
                 )
 
             kwargs = dict(
+                description=trigger_config.description,
                 last_enabled_at=last_enabled_at,
                 name=trigger_config.name,
                 pipeline_uuid=trigger_config.pipeline_uuid,
@@ -345,12 +341,14 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
                 sla=trigger_config.sla,
                 start_time=trigger_config.start_time,
                 status=trigger_config.status,
+                token=trigger_config.token,
                 variables=trigger_config.variables,
             )
 
             if existing_trigger:
                 if any([
                     existing_trigger.last_enabled_at != kwargs.get('last_enabled_at'),
+                    existing_trigger.description != kwargs.get('description'),
                     existing_trigger.name != kwargs.get('name'),
                     existing_trigger.pipeline_uuid != kwargs.get('pipeline_uuid'),
                     existing_trigger.repo_path != kwargs.get('repo_path'),
@@ -360,13 +358,15 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
                     existing_trigger.sla != kwargs.get('sla'),
                     existing_trigger.start_time != kwargs.get('start_time'),
                     existing_trigger.status != kwargs.get('status'),
+                    existing_trigger.token != kwargs.get('token'),
                     existing_trigger.variables != kwargs.get('variables'),
                 ]):
-                    if existing_trigger.token is None:
+                    if existing_trigger.token is None and kwargs.get('token') is None:
                         kwargs['token'] = uuid.uuid4().hex
                     existing_trigger.update(**kwargs)
             else:
-                kwargs['token'] = uuid.uuid4().hex
+                if kwargs.get('token') is None:
+                    kwargs['token'] = uuid.uuid4().hex
                 triggers_to_create.append(kwargs)
 
         db_connection.session.bulk_save_objects(
@@ -424,16 +424,19 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
         if self.schedule_interval is None:
             return None
 
-        if self.schedule_interval == ScheduleInterval.ONCE or \
-                self.schedule_interval == ScheduleInterval.ALWAYS_ON:
+        if (
+            self.schedule_interval == ScheduleInterval.ONCE
+            or self.schedule_interval == ScheduleInterval.ALWAYS_ON
+        ):
             current_execution_date = now
         elif self.schedule_interval == ScheduleInterval.DAILY:
             current_execution_date = now.replace(second=0, microsecond=0, minute=0, hour=0)
         elif self.schedule_interval == ScheduleInterval.HOURLY:
             current_execution_date = now.replace(second=0, microsecond=0, minute=0)
         elif self.schedule_interval == ScheduleInterval.WEEKLY:
-            current_execution_date = now.replace(second=0, microsecond=0, minute=0, hour=0) - \
-                timedelta(days=now.weekday())
+            current_execution_date = now.replace(
+                second=0, microsecond=0, minute=0, hour=0
+            ) - timedelta(days=now.weekday())
         elif self.schedule_interval == ScheduleInterval.MONTHLY:
             current_execution_date = now.replace(second=0, microsecond=0, minute=0, hour=0, day=1)
         else:
@@ -480,8 +483,10 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
         if current_execution_date is None:
             return None
 
-        if self.schedule_interval == ScheduleInterval.ONCE or \
-                self.schedule_interval == ScheduleInterval.ALWAYS_ON:
+        if (
+            self.schedule_interval == ScheduleInterval.ONCE
+            or self.schedule_interval == ScheduleInterval.ALWAYS_ON
+        ):
             pass
         elif self.schedule_interval == ScheduleInterval.DAILY:
             next_execution_date = current_execution_date + timedelta(days=1)
@@ -502,6 +507,7 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
         self,
         previous_runtimes: List[int] = None,
         pipeline: Pipeline = None,
+        repo_config=None,
     ) -> bool:
         """
         Determine whether a pipeline schedule should be executed based on its configuration and
@@ -532,13 +538,20 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
         if self.status != ScheduleStatus.ACTIVE:
             return False
 
-        if not self.landing_time_enabled() and \
-                self.start_time is not None and \
-                compare(now, self.start_time.replace(tzinfo=pytz.UTC)) == -1:
+        if (
+            not self.landing_time_enabled()
+            and self.start_time is not None
+            and compare(now, self.start_time.replace(tzinfo=pytz.UTC)) == -1
+        ):
             return False
 
         try:
-            Pipeline.get(self.pipeline_uuid)
+            if pipeline is None:
+                pipeline = Pipeline.get(
+                    self.pipeline_uuid,
+                    repo_path=get_repo_path(),
+                    repo_config=repo_config,
+                )
         except Exception:
             print(
                 f'[WARNING] Pipeline {self.pipeline_uuid} cannot be found '
@@ -550,7 +563,7 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
             pipeline_run_count = self.pipeline_runs_count
             if pipeline_run_count == 0:
                 return True
-            executor_count = self.pipeline.executor_count
+            executor_count = pipeline.executor_count
             # Used by streaming pipeline to launch multiple executors
             if executor_count > 1 and pipeline_run_count < executor_count:
                 return True
@@ -585,16 +598,21 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
                 the initial pipeline run.
                 """
                 create_initial_pipeline_run = self.get_settings().create_initial_pipeline_run
-                avoid_initial_pipeline_run = compare(
-                    current_execution_date,
-                    self.last_enabled_at,
-                ) == -1 if not create_initial_pipeline_run else False
+                avoid_initial_pipeline_run = (
+                    compare(
+                        current_execution_date,
+                        self.last_enabled_at,
+                    )
+                    == -1
+                    if not create_initial_pipeline_run
+                    else False
+                )
 
             # If the execution date is before start time or date pipeline schedule
             # was last enabled, don't schedule it.
             if (
-                self.start_time is not None and
-                compare(current_execution_date, self.start_time.replace(tzinfo=pytz.UTC)) == -1
+                self.start_time is not None
+                and compare(current_execution_date, self.start_time.replace(tzinfo=pytz.UTC)) == -1
             ) or avoid_initial_pipeline_run:
                 return False
 
@@ -648,27 +666,17 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
         pipeline_run=None,
         sample_size: int = None,
     ):
-        pipeline_runs = (
-            PipelineRun.
-            query.
-            filter(
-                PipelineRun.pipeline_schedule_id == self.id,
-                PipelineRun.status == PipelineRun.PipelineRunStatus.COMPLETED,
-            )
+        pipeline_runs = PipelineRun.query.filter(
+            PipelineRun.pipeline_schedule_id == self.id,
+            PipelineRun.status == PipelineRun.PipelineRunStatus.COMPLETED,
         )
 
         if pipeline_run:
-            pipeline_runs = (
-                pipeline_runs.
-                filter(
-                    PipelineRun.id != pipeline_run.id,
-                )
+            pipeline_runs = pipeline_runs.filter(
+                PipelineRun.id != pipeline_run.id,
             )
 
-        pipeline_runs = (
-            pipeline_runs.
-            order_by(PipelineRun.execution_date.desc())
-        )
+        pipeline_runs = pipeline_runs.order_by(PipelineRun.execution_date.desc())
 
         if sample_size:
             pipeline_runs = pipeline_runs.limit(sample_size)
@@ -701,15 +709,11 @@ class PipelineSchedule(PipelineScheduleProjectPlatformMixin, BaseModel):
             )
 
             for pr in pipeline_runs:
-                runtime = (
-                    pr.completed_at - pr.created_at
-                ).total_seconds()
+                runtime = (pr.completed_at - pr.created_at).total_seconds()
                 previous_runtimes.append(runtime)
 
         if pipeline_run:
-            runtime = (
-                pipeline_run.completed_at - pipeline_run.created_at
-            ).total_seconds()
+            runtime = (pipeline_run.completed_at - pipeline_run.created_at).total_seconds()
             previous_runtimes = [runtime] + previous_runtimes
 
         return previous_runtimes[:sample_size_to_use]
@@ -761,8 +765,10 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
         super().__init__(**kwargs)
 
     def __repr__(self):
-        return f'PipelineRun(id={self.id}, pipeline_uuid={self.pipeline_uuid},'\
-               f' execution_date={self.execution_date})'
+        return (
+            f'PipelineRun(id={self.id}, pipeline_uuid={self.pipeline_uuid},'
+            f' execution_date={self.execution_date})'
+        )
 
     @property
     def block_runs_count(self) -> int:
@@ -774,36 +780,35 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
 
     @property
     def execution_partition(self) -> str:
-        if self.variables and \
-                isinstance(self.variables, dict) and \
-                self.variables.get('execution_partition'):
-
+        if (
+            self.variables
+            and isinstance(self.variables, dict)
+            and self.variables.get('execution_partition')
+        ):
             return self.variables.get('execution_partition')
         if self.execution_date is None:
             return str(self.pipeline_schedule_id)
         else:
             return '/'.join([
-                        str(self.pipeline_schedule_id),
-                        self.execution_date.strftime(format='%Y%m%dT%H%M%S'),
-                    ])
+                str(self.pipeline_schedule_id),
+                self.execution_date.strftime(format='%Y%m%dT%H%M%S'),
+            ])
 
     @property
     def initial_block_runs(self) -> List['BlockRun']:
-        return [b for b in self.block_runs
-                if b.status == BlockRun.BlockRunStatus.INITIAL]
+        return [b for b in self.block_runs if b.status == BlockRun.BlockRunStatus.INITIAL]
 
     @property
     def running_block_runs(self) -> List['BlockRun']:
-        return [
-            b for b in self.block_runs
-            if b.status == BlockRun.BlockRunStatus.RUNNING
-        ]
+        return [b for b in self.block_runs if b.status == BlockRun.BlockRunStatus.RUNNING]
 
     @property
     def queued_or_running_block_runs(self) -> List['BlockRun']:
         return [
-            b for b in self.block_runs
-            if b.status in [
+            b
+            for b in self.block_runs
+            if b.status
+            in [
                 BlockRun.BlockRunStatus.QUEUED,
                 BlockRun.BlockRunStatus.RUNNING,
             ]
@@ -811,25 +816,22 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
 
     @property
     def completed_block_runs(self) -> List['BlockRun']:
-        return [b for b in self.block_runs
-                if b.status == BlockRun.BlockRunStatus.COMPLETED]
+        return [b for b in self.block_runs if b.status == BlockRun.BlockRunStatus.COMPLETED]
 
     @property
     def failed_block_runs(self) -> List['BlockRun']:
-        return [b for b in self.block_runs
-                if b.status == BlockRun.BlockRunStatus.FAILED]
+        return [b for b in self.block_runs if b.status == BlockRun.BlockRunStatus.FAILED]
 
     @property
     def pipeline(self) -> 'Pipeline':
         if project_platform_activated():
             return self.pipeline_project_platform
 
-        return Pipeline.get(self.pipeline_uuid)
+        return Pipeline.get(self.pipeline_uuid, repo_path=get_repo_path())
 
     @property
     def pipeline_type(self) -> PipelineType:
-        pipeline = Pipeline.get(self.pipeline_uuid, check_if_exists=True)
-
+        pipeline = self.pipeline
         return pipeline.type if pipeline is not None else None
 
     @property
@@ -862,13 +864,9 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
         pipeline_schedule_id: int = None,
         sample_size: int = None,
     ):
-        pipeline_runs = (
-            self.
-            query.
-            filter(
-                self.pipeline_uuid == pipeline_uuid,
-                self.status == self.PipelineRunStatus.COMPLETED,
-            )
+        pipeline_runs = self.query.filter(
+            self.pipeline_uuid == pipeline_uuid,
+            self.status == self.PipelineRunStatus.COMPLETED,
         )
 
         if pipeline_run_id is not None:
@@ -985,7 +983,7 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                     if block and block.replicated_block:
                         arr.add(block.uuid)
 
-            return arr
+            return list(arr)
 
         completed_block_uuids = _build_block_uuids(self.completed_block_runs)
         finished_block_uuids = _build_block_uuids(self.block_runs)
@@ -1042,6 +1040,13 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
 
         executable_block_runs = []
         for block_run in self.initial_block_runs:
+            block = pipeline.get_block(block_run.block_uuid)
+
+            if block is not None and block.is_dynamic_streaming:
+                if is_ready_to_process_data(pipeline, block_run, block_runs_all):
+                    executable_block_runs.append(block_run)
+                continue
+
             completed = False
             upstream_block_uuids_override = None
 
@@ -1065,13 +1070,9 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                 elif metrics.get('upstream_blocks'):
                     upstream_block_uuids_override = metrics.get('upstream_blocks') or None
 
-            block = pipeline.get_block(block_run.block_uuid)
-
             if block and is_dynamic_block_child(block):
                 if check_all_dynamic_upstreams_completed(
-                    block,
-                    block_runs_all,
-                    execution_partition=self.execution_partition
+                    block, block_runs_all, execution_partition=self.execution_partition
                 ):
                     completed = True
                 else:
@@ -1086,11 +1087,9 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                         uuids_to_check.append(upstream_block_uuid)
 
                 if allow_blocks_to_fail:
-                    completed = all(uuid in finished_block_uuids
-                                    for uuid in uuids_to_check)
+                    completed = all(uuid in finished_block_uuids for uuid in uuids_to_check)
                 else:
-                    completed = all(uuid in completed_block_uuids
-                                    for uuid in uuids_to_check)
+                    completed = all(uuid in completed_block_uuids for uuid in uuids_to_check)
             else:
                 metrics = block_run.metrics
 
@@ -1126,15 +1125,17 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                             upstream_block_uuids_override = [
                                 controller_block_uuid,
                             ]
-                    elif metrics.get('controller') and \
-                            metrics.get('child') and not \
-                            metrics.get('run_in_parallel') and \
-                            metrics.get('upstream_block_uuids'):
-
+                    elif (
+                        metrics.get('controller')
+                        and metrics.get('child')
+                        and not metrics.get('run_in_parallel')
+                        and metrics.get('upstream_block_uuids')
+                    ):
                         upstream_block_uuids_override = metrics.get('upstream_block_uuids')
                 elif metrics and metrics.get('dynamic_upstream_block_uuids'):
-                    upstream_block_uuids_override = \
+                    upstream_block_uuids_override = (
                         metrics.get('dynamic_upstream_block_uuids') or []
+                    )
 
                 incomplete = False
                 if block:
@@ -1164,15 +1165,20 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                             up_uuids.append(upstream_block.uuid)
 
                         if len(up_uuids_dynamic_children) >= 1:
-                            upstream_block_uuids_override = \
-                                (upstream_block_uuids_override or []) + \
-                                up_uuids + up_uuids_dynamic_children
-
-                completed = not incomplete and block is not None and \
-                    block.all_upstream_blocks_completed(
+                            upstream_block_uuids_override = (
+                                (upstream_block_uuids_override or [])
+                                + up_uuids
+                                + up_uuids_dynamic_children
+                            )
+
+                completed = (
+                    not incomplete
+                    and block is not None
+                    and block.all_upstream_blocks_completed(
                         completed_block_uuids,
                         upstream_block_uuids_override,
                     )
+                )
 
             if completed:
                 executable_block_runs.append(block_run)
@@ -1203,15 +1209,19 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
         pipeline = self.pipeline
 
         failed_block_uuids = set(
-            b.block_uuid for b in self.block_runs
-            if b.status in [
+            b.block_uuid
+            for b in self.block_runs
+            if b.status
+            in [
                 BlockRun.BlockRunStatus.UPSTREAM_FAILED,
                 BlockRun.BlockRunStatus.FAILED,
             ]
         )
         condition_failed_block_uuids = set(
-            b.block_uuid for b in self.block_runs
-            if b.status in [
+            b.block_uuid
+            for b in self.block_runs
+            if b.status
+            in [
                 BlockRun.BlockRunStatus.CONDITION_FAILED,
             ]
         )
@@ -1237,13 +1247,13 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                         upstream_block_uuids = block.upstream_block_uuids
                         # If an upstream dynamic child fails, the block_run.block_uuid won’t match.
                         if block_run.block_uuid == block.uuid:
-                            block_uuids = list(block_uuids) + [pipeline.get_block(
-                                block_inner,
-                            ).uuid for block_inner in block_uuids]
-                if any(
-                    b in block_uuids
-                    for b in upstream_block_uuids
-                ):
+                            block_uuids = list(block_uuids) + [
+                                pipeline.get_block(
+                                    block_inner,
+                                ).uuid
+                                for block_inner in block_uuids
+                            ]
+                if any(b in block_uuids for b in upstream_block_uuids):
                     block_run.update(status=status)
                     updated_status = True
 
@@ -1299,9 +1309,9 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
     @classmethod
     @safe_db_query
     def batch_update_status(self, pipeline_run_ids: List[int], status):
-        PipelineRun.query.filter(PipelineRun.id.in_(pipeline_run_ids)).update({
-            PipelineRun.status: status
-        }, synchronize_session=False)
+        PipelineRun.query.filter(PipelineRun.id.in_(pipeline_run_ids)).update(
+            {PipelineRun.status: status}, synchronize_session=False
+        )
         db_connection.session.commit()
 
     @classmethod
@@ -1365,9 +1375,12 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
         )
 
         from mage_ai.usage_statistics.logger import UsageStatisticLogger
-        asyncio.run(UsageStatisticLogger().pipeline_runs_impression(
-            lambda: self.query.filter(self.status == self.PipelineRunStatus.COMPLETED).count(),
-        ))
+
+        asyncio.run(
+            UsageStatisticLogger().pipeline_runs_impression(
+                lambda: self.query.filter(self.status == self.PipelineRunStatus.COMPLETED).count(),
+            )
+        )
 
     @safe_db_query
     def create_block_run(
@@ -1407,11 +1420,11 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
             if block.replicated_block:
                 replicated_block = pipeline.get_block(block.replicated_block)
                 if replicated_block:
-                    block_uuid = f'{block.uuid}:{replicated_block.uuid}'
+                    block_uuid = block.uuid_replicated
                 else:
                     raise Exception(
-                        f'Replicated block {block.replicated_block} ' +
-                        f'does not exist in pipeline {pipeline.uuid}.',
+                        f'Replicated block {block.replicated_block} '
+                        + f'does not exist in pipeline {pipeline.uuid}.',
                     )
             elif block.is_data_integration():
                 controller_uuid = block.controller_uuid
@@ -1422,10 +1435,12 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
 
                 block_arr.append((
                     controller_uuid,
-                    dict(metrics=dict(
-                        controller=1,
-                        original_block_uuid=block_uuid,
-                    )),
+                    dict(
+                        metrics=dict(
+                            controller=1,
+                            original_block_uuid=block_uuid,
+                        )
+                    ),
                 ))
 
             flags = []
@@ -1448,9 +1463,11 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                 flags.append(DynamicBlockFlag.REDUCE_OUTPUT)
 
             if len(flags) >= 1:
-                create_options['metrics'] = dict(metadata=dict(
-                    flags=flags,
-                ))
+                create_options['metrics'] = dict(
+                    metadata=dict(
+                        flags=flags,
+                    )
+                )
 
             block_arr.append((block_uuid, create_options))
 
@@ -1467,10 +1484,7 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
         return [self.create_block_run(block_uuid, **options) for block_uuid, options in block_arr]
 
     def any_blocks_failed(self) -> bool:
-        return any(
-            b.status == BlockRun.BlockRunStatus.FAILED
-            for b in self.block_runs
-        )
+        return any(b.status == BlockRun.BlockRunStatus.FAILED for b in self.block_runs)
 
     def all_blocks_completed(self, include_failed_blocks: bool = False) -> bool:
         statuses = [
@@ -1518,16 +1532,19 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
         variables['event'] = merge_dict(variables.get('event', {}), event_variables)
         variables['execution_date'] = self.execution_date
         variables['execution_partition'] = self.execution_partition
+        variables['pipeline_run_id'] = self.id
 
         interval_end_datetime = variables.get('interval_end_datetime')
         interval_seconds = variables.get('interval_seconds')
         interval_start_datetime = variables.get('interval_start_datetime')
         interval_start_datetime_previous = variables.get('interval_start_datetime_previous')
 
-        if interval_end_datetime or \
-                interval_seconds or \
-                interval_start_datetime or \
-                interval_start_datetime_previous:
+        if (
+            interval_end_datetime
+            or interval_seconds
+            or interval_start_datetime
+            or interval_start_datetime_previous
+        ):
             if interval_end_datetime:
                 try:
                     variables['interval_end_datetime'] = dateutil.parser.parse(
@@ -1585,9 +1602,7 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
                     # to call get_next again to go back to the original state
                     cron_itr.get_next()
                     interval_end_datetime = cron_itr.get_next(datetime)
-                    interval_seconds = (
-                        interval_end_datetime.timestamp() - current.timestamp()
-                    )
+                    interval_seconds = interval_end_datetime.timestamp() - current.timestamp()
                 except Exception:
                     pass
 
@@ -1633,7 +1648,7 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
 
     @property
     def logs(self):
-        pipeline = Pipeline.get(self.pipeline_run.pipeline_uuid)
+        pipeline = self.pipeline_run.pipeline_schedule.pipeline
         return LoggerManagerFactory.get_logger_manager(
             pipeline_uuid=pipeline.uuid,
             block_uuid=clean_name(self.block_uuid),
@@ -1645,7 +1660,7 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
         if project_platform_activated():
             return await self.logs_async_project_platform(repo_path=repo_path)
 
-        pipeline = await Pipeline.get_async(self.pipeline_run.pipeline_uuid)
+        pipeline = await Pipeline.get_async(self.pipeline_run.pipeline_uuid, repo_path=repo_path)
         return await LoggerManagerFactory.get_logger_manager(
             pipeline_uuid=pipeline.uuid,
             block_uuid=clean_name(self.block_uuid),
@@ -1656,17 +1671,15 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
     @classmethod
     @safe_db_query
     def batch_update_status(self, block_run_ids: List[int], status):
-        BlockRun.query.filter(BlockRun.id.in_(block_run_ids)).update({
-            BlockRun.status: status
-        }, synchronize_session=False)
+        BlockRun.query.filter(BlockRun.id.in_(block_run_ids)).update(
+            {BlockRun.status: status}, synchronize_session=False
+        )
         db_connection.session.commit()
 
     @classmethod
     @safe_db_query
     def batch_delete(self, block_run_ids: List[int]):
-        BlockRun.query.filter(BlockRun.id.in_(block_run_ids)).delete(
-            synchronize_session=False
-        )
+        BlockRun.query.filter(BlockRun.id.in_(block_run_ids)).delete(synchronize_session=False)
         db_connection.session.commit()
 
     @classmethod
@@ -1680,8 +1693,13 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
             return block_runs[0]
         return None
 
-    def get_outputs(self, sample_count: int = None) -> List[Dict]:
+    def get_outputs(
+        self,
+        exclude_blank_variable_uuids: bool = False,
+        sample_count: int = None,
+    ) -> List[Dict]:
         pipeline = Pipeline.get(self.pipeline_run.pipeline_uuid)
+
         block = pipeline.get_block(self.block_uuid)
         block_uuid = self.block_uuid
 
@@ -1755,6 +1773,7 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
                 ]
 
         return block.get_outputs(
+            exclude_blank_variable_uuids=exclude_blank_variable_uuids,
             execution_partition=self.pipeline_run.execution_partition,
             sample_count=sample_count,
             block_uuid=block_uuid,
@@ -1783,9 +1802,7 @@ class EventMatcher(BaseModel):
     @classmethod
     def active_event_matchers(self) -> List['EventMatcher']:
         return self.query.filter(
-            EventMatcher.pipeline_schedules.any(
-                PipelineSchedule.status == ScheduleStatus.ACTIVE
-            )
+            EventMatcher.pipeline_schedules.any(PipelineSchedule.status == ScheduleStatus.ACTIVE)
         ).all()
 
     @classmethod
@@ -1846,8 +1863,7 @@ class EventMatcher(BaseModel):
         return [t[0] for t in event_matchers_and_pipeline_schedule_ids]
 
     def active_pipeline_schedules(self) -> List[PipelineSchedule]:
-        return [p for p in self.pipeline_schedules
-                if p.status == ScheduleStatus.ACTIVE]
+        return [p for p in self.pipeline_schedules if p.status == ScheduleStatus.ACTIVE]
 
     def match(self, config: Dict) -> bool:
         def __match_dict(sub_pattern, sub_config):
@@ -1863,6 +1879,7 @@ class EventMatcher(BaseModel):
                 elif not __match_dict(v, sub_config[k]):
                     return False
             return True
+
         return __match_dict(self.pattern, config)
 
 
@@ -1913,3 +1930,24 @@ class Backfill(BaseModel):
                 Backfill.pipeline_schedule_id.in_(pipeline_schedule_ids),
             )
         return []
+
+    @property
+    def pipeline_run_status_counts(self) -> Dict:
+        status_counts = dict()
+        execution_dates_counted = set()
+
+        # Sort the pipeline runs by id in reverse order so the first pipeline run
+        # checked is the latest pipeline run created for a given execution date.
+        pipeline_runs_sorted = sorted(
+            self.pipeline_runs,
+            key=lambda pr: (pr.execution_date, pr.id),
+            reverse=True,
+        )
+
+        for pr in pipeline_runs_sorted:
+            # Only count a pipeline run once per execution date
+            if pr.execution_date not in execution_dates_counted:
+                status_counts[pr.status] = (status_counts.get(pr.status) or 0) + 1
+                execution_dates_counted.add(pr.execution_date)
+
+        return status_counts