metricflow 0.207.0.dev1__tar.gz → 0.207.0.dev2__tar.gz

{metricflow-0.207.0.dev1 → metricflow-0.207.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metricflow
-Version: 0.207.0.dev1
+Version: 0.207.0.dev2
 Summary: Translates a simple metric definition into reusable SQL and executes it against the SQL engine of your choice.
 Project-URL: Documentation, https://docs.getdbt.com/docs/build/about-metricflow
 Project-URL: Source Code, https://github.com/dbt-labs/metricflow
@@ -18,11 +18,11 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: <3.13,>=3.8
 Requires-Dist: dbt-semantic-interfaces<2.0.0,>=0.6.1
-Requires-Dist: dbt-semantic-interfaces==0.6.1
+Requires-Dist: dbt-semantic-interfaces==0.6.5
 Requires-Dist: graphviz<0.21,>=0.18.2
 Requires-Dist: jinja2>=3.1.3
 Requires-Dist: more-itertools<10.2.0,>=8.10.0
-Requires-Dist: pydantic<1.11.0,>=1.10.0
+Requires-Dist: pydantic<3.0,>=1.10.0
 Requires-Dist: python-dateutil<2.10.0,>=2.9.0
 Requires-Dist: rapidfuzz<4.0,>=3.0
 Requires-Dist: tabulate>=0.8.9

{metricflow-0.207.0.dev1 → metricflow-0.207.0.dev2}/metricflow/dataflow/builder/dataflow_plan_builder.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import time
 from dataclasses import dataclass
-from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
+from typing import Dict, FrozenSet, List, Optional, Sequence, Set, Tuple, Union
 
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
 from dbt_semantic_interfaces.implementations.metric import PydanticMetricTimeWindow
@@ -82,7 +82,10 @@ from metricflow.dataflow.nodes.where_filter import WhereConstraintNode
 from metricflow.dataflow.nodes.window_reaggregation_node import WindowReaggregationNode
 from metricflow.dataflow.nodes.write_to_data_table import WriteToResultDataTableNode
 from metricflow.dataflow.nodes.write_to_table import WriteToResultTableNode
-from metricflow.dataflow.optimizer.dataflow_plan_optimizer import DataflowPlanOptimizer
+from metricflow.dataflow.optimizer.dataflow_optimizer_factory import (
+    DataflowPlanOptimization,
+    DataflowPlanOptimizerFactory,
+)
 from metricflow.dataset.dataset_classes import DataSet
 from metricflow.plan_conversion.node_processor import (
     PredicateInputType,
@@ -143,7 +146,7 @@ class DataflowPlanBuilder:
         query_spec: MetricFlowQuerySpec,
         output_sql_table: Optional[SqlTable] = None,
         output_selection_specs: Optional[InstanceSpecSet] = None,
-        optimizers: Sequence[DataflowPlanOptimizer] = (),
+        optimizations: FrozenSet[DataflowPlanOptimization] = frozenset(),
     ) -> DataflowPlan:
         """Generate a plan for reading the results of a query with the given spec into a data_table or table."""
         # Workaround for a Pycharm type inspection issue with decorators.
@@ -152,7 +155,7 @@ class DataflowPlanBuilder:
             query_spec=query_spec,
             output_sql_table=output_sql_table,
             output_selection_specs=output_selection_specs,
-            optimizers=optimizers,
+            optimizations=optimizations,
         )
 
     def _build_query_output_node(
@@ -184,7 +187,11 @@ class DataflowPlanBuilder:
             )
         )
 
-        predicate_pushdown_state = PredicatePushdownState(time_range_constraint=query_spec.time_range_constraint)
+        predicate_pushdown_state = PredicatePushdownState(
+            time_range_constraint=query_spec.time_range_constraint,
+            where_filter_specs=(),
+            pushdown_enabled_types=frozenset({PredicateInputType.TIME_RANGE_CONSTRAINT}),
+        )
 
         return self._build_metrics_output_node(
             metric_specs=tuple(
@@ -206,7 +213,7 @@ class DataflowPlanBuilder:
         query_spec: MetricFlowQuerySpec,
         output_sql_table: Optional[SqlTable],
         output_selection_specs: Optional[InstanceSpecSet],
-        optimizers: Sequence[DataflowPlanOptimizer],
+        optimizations: FrozenSet[DataflowPlanOptimization],
     ) -> DataflowPlan:
         metrics_output_node = self._build_query_output_node(query_spec=query_spec)
 
@@ -220,7 +227,11 @@ class DataflowPlanBuilder:
 
         plan_id = DagId.from_id_prefix(StaticIdPrefix.DATAFLOW_PLAN_PREFIX)
         plan = DataflowPlan(sink_nodes=[sink_node], plan_id=plan_id)
-        for optimizer in optimizers:
+        return self._optimize_plan(plan, optimizations)
+
+    def _optimize_plan(self, plan: DataflowPlan, optimizations: FrozenSet[DataflowPlanOptimization]) -> DataflowPlan:
+        optimizer_factory = DataflowPlanOptimizerFactory(self._node_data_set_resolver)
+        for optimizer in optimizer_factory.get_optimizers(optimizations):
             logger.info(f"Applying {optimizer.__class__.__name__}")
             try:
                 plan = optimizer.optimize(plan)
@@ -251,6 +262,7 @@ class DataflowPlanBuilder:
         disabled_pushdown_state = PredicatePushdownState.with_pushdown_disabled()
         time_range_only_pushdown_state = PredicatePushdownState(
             time_range_constraint=predicate_pushdown_state.time_range_constraint,
+            where_filter_specs=tuple(),
             pushdown_enabled_types=frozenset([PredicateInputType.TIME_RANGE_CONSTRAINT]),
         )
 
@@ -511,7 +523,6 @@ class DataflowPlanBuilder:
             ),
             descendent_filter_specs=metric_spec.filter_specs,
         )
-
         logger.info(
             f"For\n{indent(mf_pformat(metric_spec))}"
             f"\nneeded measure is:"
@@ -568,6 +579,9 @@ class DataflowPlanBuilder:
 
             # If metric is offset, we'll apply where constraint after offset to avoid removing values
             # unexpectedly. Time constraint will be applied by INNER JOINing to time spine.
+            # We may consider encapsulating this in pushdown state later, but as of this moment pushdown
+            # is about post-join to pre-join for dimension access, and relies on the builder to collect
+            # predicates from query and metric specs and make them available at measure level.
             if not metric_spec.has_time_offset:
                 filter_specs.extend(metric_spec.filter_specs)
 
@@ -623,8 +637,7 @@ class DataflowPlanBuilder:
             )
 
             if len(metric_spec.filter_specs) > 0:
-                merged_where_filter = WhereFilterSpec.merge_iterable(metric_spec.filter_specs)
-                output_node = WhereConstraintNode(parent_node=output_node, where_constraint=merged_where_filter)
+                output_node = WhereConstraintNode(parent_node=output_node, where_specs=metric_spec.filter_specs)
             if not extraneous_linkable_specs.is_subset_of(queried_linkable_specs):
                 output_node = FilterElementsNode(
                     parent_node=output_node,
@@ -723,17 +736,21 @@ class DataflowPlanBuilder:
 
         return CombineAggregatedOutputsNode(parent_nodes=output_nodes)
 
-    def build_plan_for_distinct_values(self, query_spec: MetricFlowQuerySpec) -> DataflowPlan:
+    def build_plan_for_distinct_values(
+        self, query_spec: MetricFlowQuerySpec, optimizations: FrozenSet[DataflowPlanOptimization] = frozenset()
+    ) -> DataflowPlan:
         """Generate a plan that would get the distinct values of a linkable instance.
 
         e.g. distinct listing__country_latest for bookings by listing__country_latest
         """
         # Workaround for a Pycharm type inspection issue with decorators.
         # noinspection PyArgumentList
-        return self._build_plan_for_distinct_values(query_spec)
+        return self._build_plan_for_distinct_values(query_spec, optimizations=optimizations)
 
     @log_runtime()
-    def _build_plan_for_distinct_values(self, query_spec: MetricFlowQuerySpec) -> DataflowPlan:
+    def _build_plan_for_distinct_values(
+        self, query_spec: MetricFlowQuerySpec, optimizations: FrozenSet[DataflowPlanOptimization]
+    ) -> DataflowPlan:
         assert not query_spec.metric_specs, "Can't build distinct values plan with metrics."
         query_level_filter_specs: Sequence[WhereFilterSpec] = ()
         if query_spec.filter_intersection is not None and len(query_spec.filter_intersection.where_filters) > 0:
@@ -751,7 +768,9 @@ class DataflowPlanBuilder:
         required_linkable_specs, _ = self.__get_required_and_extraneous_linkable_specs(
             queried_linkable_specs=query_spec.linkable_specs, filter_specs=query_level_filter_specs
         )
-        predicate_pushdown_state = PredicatePushdownState(time_range_constraint=query_spec.time_range_constraint)
+        predicate_pushdown_state = PredicatePushdownState(
+            time_range_constraint=query_spec.time_range_constraint, where_filter_specs=query_level_filter_specs
+        )
         dataflow_recipe = self._find_dataflow_recipe(
             linkable_spec_set=required_linkable_specs, predicate_pushdown_state=predicate_pushdown_state
         )
@@ -763,9 +782,7 @@ class DataflowPlanBuilder:
             output_node = JoinOnEntitiesNode(left_node=output_node, join_targets=dataflow_recipe.join_targets)
 
         if len(query_level_filter_specs) > 0:
-            output_node = WhereConstraintNode(
-                parent_node=output_node, where_constraint=WhereFilterSpec.merge_iterable(query_level_filter_specs)
-            )
+            output_node = WhereConstraintNode(parent_node=output_node, where_specs=query_level_filter_specs)
         if query_spec.time_range_constraint:
             output_node = ConstrainTimeRangeNode(
                 parent_node=output_node, time_range_constraint=query_spec.time_range_constraint
@@ -782,7 +799,8 @@ class DataflowPlanBuilder:
             parent_node=output_node, order_by_specs=query_spec.order_by_specs, limit=query_spec.limit
         )
 
-        return DataflowPlan(sink_nodes=[sink_node])
+        plan = DataflowPlan(sink_nodes=[sink_node])
+        return self._optimize_plan(plan, optimizations)
 
     @staticmethod
     def build_sink_node(
@@ -954,7 +972,14 @@ class DataflowPlanBuilder:
             node_data_set_resolver=self._node_data_set_resolver,
         )
 
-        if predicate_pushdown_state.has_pushdown_potential:
+        if predicate_pushdown_state.has_pushdown_potential and default_join_type is not SqlJoinType.FULL_OUTER:
+            # TODO: encapsulate join type and distinct values state and eventually move this to a DataflowPlanOptimizer
+            # This works today because all of our subsequent join configuration operations preserve the join type
+            # as-is, or else switch it to a CROSS JOIN or INNER JOIN type, both of which are safe for predicate
+            # pushdown. However, there is currently no way to enforce that invariant, so we will need to move
+            # to a model where we evaluate the join nodes themselves and decide on whether or not to push down
+            # the predicate. This will be much more straightforward once we finish encapsulating our existing
+            # time range constraint pushdown controls into this mechanism.
             candidate_nodes_for_left_side_of_join = list(
                 node_processor.apply_matching_filter_predicates(
                     source_nodes=candidate_nodes_for_left_side_of_join,
@@ -1504,12 +1529,11 @@ class DataflowPlanBuilder:
             )
 
         pre_aggregate_node: DataflowPlanNode = cumulative_metric_constrained_node or unaggregated_measure_node
-        merged_where_filter_spec = WhereFilterSpec.merge_iterable(metric_input_measure_spec.filter_specs)
         if len(metric_input_measure_spec.filter_specs) > 0:
             # Apply where constraint on the node
             pre_aggregate_node = WhereConstraintNode(
                 parent_node=pre_aggregate_node,
-                where_constraint=merged_where_filter_spec,
+                where_specs=metric_input_measure_spec.filter_specs,
             )
 
         if non_additive_dimension_spec is not None:
@@ -1579,7 +1603,7 @@ class DataflowPlanBuilder:
             ]
             if len(queried_filter_specs) > 0:
                 output_node = WhereConstraintNode(
-                    parent_node=output_node, where_constraint=WhereFilterSpec.merge_iterable(queried_filter_specs)
+                    parent_node=output_node, where_specs=queried_filter_specs, always_apply=True
                 )
 
             # TODO: this will break if you query by agg_time_dimension but apply a time constraint on metric_time.

{metricflow-0.207.0.dev1 → metricflow-0.207.0.dev2}/metricflow/dataflow/builder/node_data_set.py

@@ -2,8 +2,11 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Dict, Optional, Sequence
 
+from metricflow_semantics.dag.id_prefix import StaticIdPrefix
+from metricflow_semantics.dag.sequential_id import SequentialIdGenerator
 from metricflow_semantics.mf_logging.runtime import log_block_runtime
 from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver
+from typing_extensions import override
 
 from metricflow.dataflow.dataflow_plan import (
     DataflowPlanNode,
@@ -93,3 +96,11 @@ class DataflowPlanNodeOutputDataSetResolver(DataflowToSqlQueryPlanConverter):
             semantic_manifest_lookup=self._semantic_manifest_lookup,
             _node_to_output_data_set=dict(self._node_to_output_data_set),
         )
+
+    @override
+    def _next_unique_table_alias(self) -> str:
+        """Return the next unique table alias to use in generating queries.
+
+        This uses a separate prefix in order to minimize subquery ID thrash.
+        """
+        return SequentialIdGenerator.create_next_id(StaticIdPrefix.NODE_RESOLVER_SUB_QUERY).str_value

{metricflow-0.207.0.dev1 → metricflow-0.207.0.dev2}/metricflow/dataflow/dataflow_plan.py

@@ -215,6 +215,11 @@ class DataflowPlan(MetricFlowDag[DataflowPlanNode]):
     def sink_node(self) -> DataflowPlanNode:  # noqa: D102
         return self._sink_nodes[0]
 
+    @property
+    def node_count(self) -> int:
+        """Returns the number of nodes in the DataflowPlan."""
+        return len(DataflowPlan.__all_nodes_in_subgraph(self.sink_node))
+
     @staticmethod
     def __all_nodes_in_subgraph(node: DataflowPlanNode) -> Sequence[DataflowPlanNode]:
         """Node accessor for retrieving a flattened sequence of all nodes in the subgraph upstream of the input node.

{metricflow-0.207.0.dev1 → metricflow-0.207.0.dev2}/metricflow/dataflow/nodes/combine_aggregated_outputs.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Sequence, Union
+from typing import Sequence
 
 from metricflow_semantics.dag.id_prefix import IdPrefix, StaticIdPrefix
 from metricflow_semantics.visitor import VisitorOutputT
@@ -16,8 +16,13 @@ class CombineAggregatedOutputsNode(DataflowPlanNode):
 
     def __init__(  # noqa: D107
         self,
-        parent_nodes: Sequence[Union[DataflowPlanNode, DataflowPlanNode]],
+        parent_nodes: Sequence[DataflowPlanNode],
     ) -> None:
+        num_parents = len(parent_nodes)
+        assert num_parents > 1, (
+            "The CombineAggregatedOutputsNode is intended to merge the output datasets from 2 or more nodes, but this "
+            f"node is being initialized with with only {num_parents} parent(s)."
+        )
         super().__init__(node_id=self.create_unique_id(), parent_nodes=parent_nodes)
 
     @classmethod
@@ -37,5 +42,4 @@ class CombineAggregatedOutputsNode(DataflowPlanNode):
     def with_new_parents(  # noqa: D102
         self, new_parent_nodes: Sequence[DataflowPlanNode]
     ) -> CombineAggregatedOutputsNode:
-        assert len(new_parent_nodes) == 1
         return CombineAggregatedOutputsNode(parent_nodes=new_parent_nodes)

metricflow-0.207.0.dev2/metricflow/dataflow/nodes/where_filter.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+from metricflow_semantics.dag.id_prefix import IdPrefix, StaticIdPrefix
+from metricflow_semantics.dag.mf_dag import DisplayedProperty
+from metricflow_semantics.specs.spec_classes import WhereFilterSpec
+from metricflow_semantics.visitor import VisitorOutputT
+
+from metricflow.dataflow.dataflow_plan import DataflowPlanNode, DataflowPlanNodeVisitor
+
+
+class WhereConstraintNode(DataflowPlanNode):
+    """Remove rows using a WHERE clause."""
+
+    def __init__(
+        self,
+        parent_node: DataflowPlanNode,
+        where_specs: Sequence[WhereFilterSpec],
+        always_apply: bool = False,
+    ) -> None:
+        """Initializer.
+
+        WhereConstraintNodes must always have exactly one parent, since they always wrap a single subquery input.
+
+        The always_apply parameter serves as an indicator for a WhereConstraintNode that is added to a plan in order
+        to clean up null outputs from a pre-join filter. For example, when doing time spine joins to fill null values
+        for metric outputs sometimes that join will result in rows with null values for various dimension attributes.
+        By re-applying the filter expression after the join step we will discard those unexpected output rows created
+        by the join (rather than the underlying inputs). In this case, we must ensure that the filters defined in this
+        node are always applied at the moment this node is processed, regardless of whether or not they've been pushed
+        down through the DAG.
+        """
+        self._where_specs = where_specs
+        self.parent_node = parent_node
+        self.always_apply = always_apply
+        super().__init__(node_id=self.create_unique_id(), parent_nodes=(parent_node,))
+
+    @classmethod
+    def id_prefix(cls) -> IdPrefix:  # noqa: D102
+        return StaticIdPrefix.DATAFLOW_NODE_WHERE_CONSTRAINT_ID_PREFIX
+
+    @property
+    def where(self) -> WhereFilterSpec:
+        """Returns the specs for the elements that it should pass."""
+        return WhereFilterSpec.merge_iterable(self._where_specs)
+
+    @property
+    def input_where_specs(self) -> Sequence[WhereFilterSpec]:
+        """Returns the discrete set of input where filter specs for this node.
+
+        This is useful for things like predicate pushdown, where we need to differentiate between individual specs
+        for pushdown operations on the filter spec level. We merge them for things like rendering and node comparisons,
+        but in some cases we may be able to push down a subset of the input specs for efficiency reasons.
+        """
+        return self._where_specs
+
+    def accept(self, visitor: DataflowPlanNodeVisitor[VisitorOutputT]) -> VisitorOutputT:  # noqa: D102
+        return visitor.visit_where_constraint_node(self)
+
+    @property
+    def description(self) -> str:  # noqa: D102
+        # Can't put the where condition here as it can cause rendering issues when there are SQL execution parameters.
+        # e.g. "Constrain Output with WHERE listing__country = :1"
+        return "Constrain Output with WHERE"
+
+    @property
+    def displayed_properties(self) -> Sequence[DisplayedProperty]:  # noqa: D102
+        properties = tuple(super().displayed_properties) + (DisplayedProperty("where_condition", self.where),)
+        if self.always_apply:
+            properties = properties + (DisplayedProperty("All filters always applied:", self.always_apply),)
+        return properties
+
+    def functionally_identical(self, other_node: DataflowPlanNode) -> bool:  # noqa: D102
+        return (
+            isinstance(other_node, self.__class__)
+            and other_node.where == self.where
+            and other_node.always_apply == self.always_apply
+        )
+
+    def with_new_parents(self, new_parent_nodes: Sequence[DataflowPlanNode]) -> WhereConstraintNode:  # noqa: D102
+        assert len(new_parent_nodes) == 1
+        return WhereConstraintNode(
+            parent_node=new_parent_nodes[0], where_specs=self.input_where_specs, always_apply=self.always_apply
+        )

metricflow-0.207.0.dev2/metricflow/dataflow/optimizer/dataflow_optimizer_factory.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import FrozenSet, List, Sequence
+
+from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+
+from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver
+from metricflow.dataflow.optimizer.dataflow_plan_optimizer import DataflowPlanOptimizer
+from metricflow.dataflow.optimizer.predicate_pushdown_optimizer import PredicatePushdownOptimizer
+from metricflow.dataflow.optimizer.source_scan.source_scan_optimizer import SourceScanOptimizer
+
+
+class DataflowPlanOptimization(Enum):
+    """Enumeration of optimization types available for execution.
+
+    Values indicate order of application. We apply the source scan optimizer first because it reduces input branches,
+    making for maximally parsimonious queries prior to application of predicate pushdown. Note this is safe only
+    because the SourceScanOptimizer combines from the CombineAggregatedOutputNode, and will only combine branches
+    from there to source if they are functionally identical (i.e., they have all of the same WhereConstraintNode
+    configurations).
+    """
+
+    SOURCE_SCAN = 0
+    PREDICATE_PUSHDOWN = 1
+
+    @staticmethod
+    def all_optimizations() -> FrozenSet[DataflowPlanOptimization]:
+        """Convenience method for getting a set of all available optimizations."""
+        return frozenset((DataflowPlanOptimization.SOURCE_SCAN, DataflowPlanOptimization.PREDICATE_PUSHDOWN))
+
+
+class DataflowPlanOptimizerFactory:
+    """Factory class for initializing an enumerated set of optimizers.
+
+    This allows us to centralize initialization and, most importantly, share class instances with cached high cost
+    processing between the DataflowPlanBuilder and the optimizer instances requiring that functionality.
+    """
+
+    def __init__(self, node_data_set_resolver: DataflowPlanNodeOutputDataSetResolver) -> None:
+        """Initializer.
+
+        This collects all of the initialization requirements for the optimizers it manages.
+        """
+        self._node_data_set_resolver = node_data_set_resolver
+
+    def get_optimizers(self, optimizations: FrozenSet[DataflowPlanOptimization]) -> Sequence[DataflowPlanOptimizer]:
+        """Initializes and returns a sequence of optimizers matching the input optimization requests."""
+        optimizers: List[DataflowPlanOptimizer] = []
+        for optimization in sorted(list(optimizations), key=lambda x: x.value):
+            if optimization is DataflowPlanOptimization.SOURCE_SCAN:
+                optimizers.append(SourceScanOptimizer())
+            elif optimization is DataflowPlanOptimization.PREDICATE_PUSHDOWN:
+                optimizers.append(PredicatePushdownOptimizer(self._node_data_set_resolver))
+            else:
+                assert_values_exhausted(optimization)
+
+        return tuple(optimizers)