metricflow 0.203.0.dev6__tar.gz → 0.203.0.dev7__tar.gz

{metricflow-0.203.0.dev6 → metricflow-0.203.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metricflow
-Version: 0.203.0.dev6
+Version: 0.203.0.dev7
 Summary: Translates a simple metric definition into reusable SQL and executes it against the SQL engine of your choice.
 Project-URL: Documentation, https://transform.co/metricflow
 Project-URL: Source Code, https://github.com/transform-data/metricflow

{metricflow-0.203.0.dev6 → metricflow-0.203.0.dev7}/metricflow/dataflow/builder/costing.py

@@ -132,7 +132,11 @@ class DefaultCostFunction(
         return DefaultCost.sum([x.accept(self) for x in node.parent_nodes])
 
     def visit_pass_elements_filter_node(self, node: FilterElementsNode) -> DefaultCost:  # noqa: D
-        return DefaultCost.sum([x.accept(self) for x in node.parent_nodes])
+        parent_costs = [x.accept(self) for x in node.parent_nodes]
+
+        # 1 aggregation if grouping by distinct values
+        node_cost = DefaultCost(num_aggregations=1 if node.distinct else 0)
+        return DefaultCost.sum(parent_costs + [node_cost])
 
     def visit_combine_metrics_node(self, node: CombineMetricsNode) -> DefaultCost:  # noqa: D
         return DefaultCost.sum([x.accept(self) for x in node.parent_nodes])

{metricflow-0.203.0.dev6 → metricflow-0.203.0.dev7}/metricflow/dataflow/builder/dataflow_plan_builder.py

@@ -9,10 +9,11 @@ from typing import DefaultDict, Dict, List, Optional, Sequence, Set, Tuple, Unio
 from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
 from dbt_semantic_interfaces.pretty_print import pformat_big_objects
 from dbt_semantic_interfaces.protocols.metric import MetricTimeWindow, MetricType
-from dbt_semantic_interfaces.references import TimeDimensionReference
+from dbt_semantic_interfaces.references import (
+    TimeDimensionReference,
+)
 from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
 
-from metricflow.assert_one_arg import assert_exactly_one_arg_set
 from metricflow.dag.id_generation import DATAFLOW_PLAN_PREFIX, IdGeneratorRegistry
 from metricflow.dataflow.builder.costing import DataflowPlanNodeCostFunction, DefaultCostFunction
 from metricflow.dataflow.builder.measure_additiveness import group_measure_specs_by_additiveness
@@ -36,6 +37,7 @@ from metricflow.dataflow.dataflow_plan import (
     JoinToBaseOutputNode,
     JoinToTimeSpineNode,
     OrderByLimitNode,
+    ReadSqlSourceNode,
     SemiAdditiveJoinNode,
     SinkOutput,
     WhereConstraintNode,
@@ -50,11 +52,9 @@ from metricflow.errors.errors import UnableToSatisfyQueryError
 from metricflow.filters.time_constraint import TimeRangeConstraint
 from metricflow.model.semantic_manifest_lookup import SemanticManifestLookup
 from metricflow.plan_conversion.column_resolver import DunderColumnAssociationResolver
-from metricflow.plan_conversion.node_processor import PreDimensionJoinNodeProcessor
+from metricflow.plan_conversion.node_processor import PreJoinNodeProcessor
 from metricflow.specs.column_assoc import ColumnAssociationResolver
 from metricflow.specs.specs import (
-    DimensionSpec,
-    EntitySpec,
     InstanceSpecSet,
     LinkableInstanceSpec,
     LinkableSpecSet,
@@ -74,18 +74,60 @@ logger = logging.getLogger(__name__)
 
 
 @dataclass(frozen=True)
-class MeasureRecipe:
-    """Get a recipe for how to build a dataflow plan node that outputs measures and the needed linkable instances.
-
-    The recipe involves filtering the measure node so that it only outputs the measures and the instances associated with
-    required_local_linkable_specs, then joining the nodes containing the linkable instances according to the recipes
-    in join_linkable_instances_recipes.
-    """
+class DataflowRecipe:
+    """Get a recipe for how to build a dataflow plan node that outputs measures and linkable instances as needed."""
 
-    measure_node: BaseOutput
+    source_node: BaseOutput
     required_local_linkable_specs: Tuple[LinkableInstanceSpec, ...]
     join_linkable_instances_recipes: Tuple[JoinLinkableInstancesRecipe, ...]
 
+    @property
+    def join_targets(self) -> List[JoinDescription]:
+        """Joins to be made to source node."""
+        join_targets = []
+        for join_recipe in self.join_linkable_instances_recipes:
+            # Figure out what elements to filter from the joined node.
+
+            # Sanity check - all linkable specs should have a link, or else why would we be joining them.
+            assert all([len(x.entity_links) > 0 for x in join_recipe.satisfiable_linkable_specs])
+
+            # If we're joining something in, then we need the associated entity, partitions, and time dimension
+            # specs defining the validity window (if necessary)
+            include_specs: List[LinkableInstanceSpec] = [
+                LinklessEntitySpec.from_reference(x.entity_links[0]) for x in join_recipe.satisfiable_linkable_specs
+            ]
+            include_specs.extend([x.node_to_join_dimension_spec for x in join_recipe.join_on_partition_dimensions])
+            include_specs.extend(
+                [x.node_to_join_time_dimension_spec for x in join_recipe.join_on_partition_time_dimensions]
+            )
+            if join_recipe.validity_window:
+                include_specs.extend(
+                    [
+                        join_recipe.validity_window.window_start_dimension,
+                        join_recipe.validity_window.window_end_dimension,
+                    ]
+                )
+
+            # satisfiable_linkable_specs describes what can be satisfied after the join, so remove the entity
+            # link when filtering before the join.
+            # e.g. if the node is used to satisfy "user_id__country", then the node must have the entity
+            # "user_id" and the "country" dimension so that it can be joined to the measure node.
+            include_specs.extend([x.without_first_entity_link for x in join_recipe.satisfiable_linkable_specs])
+            filtered_node_to_join = FilterElementsNode(
+                parent_node=join_recipe.node_to_join,
+                include_specs=InstanceSpecSet.create_from_linkable_specs(include_specs),
+            )
+            join_targets.append(
+                JoinDescription(
+                    join_node=filtered_node_to_join,
+                    join_on_entity=join_recipe.join_on_entity,
+                    join_on_partition_dimensions=join_recipe.join_on_partition_dimensions,
+                    join_on_partition_time_dimensions=join_recipe.join_on_partition_time_dimensions,
+                    validity_window=join_recipe.validity_window,
+                )
+            )
+        return join_targets
+
 
 @dataclass(frozen=True)
 class MeasureSpecProperties:
@@ -103,6 +145,7 @@ class DataflowPlanBuilder:
     def __init__(  # noqa: D
         self,
         source_nodes: Sequence[BaseOutput],
+        read_nodes: Sequence[ReadSqlSourceNode],
         semantic_manifest_lookup: SemanticManifestLookup,
         cost_function: DataflowPlanNodeCostFunction = DefaultCostFunction(),
         node_output_resolver: Optional[DataflowPlanNodeOutputDataSetResolver] = None,
@@ -113,6 +156,7 @@ class DataflowPlanBuilder:
         self._metric_time_dimension_reference = DataSet.metric_time_dimension_reference()
         self._cost_function = cost_function
         self._source_nodes = source_nodes
+        self._read_nodes = read_nodes
         self._column_association_resolver = (
             DunderColumnAssociationResolver(semantic_manifest_lookup)
             if not column_association_resolver
@@ -146,8 +190,8 @@ class DataflowPlanBuilder:
             time_range_constraint=query_spec.time_range_constraint,
         )
 
-        sink_node = DataflowPlanBuilder.build_sink_node_from_metrics_output_node(
-            computed_metrics_output=metrics_output_node,
+        sink_node = DataflowPlanBuilder.build_sink_node(
+            parent_node=metrics_output_node,
             order_by_specs=query_spec.order_by_specs,
             output_sql_table=output_sql_table,
             limit=query_spec.limit,
@@ -259,67 +303,48 @@ class DataflowPlanBuilder:
             join_type=combine_metrics_join_type,
         )
 
-    def build_plan_for_distinct_values(
-        self,
-        metric_specs: Sequence[MetricSpec],
-        dimension_spec: Optional[DimensionSpec] = None,
-        time_dimension_spec: Optional[TimeDimensionSpec] = None,
-        entity_spec: Optional[EntitySpec] = None,
-        time_range_constraint: Optional[TimeRangeConstraint] = None,
-        limit: Optional[int] = None,
-    ) -> DataflowPlan:
+    def build_plan_for_distinct_values(self, query_spec: MetricFlowQuerySpec) -> DataflowPlan:
         """Generate a plan that would get the distinct values of a linkable instance.
 
         e.g. distinct listing__country_latest for bookings by listing__country_latest
         """
-        assert_exactly_one_arg_set(
-            dimension_spec=dimension_spec, time_dimension_spec=time_dimension_spec, entity_spec=entity_spec
-        )
+        assert not query_spec.metric_specs, "Can't build distinct values plan with metrics."
 
-        # Doing this to keep the type checker happy, but assert_exactly_one_arg_set should ensure this.
-        linkable_spec: Optional[LinkableInstanceSpec] = dimension_spec or time_dimension_spec or entity_spec
-        assert linkable_spec
+        dataflow_recipe = self._find_dataflow_recipe(linkable_spec_set=query_spec.linkable_specs)
+        if not dataflow_recipe:
+            raise UnableToSatisfyQueryError(f"Recipe not found for linkable specs: {query_spec.linkable_specs}")
 
-        query_spec = MetricFlowQuerySpec(
-            metric_specs=tuple(metric_specs),
-            dimension_specs=(dimension_spec,) if dimension_spec else (),
-            time_dimension_specs=(time_dimension_spec,) if time_dimension_spec else (),
-            entity_specs=(entity_spec,) if entity_spec else (),
-            time_range_constraint=time_range_constraint,
-        )
-        metrics_output_node = self._build_metrics_output_node(
-            metric_specs=query_spec.metric_specs,
-            queried_linkable_specs=query_spec.linkable_specs,
-            where_constraint=query_spec.where_constraint,
-            time_range_constraint=query_spec.time_range_constraint,
-        )
+        joined_node: Optional[JoinToBaseOutputNode] = None
+        if dataflow_recipe.join_targets:
+            joined_node = JoinToBaseOutputNode(
+                left_node=dataflow_recipe.source_node, join_targets=dataflow_recipe.join_targets
+            )
+
+        where_constraint_node: Optional[WhereConstraintNode] = None
+        if query_spec.where_constraint:
+            where_constraint_node = WhereConstraintNode(
+                parent_node=joined_node or dataflow_recipe.source_node, where_constraint=query_spec.where_constraint
+            )
 
         distinct_values_node = FilterElementsNode(
-            parent_node=metrics_output_node,
-            include_specs=InstanceSpecSet.create_from_linkable_specs((linkable_spec,)),
+            parent_node=where_constraint_node or joined_node or dataflow_recipe.source_node,
+            include_specs=query_spec.linkable_specs.as_spec_set,
+            distinct=True,
         )
 
-        sink_node = self.build_sink_node_from_metrics_output_node(
-            computed_metrics_output=distinct_values_node,
-            order_by_specs=(
-                OrderBySpec(
-                    instance_spec=linkable_spec,
-                    descending=False,
-                ),
-            ),
-            limit=limit,
+        sink_node = self.build_sink_node(
+            parent_node=distinct_values_node,
+            order_by_specs=query_spec.order_by_specs,
+            limit=query_spec.limit,
         )
 
         plan_id = IdGeneratorRegistry.for_class(DataflowPlanBuilder).create_id(DATAFLOW_PLAN_PREFIX)
 
-        return DataflowPlan(
-            plan_id=plan_id,
-            sink_output_nodes=[sink_node],
-        )
+        return DataflowPlan(plan_id=plan_id, sink_output_nodes=[sink_node])
 
     @staticmethod
-    def build_sink_node_from_metrics_output_node(
-        computed_metrics_output: BaseOutput,
+    def build_sink_node(
+        parent_node: BaseOutput,
         order_by_specs: Sequence[OrderBySpec],
         output_sql_table: Optional[SqlTable] = None,
         limit: Optional[int] = None,
@@ -330,26 +355,20 @@ class DataflowPlanBuilder:
 
         if order_by_specs or limit:
             pre_result_node = OrderByLimitNode(
-                order_by_specs=list(order_by_specs),
-                limit=limit,
-                parent_node=computed_metrics_output,
+                order_by_specs=list(order_by_specs), limit=limit, parent_node=parent_node
             )
 
         if output_selection_specs:
             pre_result_node = FilterElementsNode(
-                parent_node=pre_result_node or computed_metrics_output,
-                include_specs=output_selection_specs,
+                parent_node=pre_result_node or parent_node, include_specs=output_selection_specs
             )
 
         write_result_node: SinkOutput
         if not output_sql_table:
-            write_result_node = WriteToResultDataframeNode(
-                parent_node=pre_result_node or computed_metrics_output,
-            )
+            write_result_node = WriteToResultDataframeNode(parent_node=pre_result_node or parent_node)
         else:
             write_result_node = WriteToResultTableNode(
-                parent_node=pre_result_node or computed_metrics_output,
-                output_sql_table=output_sql_table,
+                parent_node=pre_result_node or parent_node, output_sql_table=output_sql_table
             )
 
         return write_result_node
@@ -397,6 +416,21 @@ class DataflowPlanBuilder:
                 nodes.append(source_node)
         return nodes
 
+    def _select_read_nodes_with_linkable_specs(
+        self, linkable_specs: LinkableSpecSet, read_nodes: Sequence[ReadSqlSourceNode]
+    ) -> Dict[BaseOutput, Set[LinkableInstanceSpec]]:
+        """Find source nodes with requested linkable specs and no measures."""
+        nodes_to_linkable_specs: Dict[BaseOutput, Set[LinkableInstanceSpec]] = {}
+        linkable_specs_set = set(linkable_specs.as_tuple)
+        for read_node in read_nodes:
+            output_spec_set = self._node_data_set_resolver.get_output_data_set(read_node).instance_set.spec_set
+            linkable_specs_in_node = set(output_spec_set.linkable_specs)
+            requested_linkable_specs_in_node = linkable_specs_set.intersection(linkable_specs_in_node)
+            if requested_linkable_specs_in_node:
+                nodes_to_linkable_specs[read_node] = requested_linkable_specs_in_node
+
+        return nodes_to_linkable_specs
+
     def _find_non_additive_dimension_in_linkable_specs(
         self,
         agg_time_dimension: TimeDimensionReference,
@@ -449,39 +483,38 @@ class DataflowPlanBuilder:
             non_additive_dimension_spec=non_additive_dimension_spec,
         )
 
-    def _find_measure_recipe(
+    def _find_dataflow_recipe(
         self,
-        measure_spec_properties: MeasureSpecProperties,
-        linkable_specs: Sequence[LinkableInstanceSpec],
+        linkable_spec_set: LinkableSpecSet,
+        measure_spec_properties: Optional[MeasureSpecProperties] = None,
         time_range_constraint: Optional[TimeRangeConstraint] = None,
-    ) -> Optional[MeasureRecipe]:
-        """Find a recipe for getting measure_specs along with the linkable specs.
-
-        Prior to calling this method we should always be checking that all input measure specs come from
-        the same base semantic model, otherwise the internal conditions here will be impossible to satisfy
-        """
-        measure_specs = measure_spec_properties.measure_specs
-        node_processor = PreDimensionJoinNodeProcessor(
-            semantic_model_lookup=self._semantic_model_lookup,
-            node_data_set_resolver=self._node_data_set_resolver,
-        )
-
-        source_nodes: Sequence[BaseOutput] = self._source_nodes
-
-        # We only care about nodes that have all required measures
-        potential_measure_nodes: Sequence[BaseOutput] = self._select_source_nodes_with_measures(
-            measure_specs=set(measure_specs), source_nodes=source_nodes
-        )
+    ) -> Optional[DataflowRecipe]:
+        linkable_specs = linkable_spec_set.as_tuple
+        if measure_spec_properties:
+            source_nodes = self._source_nodes
+            potential_source_nodes: Sequence[BaseOutput] = self._select_source_nodes_with_measures(
+                measure_specs=set(measure_spec_properties.measure_specs), source_nodes=source_nodes
+            )
+        else:
+            # Only read nodes can be source nodes for queries without measures
+            source_nodes = self._read_nodes
+            source_nodes_to_linkable_specs = self._select_read_nodes_with_linkable_specs(
+                linkable_specs=linkable_spec_set, read_nodes=source_nodes
+            )
+            potential_source_nodes = list(source_nodes_to_linkable_specs.keys())
 
-        logger.info(f"There are {len(potential_measure_nodes)} potential measure source nodes")
+        logger.info(f"There are {len(potential_source_nodes)} potential source nodes")
 
         logger.info(f"Starting search with {len(source_nodes)} source nodes")
         start_time = time.time()
-        # Only apply the time constraint to nodes that will be used for measures because some dimensional sources have
-        # measures in them, and time constraining those would result in incomplete joins.
+
+        node_processor = PreJoinNodeProcessor(
+            semantic_model_lookup=self._semantic_model_lookup,
+            node_data_set_resolver=self._node_data_set_resolver,
+        )
         if time_range_constraint:
-            potential_measure_nodes = node_processor.add_time_range_constraint(
-                source_nodes=potential_measure_nodes,
+            potential_source_nodes = node_processor.add_time_range_constraint(
+                source_nodes=potential_source_nodes,
                 metric_time_dimension_reference=self._metric_time_dimension_reference,
                 time_range_constraint=time_range_constraint,
             )
@@ -500,7 +533,6 @@ class DataflowPlanBuilder:
                 f"After adding multi-hop nodes, there are {len(nodes_available_for_joins)} nodes available for joins:\n"
                 f"{pformat_big_objects(nodes_available_for_joins)}"
             )
-
         logger.info(f"Processing nodes took: {time.time()-start_time:.2f}s")
 
         node_evaluator = NodeEvaluatorForLinkableInstances(
@@ -509,36 +541,37 @@ class DataflowPlanBuilder:
             node_data_set_resolver=self._node_data_set_resolver,
         )
 
-        # Dict from the node that contains the measure spec to the evaluation results.
+        # Dict from the node that contains the source node to the evaluation results.
         node_to_evaluation: Dict[BaseOutput, LinkableInstanceSatisfiabilityEvaluation] = {}
 
-        for node in self._sort_by_suitability(potential_measure_nodes):
+        for node in self._sort_by_suitability(potential_source_nodes):
             data_set = self._node_data_set_resolver.get_output_data_set(node)
 
-            missing_specs = [spec for spec in measure_specs if spec not in data_set.instance_set.spec_set.measure_specs]
-            if missing_specs:
-                logger.debug(
-                    f"Skipping evaluation for node since it does not have all of the measure specs {missing_specs}:"
-                    f"\n\n{dataflow_dag_as_text(node)}"
-                )
-                continue
+            if measure_spec_properties:
+                measure_specs = measure_spec_properties.measure_specs
+                missing_specs = [
+                    spec for spec in measure_specs if spec not in data_set.instance_set.spec_set.measure_specs
+                ]
+                if missing_specs:
+                    logger.debug(
+                        f"Skipping evaluation for node since it does not have all of the measure specs {missing_specs}:"
+                        f"\n\n{dataflow_dag_as_text(node)}"
+                    )
+                    continue
 
-            logger.debug(f"Evaluating measure node:\n{pformat_big_objects(measure_node=dataflow_dag_as_text(node))}")
+            logger.debug(f"Evaluating source node:\n{pformat_big_objects(source_node=dataflow_dag_as_text(node))}")
 
             start_time = time.time()
-            evaluation = node_evaluator.evaluate_node(
-                start_node=node,
-                required_linkable_specs=list(linkable_specs),
-            )
+            evaluation = node_evaluator.evaluate_node(start_node=node, required_linkable_specs=list(linkable_specs))
             logger.info(f"Evaluation of {node} took {time.time() - start_time:.2f}s")
 
             logger.debug(
-                f"Evaluation for measure node is:\n"
+                f"Evaluation for source node is:\n"
                 f"{pformat_big_objects(node=dataflow_dag_as_text(node), evaluation=evaluation)}"
             )
 
             if len(evaluation.unjoinable_linkable_specs) > 0:
-                logger.debug(
+                logger.info(
                     f"Skipping {node.node_id} since it contains un-joinable specs: "
                     f"{evaluation.unjoinable_linkable_specs}"
                 )
@@ -555,7 +588,7 @@ class DataflowPlanBuilder:
                 logger.info("Not evaluating other nodes since we found one that doesn't require joins")
                 break
 
-        logger.info(f"Found {len(node_to_evaluation)} candidate measure nodes.")
+        logger.info(f"Found {len(node_to_evaluation)} candidate source nodes.")
 
         if len(node_to_evaluation) > 0:
             # All source nodes cost the same. Find evaluation with lowest number of joins.
@@ -572,8 +605,8 @@ class DataflowPlanBuilder:
                 )
             )
 
-            # Nodes containing the linkable instances will be joined to the node containing the measure, so these
-            # entities will need to be present in the measure node.
+            # Nodes containing the linkable instances will be joined to the source node, so these
+            # entities will need to be present in the source node.
             required_local_entity_specs = tuple(x.join_on_entity for x in evaluation.join_recipes)
             # Same thing with partitions.
             required_local_dimension_specs = tuple(
@@ -585,8 +618,8 @@ class DataflowPlanBuilder:
                 for y in x.join_on_partition_time_dimensions
             )
 
-            return MeasureRecipe(
-                measure_node=node_with_lowest_cost_plan,
+            return DataflowRecipe(
+                source_node=node_with_lowest_cost_plan,
                 required_local_linkable_specs=(
                     evaluation.local_linkable_specs
                     + required_local_entity_specs
@@ -756,10 +789,10 @@ class DataflowPlanBuilder:
         )
 
         find_recipe_start_time = time.time()
-        measure_recipe = self._find_measure_recipe(
+        measure_recipe = self._find_dataflow_recipe(
             measure_spec_properties=measure_properties,
             time_range_constraint=cumulative_metric_adjusted_time_constraint or time_range_constraint,
-            linkable_specs=required_linkable_specs.as_tuple,
+            linkable_spec_set=required_linkable_specs,
         )
         logger.info(
             f"With {len(self._source_nodes)} source nodes, finding a recipe took "
@@ -779,7 +812,7 @@ class DataflowPlanBuilder:
         time_range_node: Optional[JoinOverTimeRangeNode] = None
         if cumulative and metric_time_dimension_requested:
             time_range_node = JoinOverTimeRangeNode(
-                parent_node=measure_recipe.measure_node,
+                parent_node=measure_recipe.source_node,
                 window=cumulative_window,
                 grain_to_date=cumulative_grain_to_date,
                 time_range_constraint=time_range_constraint,
@@ -790,7 +823,7 @@ class DataflowPlanBuilder:
         if metric_spec.offset_window or metric_spec.offset_to_grain:
             assert metric_time_dimension_specs, "Joining to time spine requires querying with metric time."
             join_to_time_spine_node = JoinToTimeSpineNode(
-                parent_node=time_range_node or measure_recipe.measure_node,
+                parent_node=time_range_node or measure_recipe.source_node,
                 metric_time_dimension_specs=metric_time_dimension_specs,
                 time_range_constraint=time_range_constraint,
                 offset_window=metric_spec.offset_window,
@@ -799,7 +832,7 @@ class DataflowPlanBuilder:
 
         # Only get the required measure and the local linkable instances so that aggregations work correctly.
         filtered_measure_source_node = FilterElementsNode(
-            parent_node=join_to_time_spine_node or time_range_node or measure_recipe.measure_node,
+            parent_node=join_to_time_spine_node or time_range_node or measure_recipe.source_node,
             include_specs=InstanceSpecSet.merge(
                 (
                     InstanceSpecSet(measure_specs=measure_specs),
@@ -808,49 +841,7 @@ class DataflowPlanBuilder:
             ),
         )
 
-        join_targets = []
-        for join_recipe in measure_recipe.join_linkable_instances_recipes:
-            # Figure out what elements to filter from the joined node.
-
-            # Sanity check - all linkable specs should have a link, or else why would we be joining them.
-            assert all([len(x.entity_links) > 0 for x in join_recipe.satisfiable_linkable_specs])
-
-            # If we're joining something in, then we need the associated entity, partitions, and time dimension
-            # specs defining the validity window (if necessary)
-            include_specs: List[LinkableInstanceSpec] = [
-                LinklessEntitySpec.from_reference(x.entity_links[0]) for x in join_recipe.satisfiable_linkable_specs
-            ]
-            include_specs.extend([x.node_to_join_dimension_spec for x in join_recipe.join_on_partition_dimensions])
-            include_specs.extend(
-                [x.node_to_join_time_dimension_spec for x in join_recipe.join_on_partition_time_dimensions]
-            )
-            if join_recipe.validity_window:
-                include_specs.extend(
-                    [
-                        join_recipe.validity_window.window_start_dimension,
-                        join_recipe.validity_window.window_end_dimension,
-                    ]
-                )
-
-            # satisfiable_linkable_specs describes what can be satisfied after the join, so remove the entity
-            # link when filtering before the join.
-            # e.g. if the node is used to satisfy "user_id__country", then the node must have the entity
-            # "user_id" and the "country" dimension so that it can be joined to the measure node.
-            include_specs.extend([x.without_first_entity_link for x in join_recipe.satisfiable_linkable_specs])
-            filtered_node_to_join = FilterElementsNode(
-                parent_node=join_recipe.node_to_join,
-                include_specs=InstanceSpecSet.create_from_linkable_specs(include_specs),
-            )
-            join_targets.append(
-                JoinDescription(
-                    join_node=filtered_node_to_join,
-                    join_on_entity=join_recipe.join_on_entity,
-                    join_on_partition_dimensions=join_recipe.join_on_partition_dimensions,
-                    join_on_partition_time_dimensions=join_recipe.join_on_partition_time_dimensions,
-                    validity_window=join_recipe.validity_window,
-                )
-            )
-
+        join_targets = measure_recipe.join_targets
         unaggregated_measure_node: BaseOutput
         if len(join_targets) > 0:
             filtered_measures_with_joined_elements = JoinToBaseOutputNode(

{metricflow-0.203.0.dev6 → metricflow-0.203.0.dev7}/metricflow/dataflow/builder/source_node.py

@@ -46,3 +46,9 @@ class SourceNodeBuilder:
                         )
                     )
         return source_nodes
+
+    def create_read_nodes_from_data_sets(
+        self, data_sets: Sequence[SemanticModelDataSet]
+    ) -> Sequence[ReadSqlSourceNode]:
+        """Creates read nodes from SemanticModelDataSets."""
+        return [ReadSqlSourceNode(data_set) for data_set in data_sets]

{metricflow-0.203.0.dev6 → metricflow-0.203.0.dev7}/metricflow/dataflow/dataflow_plan.py

@@ -1098,10 +1098,12 @@ class FilterElementsNode(BaseOutput):
         parent_node: BaseOutput,
         include_specs: InstanceSpecSet,
         replace_description: Optional[str] = None,
+        distinct: bool = False,
     ) -> None:
         self._include_specs = include_specs
         self._replace_description = replace_description
         self._parent_node = parent_node
+        self._distinct = distinct
         super().__init__(node_id=self.create_unique_id(), parent_nodes=[parent_node])
 
     @classmethod
@@ -1113,6 +1115,11 @@ class FilterElementsNode(BaseOutput):
         """Returns the specs for the elements that it should pass."""
         return self._include_specs
 
+    @property
+    def distinct(self) -> bool:
+        """True if you only want the distinct values for the selected specs."""
+        return self._distinct
+
     def accept(self, visitor: DataflowPlanNodeVisitor[VisitorOutputT]) -> VisitorOutputT:  # noqa: D
         return visitor.visit_pass_elements_filter_node(self)
 
@@ -1132,7 +1139,7 @@ class FilterElementsNode(BaseOutput):
         if not self._replace_description:
             additional_properties = [
                 DisplayedProperty("include_spec", include_spec) for include_spec in self._include_specs.all_specs
-            ]
+            ] + [DisplayedProperty("distinct", self._distinct)]
         return super().displayed_properties + additional_properties
 
     @property
@@ -1140,13 +1147,18 @@ class FilterElementsNode(BaseOutput):
         return self._parent_node
 
     def functionally_identical(self, other_node: DataflowPlanNode) -> bool:  # noqa: D
-        return isinstance(other_node, self.__class__) and other_node.include_specs == self.include_specs
+        return (
+            isinstance(other_node, self.__class__)
+            and other_node.include_specs == self.include_specs
+            and other_node.distinct == self.distinct
+        )
 
     def with_new_parents(self, new_parent_nodes: Sequence[BaseOutput]) -> FilterElementsNode:  # noqa: D
         assert len(new_parent_nodes) == 1
         return FilterElementsNode(
             parent_node=new_parent_nodes[0],
             include_specs=self.include_specs,
+            distinct=self.distinct,
             replace_description=self._replace_description,
         )
 

{metricflow-0.203.0.dev6 → metricflow-0.203.0.dev7}/metricflow/engine/metricflow_engine.py

@@ -350,6 +350,7 @@ class MetricFlowEngine(AbstractMetricFlowEngine):
 
         source_node_builder = SourceNodeBuilder(self._semantic_manifest_lookup)
         source_nodes = source_node_builder.create_from_data_sets(self._source_data_sets)
+        read_nodes = source_node_builder.create_read_nodes_from_data_sets(self._source_data_sets)
 
         node_output_resolver = DataflowPlanNodeOutputDataSetResolver(
             column_association_resolver=DunderColumnAssociationResolver(semantic_manifest_lookup),
@@ -358,6 +359,7 @@ class MetricFlowEngine(AbstractMetricFlowEngine):
 
         self._dataflow_plan_builder = DataflowPlanBuilder(
             source_nodes=source_nodes,
+            read_nodes=read_nodes,
             semantic_manifest_lookup=self._semantic_manifest_lookup,
         )
         self._to_sql_query_plan_converter = DataflowToSqlQueryPlanConverter(
@@ -374,7 +376,7 @@ class MetricFlowEngine(AbstractMetricFlowEngine):
         self._query_parser = MetricFlowQueryParser(
             column_association_resolver=self._column_association_resolver,
             model=self._semantic_manifest_lookup,
-            source_nodes=source_nodes,
+            read_nodes=read_nodes,
             node_output_resolver=node_output_resolver,
         )
 
@@ -483,12 +485,15 @@ class MetricFlowEngine(AbstractMetricFlowEngine):
                 time_dimension_specs=query_spec.time_dimension_specs,
             )
 
-        dataflow_plan = self._dataflow_plan_builder.build_plan(
-            query_spec=query_spec,
-            output_sql_table=output_table,
-            output_selection_specs=output_selection_specs,
-            optimizers=(SourceScanOptimizer(),),
-        )
+        if query_spec.metric_specs:
+            dataflow_plan = self._dataflow_plan_builder.build_plan(
+                query_spec=query_spec,
+                output_sql_table=output_table,
+                output_selection_specs=output_selection_specs,
+                optimizers=(SourceScanOptimizer(),),
+            )
+        else:
+            dataflow_plan = self._dataflow_plan_builder.build_plan_for_distinct_values(query_spec=query_spec)
 
         if len(dataflow_plan.sink_output_nodes) > 1:
             raise NotImplementedError(