pytrilogy 0.0.3.104__py3-none-any.whl → 0.0.3.106__py3-none-any.whl

{pytrilogy-0.0.3.104.dist-info → pytrilogy-0.0.3.106.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.104
+Version: 0.0.3.106
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.104.dist-info → pytrilogy-0.0.3.106.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.104.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=HyZF9WId40s9G3BjFS2OBMeHI7XNeE-YU1cFLvNOSWk,304
+pytrilogy-0.0.3.106.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=6_By-LphYYIXu7GSa5PwnchymrRabR6qiUwPJWX62EE,304
 trilogy/constants.py,sha256=g_zkVCNjGop6coZ1kM8eXXAzCnUN22ldx3TYFz0E9sc,1747
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=KgCAQhHPT-j0rPkBbALX0f84W9-Q-bkjHayGuavg99w,16490
@@ -37,26 +37,26 @@ trilogy/core/optimizations/predicate_pushdown.py,sha256=5ubatgq1IwWQ4L2FDt4--y16
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=AcMU1d5uCo8I1PFCkBtmcC6iFmM9vN6xSdKxSVMGfpA,23080
 trilogy/core/processing/discovery_node_factory.py,sha256=p23jiiHyhrW-Q8ndbnRlqMHJKT8ZqPOA89SzE4xaFFo,15445
-trilogy/core/processing/discovery_utility.py,sha256=wIuLsE6yuVykeYZdIqRSagivDNU3-ooiS7z6in4yqho,11518
+trilogy/core/processing/discovery_utility.py,sha256=ZJTTWm34zpR30p-xzElUJCTdx-XT44skKG0-LBfEOg4,12525
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
-trilogy/core/processing/utility.py,sha256=1_oNnk6lWiy-D7LKYr07kU_v7iAM4i6ITUAS4bIiCr4,23444
+trilogy/core/processing/utility.py,sha256=ESs6pKqVP2c9eMdfB2JNjw7D7YnoezVwbLFx1D6OUYA,26088
 trilogy/core/processing/node_generators/__init__.py,sha256=iVJ-crowPxYeut-hFjyEjfibKIDq7PfB4LEuDAUCjGY,943
 trilogy/core/processing/node_generators/basic_node.py,sha256=74LoVZXLinRvSzk2LmI1kwza96TnuH3ELoYRIbHB29A,5578
 trilogy/core/processing/node_generators/common.py,sha256=xF32Kf6B08dZgKs2SOow1HomptSiSC057GCUCHFlS5s,9464
 trilogy/core/processing/node_generators/constant_node.py,sha256=LfpDq2WrBRZ3tGsLxw77LuigKfhbteWWh9L8BGdMGwk,1146
-trilogy/core/processing/node_generators/filter_node.py,sha256=ndPznkcFu_cdCNgaRpgot8oqnzdHv4KAIfjeUIzrE2w,10816
+trilogy/core/processing/node_generators/filter_node.py,sha256=cJ5od1fAfvalaUDO2O4Y6Yrr2RukOCqey7f3zrKSBbI,10808
 trilogy/core/processing/node_generators/group_node.py,sha256=NdK1rl6Ze94XFWtgeC2dlRiL4pS3lh1ArKGPEltLtnw,8525
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=a505AEixjsjp5jI8Ng3H5KF_AaehkS6HfRfTef64l_o,7063
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=hNcZxnDLTZyYJWfojg769zH9HB9PfZfESmpN1lcHWXg,23172
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
 trilogy/core/processing/node_generators/rowset_node.py,sha256=MuVNIexXhqGONho_mewqMOwaYXNUnjjvyPvk_RDGNYE,5943
-trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
+trilogy/core/processing/node_generators/select_merge_node.py,sha256=ORF9H7A-yT2wzQZYVex2asmm7_y0b2_lP6U0e48asNA,25290
 trilogy/core/processing/node_generators/select_node.py,sha256=Ta1G39V94gjX_AgyZDz9OqnwLz4BjY3D6Drx9YpziMQ,3555
 trilogy/core/processing/node_generators/synonym_node.py,sha256=AnAsa_Wj50NJ_IK0HSgab_7klYmKVrv0WI1uUe-GvEY,3766
 trilogy/core/processing/node_generators/union_node.py,sha256=NxQbnRRoYMI4WjMeph41yk4E6yipj53qdGuNt-Mozxw,2818
-trilogy/core/processing/node_generators/unnest_node.py,sha256=7uOZzBidEEKeZE0VW_XlgHGhEYf_snEHtV8UgJ_ZjyY,4048
+trilogy/core/processing/node_generators/unnest_node.py,sha256=u_hVHFYMz-ZylDdHH9mhFSRpxuKcTGvrrOP0rxrY_Xg,3901
 trilogy/core/processing/node_generators/window_node.py,sha256=A90linr4pkZtTNfn9k2YNLqrJ_SFII3lbHxB-BC6mI8,6688
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=m2YQ4OmG0N2O61a7NEq1ZzbTa7JsCC00lxB2ymjcYRI,8224
@@ -119,8 +119,8 @@ trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.104.dist-info/METADATA,sha256=IJmkrwnxe7gz3s89ZYVrDe6SkRY2cf6xNpmj5GTXkSE,11839
-pytrilogy-0.0.3.104.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.104.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.104.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.104.dist-info/RECORD,,
+pytrilogy-0.0.3.106.dist-info/METADATA,sha256=NM64Zgq3r16YOeN1tu1QI2sM-NyoLm86gnecFfRGQVs,11839
+pytrilogy-0.0.3.106.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.106.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.106.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.106.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.104"
+__version__ = "0.0.3.106"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/processing/discovery_utility.py

@@ -27,18 +27,22 @@ def calculate_effective_parent_grain(
 ) -> BuildGrain:
     # calculate the effective grain of the parent node
     # this is the union of all parent grains
-    if isinstance(node, MergeNode):
+    if isinstance(node, QueryDatasource):
         grain = BuildGrain()
-        qds = node.resolve()
+        qds = node
         if not qds.joins:
             return qds.datasources[0].grain
+        seen = set()
         for join in qds.joins:
             if isinstance(join, UnnestJoin):
+                grain += BuildGrain(components=set([x.address for x in join.concepts]))
                 continue
             pairs = join.concept_pairs or []
             for key in pairs:
                 left = key.existing_datasource
+                logger.info(f"adding left grain {left.grain} for join key {key.left}")
                 grain += left.grain
+                seen.add(left.name)
             keys = [key.right for key in pairs]
             join_grain = BuildGrain.from_concepts(keys)
             if join_grain == join.right_datasource.grain:
@@ -48,6 +52,24 @@ def calculate_effective_parent_grain(
                     f"join changes grain, adding {join.right_datasource.grain} to {grain}"
                 )
                 grain += join.right_datasource.grain
+            seen.add(join.right_datasource.name)
+        for x in qds.datasources:
+            # if we haven't seen it, it's still contributing to grain
+            # unless used ONLY in a subselect
+            # so the existence check is a [bad] proxy for that
+            if x.name not in seen and not (
+                qds.condition
+                and qds.condition.existence_arguments
+                and any(
+                    [
+                        c.address in block
+                        for c in x.output_concepts
+                        for block in qds.condition.existence_arguments
+                    ]
+                )
+            ):
+                logger.info(f"adding unjoined grain {x.grain} for datasource {x.name}")
+                grain += x.grain
         return grain
     else:
         return node.grain or BuildGrain()
@@ -75,7 +97,7 @@ def check_if_group_required(
     if comp_grain.issubset(target_grain):
 
         logger.info(
-            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, target: {target_grain}, grain is subset of target, no group node required"
+            f"{padding}{LOGGER_PREFIX} Group requirement check:  {comp_grain}, target: {target_grain}, grain is subset of target, no group node required"
         )
         return GroupRequiredResponse(target_grain, comp_grain, False)
     # find out what extra is in the comp grain vs target grain

trilogy/core/processing/node_generators/filter_node.py

@@ -4,7 +4,6 @@ from trilogy.constants import logger
 from trilogy.core.models.build import (
     BuildConcept,
     BuildFilterItem,
-    BuildGrain,
     BuildWhereClause,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
@@ -238,10 +237,10 @@ def gen_filter_node(
         if not parent.preexisting_conditions == where.conditional:
             parent.add_condition(where.conditional)
         parent.add_existence_concepts(flattened_existence, False)
-        parent.grain = BuildGrain.from_concepts(
-            parent.output_concepts,
-            environment=environment,
-        )
+        # parent.grain = BuildGrain.from_concepts(
+        #     parent.output_concepts,
+        #     environment=environment,
+        # )
         parent.rebuild_cache()
         filter_node = parent
     else:

trilogy/core/processing/node_generators/select_merge_node.py

@@ -224,6 +224,72 @@ def create_pruned_concept_graph(
     return g
 
 
+# def deduplicate_nodes(subgraph: nx.DiGraph, nodes: list[str], partial_map: dict[str, list[str]], depth: int) -> list[str]:
+#     """
+#     Remove duplicate datasource nodes that are connected to the same concepts
+#     and have the same partial state, keeping the one with the most unique concepts.
+
+#     Args:
+#         subgraph: NetworkX DiGraph containing the nodes and edges
+#         nodes: List of node names to deduplicate
+#         partial_map: Map of datasource to partial nodes
+
+#     Returns:
+#         List of deduplicated node names
+#     """
+#     # Filter for datasource nodes only
+#     ds_nodes = [node for node in nodes if node.startswith("ds~")]
+#     non_ds_nodes = [node for node in nodes if not node.startswith("ds~")]
+
+#     if len(ds_nodes) <= 1:
+#         return nodes  # No deduplication needed
+
+#     # Build a map of each datasource to its connected concepts and partial state
+#     ds_info = {}
+
+#     for ds_node in ds_nodes:
+#         # Get connected concept nodes (nodes starting with "c~")
+#         connected_concepts = set()
+#         for neighbor in subgraph.neighbors(ds_node):
+#             if neighbor.startswith("c~"):
+#                 connected_concepts.add(neighbor)
+
+#         # Get partial state for this datasource
+#         partial_state = tuple(sorted(partial_map.get(ds_node, [])))
+
+#         ds_info[ds_node] = {
+#             'concepts': connected_concepts,
+#             'partial_state': partial_state
+#         }
+
+#     # Find datasources to remove (those that are subsets of others)
+#     nodes_to_remove = set()
+#     logger.info('LOOK HERE')
+#     logger.info(ds_info)
+#     for ds_a, info_a in ds_info.items():
+#         for ds_b, info_b in ds_info.items():
+#             if ds_a != ds_b and ds_a not in nodes_to_remove:
+#                 # Check if ds_a is a subset of ds_b (same partial state and concepts are subset)
+#                 if (info_a['partial_state'] == info_b['partial_state'] and
+#                     info_a['concepts'].issubset(info_b['concepts']) and
+#                     len(info_a['concepts']) < len(info_b['concepts'])):
+#                     # ds_a connects to fewer concepts than ds_b, so remove ds_a
+#                     nodes_to_remove.add(ds_a)
+#                 elif (info_a['partial_state'] == info_b['partial_state'] and
+#                       info_a['concepts'] == info_b['concepts']):
+#                     # Exact same concepts and partial state - keep one arbitrarily
+#                     # (keep the lexicographically smaller one for consistency)
+#                     if ds_a > ds_b:
+#                         nodes_to_remove.add(ds_a)
+
+#     # Keep datasource nodes that weren't marked for removal
+#     logger.info(f"{padding(depth)}{LOGGER_PREFIX} Removing duplicate datasource nodes: {nodes_to_remove}")
+#     deduplicated_ds_nodes = [ds for ds in ds_nodes if ds not in nodes_to_remove]
+
+#     # Return deduplicated datasource nodes plus all non-datasource nodes
+#     return deduplicated_ds_nodes + non_ds_nodes
+
+
 def resolve_subgraphs(
     g: ReferenceGraph,
     relevant: list[BuildConcept],

trilogy/core/processing/node_generators/unnest_node.py

@@ -4,7 +4,6 @@ from trilogy.constants import logger
 from trilogy.core.models.build import (
     BuildConcept,
     BuildFunction,
-    BuildGrain,
     BuildWhereClause,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
@@ -104,10 +103,6 @@ def gen_unnest_node(
         preexisting_conditions=(
             conditional if conditional and local_conditions is False else None
         ),
-        grain=BuildGrain.from_concepts(
-            concepts=base.output_concepts,
-            environment=environment,
-        ),
     )
     # qds = new.resolve()
     # assert qds.source_map[concept.address] == {base.resolve()}

trilogy/core/processing/utility.py

@@ -90,13 +90,86 @@ class GroupRequiredResponse:
     required: bool
 
 
+def find_all_connecting_concepts(g: nx.Graph, ds1: str, ds2: str) -> set[str]:
+    """Find all concepts that connect two datasources"""
+    concepts1 = set(g.neighbors(ds1))
+    concepts2 = set(g.neighbors(ds2))
+    return concepts1 & concepts2
+
+
+def get_connection_keys(
+    all_connections: dict[tuple[str, str], set[str]], left: str, right: str
+) -> set[str]:
+    """Get all concepts that connect two datasources"""
+    lookup = sorted([left, right])
+    key: tuple[str, str] = (lookup[0], lookup[1])
+    return all_connections.get(key, set())
+
+
+def get_join_type(
+    left: str,
+    right: str,
+    partials: dict[str, list[str]],
+    nullables: dict[str, list[str]],
+    all_connecting_keys: set[str],
+) -> JoinType:
+    left_is_partial = any(key in partials.get(left, []) for key in all_connecting_keys)
+    left_is_nullable = any(
+        key in nullables.get(left, []) for key in all_connecting_keys
+    )
+    right_is_partial = any(
+        key in partials.get(right, []) for key in all_connecting_keys
+    )
+    right_is_nullable = any(
+        key in nullables.get(right, []) for key in all_connecting_keys
+    )
+
+    if left_is_nullable and right_is_nullable:
+        join_type = JoinType.FULL
+    elif left_is_partial and right_is_partial:
+        join_type = JoinType.FULL
+    elif left_is_partial:
+        join_type = JoinType.FULL
+    elif right_is_nullable:
+        join_type = JoinType.RIGHT_OUTER
+    elif right_is_partial or left_is_nullable:
+        join_type = JoinType.LEFT_OUTER
+    # we can't inner join if the left was an outer join
+    else:
+        join_type = JoinType.INNER
+    return join_type
+
+
+def reduce_join_types(join_types: Set[JoinType]) -> JoinType:
+    final_join_type = JoinType.INNER
+    if any([x == JoinType.FULL for x in join_types]):
+        final_join_type = JoinType.FULL
+    elif any([x == JoinType.LEFT_OUTER for x in join_types]):
+        final_join_type = JoinType.LEFT_OUTER
+    elif any([x == JoinType.RIGHT_OUTER for x in join_types]):
+        final_join_type = JoinType.RIGHT_OUTER
+
+    return final_join_type
+
+
 def resolve_join_order_v2(
     g: nx.Graph, partials: dict[str, list[str]], nullables: dict[str, list[str]]
 ) -> list[JoinOrderOutput]:
     datasources = [x for x in g.nodes if x.startswith("ds~")]
     concepts = [x for x in g.nodes if x.startswith("c~")]
 
+    # Pre-compute all possible connections between datasources
+    all_connections: dict[tuple[str, str], set[str]] = {}
+    for i, ds1 in enumerate(datasources):
+        for ds2 in datasources[i + 1 :]:
+            connecting_concepts = find_all_connecting_concepts(g, ds1, ds2)
+            if connecting_concepts:
+                key = tuple(sorted([ds1, ds2]))
+                all_connections[key] = connecting_concepts
+
     output: list[JoinOrderOutput] = []
+
+    # create our map of pivots, or common join concepts
     pivot_map = {
         concept: [x for x in g.neighbors(concept) if x in datasources]
         for concept in concepts
@@ -108,8 +181,9 @@ def resolve_join_order_v2(
         )
     )
     solo = [x for x in pivot_map if len(pivot_map[x]) == 1]
-    eligible_left = set()
+    eligible_left: set[str] = set()
 
+    # while we have pivots, keep joining them in
     while pivots:
         next_pivots = [
             x for x in pivots if any(y in eligible_left for y in pivot_map[x])
@@ -120,7 +194,7 @@ def resolve_join_order_v2(
         else:
             root = pivots.pop(0)
 
-        # sort so less partials is last and eligible lefts are
+        # sort so less partials is last and eligible lefts are first
         def score_key(x: str) -> tuple[int, int, str]:
             base = 1
             # if it's left, higher weight
@@ -133,79 +207,56 @@ def resolve_join_order_v2(
                 base -= 1
             return (base, len(x), x)
 
-        # get remainig un-joined datasets
+        # get remaining un-joined datasets
         to_join = sorted(
             [x for x in pivot_map[root] if x not in eligible_left], key=score_key
         )
         while to_join:
             # need to sort this to ensure we join on the best match
-            base = sorted(
-                [x for x in pivot_map[root] if x in eligible_left], key=score_key
-            )
+            # but check ALL left in case there are non-pivt keys to join on
+            base = sorted([x for x in eligible_left], key=score_key)
             if not base:
                 new = to_join.pop()
                 eligible_left.add(new)
                 base = [new]
             right = to_join.pop()
             # we already joined it
-            # this could happen if the same pivot is shared with multiple Dses
+            # this could happen if the same pivot is shared with multiple DSes
             if right in eligible_left:
                 continue
+
             joinkeys: dict[str, set[str]] = {}
             # sorting puts the best candidate last for pop
             # so iterate over the reversed list
             join_types = set()
+
             for left_candidate in reversed(base):
-                common = nx.common_neighbors(g, left_candidate, right)
+                # Get all concepts that connect these two datasources
+                all_connecting_keys = get_connection_keys(
+                    all_connections, left_candidate, right
+                )
 
-                if not common:
+                if not all_connecting_keys:
                     continue
+
+                # Check if we already have this exact set of keys
                 exists = False
                 for _, v in joinkeys.items():
-                    if v == common:
+                    if v == all_connecting_keys:
                         exists = True
                 if exists:
                     continue
-                left_is_partial = any(
-                    key in partials.get(left_candidate, []) for key in common
-                )
-                left_is_nullable = any(
-                    key in nullables.get(left_candidate, []) for key in common
-                )
-                right_is_partial = any(key in partials.get(right, []) for key in common)
-                # we don't care if left is nullable for join type (just keys), but if we did
-                # left_is_nullable = any(
-                #     key in nullables.get(left_candidate, []) for key in common
-                # )
-                right_is_nullable = any(
-                    key in nullables.get(right, []) for key in common
-                )
-                if left_is_nullable and right_is_nullable:
-                    join_type = JoinType.FULL
-                elif left_is_partial and right_is_partial:
-                    join_type = JoinType.FULL
-                elif left_is_partial:
-                    join_type = JoinType.FULL
-                elif right_is_nullable:
-                    join_type = JoinType.RIGHT_OUTER
-                elif right_is_partial or left_is_nullable:
-                    join_type = JoinType.LEFT_OUTER
-                # we can't inner join if the left was an outer join
-                else:
-                    join_type = JoinType.INNER
 
+                join_type = get_join_type(
+                    left_candidate, right, partials, nullables, all_connecting_keys
+                )
                 join_types.add(join_type)
-                joinkeys[left_candidate] = common
-            final_join_type = JoinType.INNER
-            if any([x == JoinType.FULL for x in join_types]):
-                final_join_type = JoinType.FULL
-            elif any([x == JoinType.LEFT_OUTER for x in join_types]):
-                final_join_type = JoinType.LEFT_OUTER
-            elif any([x == JoinType.RIGHT_OUTER for x in join_types]):
-                final_join_type = JoinType.RIGHT_OUTER
+                joinkeys[left_candidate] = all_connecting_keys
+
+            final_join_type = reduce_join_types(join_types)
+
             output.append(
                 JoinOrderOutput(
-                    # left=left_candidate,
                     right=right,
                     type=final_join_type,
                     keys=joinkeys,
@@ -216,7 +267,6 @@ def resolve_join_order_v2(
     for concept in solo:
         for ds in pivot_map[concept]:
             # if we already have it, skip it
-
             if ds in eligible_left:
                 continue
             # if we haven't had ANY left datasources yet
@@ -224,17 +274,39 @@ def resolve_join_order_v2(
             if not eligible_left:
                 eligible_left.add(ds)
                 continue
-            # otherwise do a full out join
-            output.append(
-                JoinOrderOutput(
-                    # pick random one to be left
-                    left=list(eligible_left)[0],
-                    right=ds,
-                    type=JoinType.FULL,
-                    keys={},
+            # otherwise do a full outer join
+            # Try to find if there are any connecting keys with existing left tables
+            best_left = None
+            best_keys: set[str] = set()
+            for existing_left in eligible_left:
+                connecting_keys = get_connection_keys(
+                    all_connections, existing_left, ds
+                )
+                if connecting_keys and len(connecting_keys) > len(best_keys):
+                    best_left = existing_left
+                    best_keys = connecting_keys
+
+            if best_left and best_keys:
+                output.append(
+                    JoinOrderOutput(
+                        left=best_left,
+                        right=ds,
+                        type=JoinType.FULL,
+                        keys={best_left: best_keys},
+                    )
+                )
+            else:
+                output.append(
+                    JoinOrderOutput(
+                        # pick random one to be left
+                        left=list(eligible_left)[0],
+                        right=ds,
+                        type=JoinType.FULL,
+                        keys={},
+                    )
                 )
-            )
             eligible_left.add(ds)
+
     # only once we have all joins
     # do we know if some inners need to be left outers
     for review_join in output:
@@ -248,6 +320,7 @@ def resolve_join_order_v2(
             ]
         ):
             review_join.type = JoinType.LEFT_OUTER
+
     return output
 
 
@@ -352,7 +425,9 @@ def resolve_instantiated_concept(
     )
 
 
-def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
+def reduce_concept_pairs(
+    input: list[ConceptPair], right_source: QueryDatasource | BuildDatasource
+) -> list[ConceptPair]:
     left_keys = set()
     right_keys = set()
     for pair in input:
@@ -361,7 +436,10 @@ def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
         if pair.right.purpose == Purpose.KEY:
             right_keys.add(pair.right.address)
     final: list[ConceptPair] = []
+    seen_right_keys = set()
     for pair in input:
+        if pair.right.address in seen_right_keys:
+            continue
         if (
             pair.left.purpose == Purpose.PROPERTY
             and pair.left.keys
@@ -374,7 +452,15 @@ def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
             and pair.right.keys.issubset(right_keys)
         ):
             continue
+
+        seen_right_keys.add(pair.right.address)
         final.append(pair)
+    all_keys = set([x.right.address for x in final])
+    if right_source.grain.components and right_source.grain.components.issubset(
+        all_keys
+    ):
+        return [x for x in final if x.right.address in right_source.grain.components]
+
     return final
 
 
@@ -443,7 +529,8 @@ def get_node_joins(
                     )
                     for k, v in j.keys.items()
                     for concept in v
-                ]
+                ],
+                ds_node_map[j.right],
             ),
         )
         for j in joins