pytrilogy 0.0.3.97__tar.gz → 0.0.3.98__tar.gz

{pytrilogy-0.0.3.97/pytrilogy.egg-info → pytrilogy-0.0.3.98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.97
+Version: 0.0.3.98
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.97
+Version: 0.0.3.98
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/pytrilogy.egg-info/SOURCES.txt

@@ -149,6 +149,7 @@ trilogy/std/__init__.py
 trilogy/std/date.preql
 trilogy/std/display.preql
 trilogy/std/geography.preql
+trilogy/std/metric.preql
 trilogy/std/money.preql
 trilogy/std/net.preql
 trilogy/std/ranking.preql

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.97"
+__version__ = "0.0.3.98"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/core/functions.py

@@ -427,7 +427,7 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
             {DataType.STRING},
         ],
         output_purpose=Purpose.PROPERTY,
-        output_type=DataType.BOOL,
+        output_type=DataType.STRING,
         arg_count=2,
     ),
     FunctionType.SUBSTRING: FunctionConfig(

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/core/models/execute.py

@@ -151,7 +151,6 @@ class CTE(BaseModel):
             ]
         ):
             return False
-
         self.source.datasources = [
             ds_being_inlined,
             *[

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,4 +1,5 @@
-from typing import List, Optional
+from itertools import combinations
+from typing import Callable, List, Optional
 
 import networkx as nx
 from networkx.algorithms import approximation as ax
@@ -11,7 +12,12 @@ from trilogy.core.graph_models import (
     concept_to_node,
     prune_sources_for_conditions,
 )
-from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildConditional,
+    BuildGrain,
+    BuildWhereClause,
+)
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
@@ -29,7 +35,10 @@ def filter_pseudonyms_for_source(
         if edge in pseudonyms:
             lengths = {}
             for n in edge:
-                lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+                try:
+                    lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+                except nx.NetworkXNoPath:
+                    lengths[n] = 999
             to_remove.add(max(lengths, key=lambda x: lengths.get(x, 0)))
     for node in to_remove:
         ds_graph.remove_node(node)
@@ -84,12 +93,104 @@ def extract_ds_components(
     return graphs
 
 
+def prune_and_merge(
+    G: nx.DiGraph,
+    keep_node_lambda: Callable[[str], bool],
+) -> nx.DiGraph:
+    """
+    Prune nodes of one type and create direct connections between remaining nodes.
+
+    Args:
+        G: NetworkX graph
+        keep_node_type: The node type to keep
+        node_type_attr: Attribute name that stores node type
+
+    Returns:
+        New graph with only keep_node_type nodes and merged connections
+    """
+    # Get nodes to keep
+    nodes_to_keep = [n for n in G.nodes if keep_node_lambda(n)]
+    # Create new graph
+    new_graph = G.subgraph(nodes_to_keep).copy()
+
+    # Find paths between nodes to keep through removed nodes
+    nodes_to_remove = [n for n in G.nodes() if n not in nodes_to_keep]
+
+    for node_pair in combinations(nodes_to_keep, 2):
+        n1, n2 = node_pair
+
+        # Check if there's a path through removed nodes
+        try:
+            path = nx.shortest_path(G, n1, n2)
+            # If path exists and goes through nodes we're removing
+            if len(path) > 2 or any(node in nodes_to_remove for node in path[1:-1]):
+                new_graph.add_edge(n1, n2)
+        except nx.NetworkXNoPath:
+            continue
+
+    return new_graph
+
+
+def reinject_common_join_keys_v2(
+    G: ReferenceGraph,
+    final: nx.DiGraph,
+    nodelist: list[str],
+    synonyms: set[str] = set(),
+) -> bool:
+    # when we've discovered a concept join, for each pair of ds nodes
+    # check if they have more keys in common
+    # and inject those to discovery as join conditions
+    def is_ds_node(n: str) -> bool:
+        return n.startswith("ds~")
+
+    ds_graph = prune_and_merge(final, is_ds_node)
+    injected = False
+    for datasource in ds_graph.nodes:
+        node1 = G.datasources[datasource]
+        neighbors = nx.all_neighbors(ds_graph, datasource)
+        for neighbor in neighbors:
+            node2 = G.datasources[neighbor]
+            common_concepts = set(
+                x.concept.address for x in node1.columns
+            ).intersection(set(x.concept.address for x in node2.columns))
+            concrete_concepts = [
+                x.concept for x in node1.columns if x.concept.address in common_concepts
+            ]
+            reduced = BuildGrain.from_concepts(concrete_concepts).components
+            existing_addresses = set()
+            for concrete in concrete_concepts:
+                logger.info(
+                    f"looking at column {concrete.address} with pseudonyms {concrete.pseudonyms}"
+                )
+                cnode = concept_to_node(concrete.with_default_grain())
+                if cnode in final.nodes:
+                    existing_addresses.add(concrete.address)
+                    continue
+            for concrete in concrete_concepts:
+                if concrete.address in synonyms:
+                    continue
+                if concrete.address not in reduced:
+                    continue
+                # skip anything that is already in the graph pseudonyms
+                if any(x in concrete.pseudonyms for x in existing_addresses):
+                    continue
+                cnode = concept_to_node(concrete.with_default_grain())
+                final.add_edge(datasource, cnode)
+                final.add_edge(neighbor, cnode)
+                logger.info(
+                    f"{LOGGER_PREFIX} reinjecting common join key {cnode} between {datasource} and {neighbor}"
+                )
+                injected = True
+    return injected
+
+
 def determine_induced_minimal_nodes(
     G: ReferenceGraph,
     nodelist: list[str],
     environment: BuildEnvironment,
     filter_downstream: bool,
     accept_partial: bool = False,
+    synonyms: set[str] = set(),
 ) -> nx.DiGraph | None:
     H: nx.Graph = nx.to_undirected(G).copy()
     nodes_to_remove = []
@@ -129,7 +230,7 @@ def determine_induced_minimal_nodes(
         return None
     path_removals = list(x for x in H.nodes if x not in paths)
     if path_removals:
-        logger.debug(f"Removing paths {path_removals} from graph")
+        # logger.debug(f"Removing paths {path_removals} from graph")
         H.remove_nodes_from(path_removals)
     # logger.debug(f"Graph after path removal {H.nodes}")
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
@@ -148,8 +249,10 @@ def determine_induced_minimal_nodes(
                 if not accept_partial:
                     continue
             final.add_edge(*edge)
-    # all concept nodes must have a parent
 
+    reinject_common_join_keys_v2(G, final, nodelist, synonyms)
+
+    # all concept nodes must have a parent
     if not all(
         [
             final.in_degree(node) > 0
@@ -302,6 +405,7 @@ def resolve_weak_components(
                 filter_downstream=filter_downstream,
                 accept_partial=accept_partial,
                 environment=environment,
+                synonyms=synonyms,
             )
 
             if not g or not g.nodes:

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/core/validation/concept.py

@@ -26,6 +26,15 @@ def validate_key_concept(
 ):
     results: list[ValidationTest] = []
     seen: dict[str, int] = {}
+
+    count = 0
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            count += 1
+    # if it only has one source, it's a key
+    if count <= 1:
+        return results
+
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/core/validation/environment.py

@@ -69,4 +69,5 @@ def validate_environment(
                 f"Environment validation failed with the following errors:\n{messages}",
                 children=exceptions,
             )
+
     return results

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/dialect/common.py

@@ -10,7 +10,6 @@ from trilogy.core.models.build import (
     BuildParamaterizedConceptReference,
     BuildParenthetical,
 )
-from trilogy.core.models.datasource import RawColumnExpr
 from trilogy.core.models.execute import (
     CTE,
     InstantiatedUnnestJoin,
@@ -65,15 +64,8 @@ def render_join_concept(
     inlined_ctes: set[str],
 ):
     if cte.name in inlined_ctes:
-        ds = cte.source.datasources[0]
-        raw_content = ds.get_alias(concept)
-        if isinstance(raw_content, RawColumnExpr):
-            rval = raw_content.text
-            return rval
-        elif isinstance(raw_content, BuildFunction):
-            rval = render_expr(raw_content, cte=cte)
-            return rval
-        return f"{quote_character}{name}{quote_character}.{quote_character}{raw_content}{quote_character}"
+        base = render_expr(concept, cte)
+        return base
     return f"{quote_character}{name}{quote_character}.{quote_character}{concept.safe_address}{quote_character}"
 
 

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/dialect/metadata.py

@@ -174,7 +174,7 @@ def raw_validation_to_result(
 ) -> Optional[MockResult]:
     """Convert raw validation tests to mock result."""
     if not raw:
-        return None
+        return MockResult([], ["check_type", "expected", "result", "ran", "query"])
     output = []
     for row in raw:
         if row.raw_query and generator and not row.generated_query:

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/executor.py

@@ -410,6 +410,13 @@ class Executor(object):
                 )
                 output.extend(results)
                 continue
+            elif isinstance(statement, ProcessedValidateStatement):
+                validate_result = handle_processed_validate_statement(
+                    statement, self.generator, self.validate_environment
+                )
+                if validate_result:
+                    output.append(validate_result)
+                continue
             if non_interactive:
                 if not isinstance(
                     statement, (ProcessedCopyStatement, ProcessedQueryPersist)

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/parsing/parse_engine.py

@@ -1002,7 +1002,7 @@ class ParseToObjects(Transformer):
     def validate_statement(self, meta: Meta, args) -> ValidateStatement:
         if len(args) == 2:
             scope = args[0]
-            targets = args[1]
+            targets = args[1].split(",")
         elif len(args) == 0:
             scope = ValidationScope.ALL
             targets = None
@@ -2295,7 +2295,7 @@ def parse_text(
             raise _create_syntax_error(210, pos, text)
 
         # Handle FROM token error
-        parsed_tokens = [x.value for x in e.token_history] if e.token_history else []
+        parsed_tokens = [x.value for x in e.token_history if x] if e.token_history else []
         if parsed_tokens == ["FROM"]:
             raise _create_syntax_error(101, pos, text)
 

{pytrilogy-0.0.3.97 → pytrilogy-0.0.3.98}/trilogy/parsing/render.py

@@ -546,8 +546,8 @@ class Renderer:
             return f"{self.to_string(arg.arguments[0])}[{self.to_string(arg.arguments[1])}]"
 
         if arg.operator == FunctionType.CASE:
-            inputs = "\n".join(args)
-            return f"CASE {inputs}\nEND"
+            inputs = "\n\t".join(args)
+            return f"CASE\n\t{inputs}\nEND"
         return f"{arg.operator.value}({inputs})"
 
     @to_string.register

pytrilogy-0.0.3.98/trilogy/std/metric.preql

@@ -0,0 +1,15 @@
+# Length and distance units
+type m numeric;      # meters
+type km numeric;     # kilometers
+type cm numeric;     # centimeters
+type mm numeric;     # millimeters
+
+# Mass units
+type kg numeric;     # kilograms
+type g numeric;      # grams
+type tonne numeric;  # metric tons (1000 kg)
+
+# Force units
+type n numeric;      # newtons
+type kn numeric;     # kilonewtons (1000 N)
+type mn numeric;     # meganewtons (1,000,000 N)