pytrilogy 0.0.3.96__py3-none-any.whl → 0.0.3.98__py3-none-any.whl

{pytrilogy-0.0.3.96.dist-info → pytrilogy-0.0.3.98.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.96
+Version: 0.0.3.98
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -334,7 +334,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96.dist-info → pytrilogy-0.0.3.98.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pytrilogy-0.0.3.96.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=hBxtfxlbUvTLp_8FCY_-wDqJM7G2RJQ2jntfMf3a0PM,303
-trilogy/constants.py,sha256=eKb_EJvSqjN9tGbdVEViwdtwwh8fZ3-jpOEDqL71y70,1691
+pytrilogy-0.0.3.98.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=NSBY37z8wVkVSFPfHPo0BTvi3hO7KMw7ENbaoX9PqBY,303
+trilogy/constants.py,sha256=SSsRMg9HTou259nMKAw-rJNBgzkWjQ3QIQXcrq9i5Kk,1717
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
-trilogy/executor.py,sha256=YfSjuJ0FVm2gHnNgmUlXijWDTUFjqq9FNakWpeEYO48,15769
+trilogy/executor.py,sha256=0yggm9Ejl1DFELUtRaPxbaU5mpKqYMHZlzSMXOFmODE,16111
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/render.py,sha256=qQWwduymauOlB517UtM-VGbVe8Cswa4UJub5aGbSO6c,1512
@@ -10,12 +10,12 @@ trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
 trilogy/authoring/__init__.py,sha256=TABMOETSMERrWuyDLR0nK4ISlqR0yaqeXrmuOdrSvAY,3060
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=nizWYDCJQ1bigQMtkNIEMNTcN0NoEAXiIHLzpelxQ24,201
-trilogy/core/enums.py,sha256=EusAzz7o_YrWf64TLIED7MfziFOJk8EHM8se5W3nyJk,8644
+trilogy/core/enums.py,sha256=H8I2Dz4POHZ4ixYCGzNs4c3KDqxLQklGLVfmje1DSMo,8877
 trilogy/core/env_processor.py,sha256=H-rr2ALj31l5oh3FqeI47Qju6OOfiXBacXNJGNZ92zQ,4521
 trilogy/core/environment_helpers.py,sha256=TRlqVctqIRBxzfjRBmpQsAVoiCcsEKBhG1B6PUE0l1M,12743
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
-trilogy/core/exceptions.py,sha256=0Lmc3awJYx94k6uifbHc-EIqlFGV6YrX0QIwP84D4a4,1150
-trilogy/core/functions.py,sha256=ESUWMRmwtavwCLl6z1NP9EFzWTJoXn3orTaaOSsj33Q,33093
+trilogy/core/exceptions.py,sha256=fI16oTNCVMMAJFSn2AFzZVapzsF5M9WbdN5e5UixwXc,2807
+trilogy/core/functions.py,sha256=oY-F0hsA9vp1ZipGTyx4QVtz_x83Ekk-lkHv6mMkHVQ,33095
 trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
 trilogy/core/optimization.py,sha256=ojpn-p79lr03SSVQbbw74iPCyoYpDYBmj1dbZ3oXCjI,8860
@@ -28,7 +28,7 @@ trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYy
 trilogy/core/models/core.py,sha256=EofJ8-kltNr_7oFhyCPqauVX1bSJzJI5xOp0eMP_vlA,12892
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
 trilogy/core/models/environment.py,sha256=hwTIRnJgaHUdCYof7U5A9NPitGZ2s9yxqiW5O2SaJ9Y,28759
-trilogy/core/models/execute.py,sha256=k2--2xUNuoaObkzutYaS5sdUFnY9zT_UKdU2rViq9XQ,42106
+trilogy/core/models/execute.py,sha256=lQTpiuNhBT4In-oQ76ImgIoTdUbs4mmyd0J0iTOZOdw,42105
 trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
@@ -49,7 +49,7 @@ trilogy/core/processing/node_generators/filter_node.py,sha256=ArBsQJl-4fWBJWCE28
 trilogy/core/processing/node_generators/group_node.py,sha256=8HJ1lkOvIXfX3xoS2IMbM_wCu_mT0J_hQ7xnTaxsVlo,6611
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=83FkcYuOFyDY0_0NWhL45MAT5J_6Y6L1h357WrJPzaI,18230
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=cBcZm3AUfx4K70MIhU0T9iKMxw_qZRVFhQ0i6qMQeoQ,21999
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
 trilogy/core/processing/node_generators/rowset_node.py,sha256=5L5u6xz1In8EaHQdcYgR2si-tz9WB9YLXURo4AkUT9A,6630
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
@@ -76,19 +76,20 @@ trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 trilogy/core/statements/common.py,sha256=VnVLULQg1TJLNUFzJaROT1tsf2ewk3IpuhvZaP36R6A,535
 trilogy/core/statements/execute.py,sha256=kiwJcVeMa4wZR-xLfM2oYOJ9DeyJkP8An38WFyJxktM,2413
 trilogy/core/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/validation/common.py,sha256=cVbDSowtLf2nl0-QVmNauAeLBBNFkSE5bRZtTHIzW20,3193
-trilogy/core/validation/concept.py,sha256=23wZYw_cGmTQuFvaRM-0T7M2b5ZwqjFMucfvfzyQxlc,4425
-trilogy/core/validation/datasource.py,sha256=HIk7iEKK99k0-WXosiIhedH-U2rDGYGdUMKEt1eMl1w,6394
-trilogy/core/validation/environment.py,sha256=yjSnEH893mTiW9o6YXBtXJWbGSR2kMQWjszDuECznLs,2784
+trilogy/core/validation/common.py,sha256=Sd-towAX1uSDe3dK51FcVtIwVrMhayEwdHqhzeJHro0,4776
+trilogy/core/validation/concept.py,sha256=PM2BxBxLvuBScSWZMPsDZVcOblDil5pNT0pHLcLhdPA,5242
+trilogy/core/validation/datasource.py,sha256=d9AQNcukIRgN2spItPsXFiNtlZva-lDnfei3i06yQCE,6489
+trilogy/core/validation/environment.py,sha256=ymvhQyt7jLK641JAAIQkqjQaAmr9C5022ILzYvDgPP0,2835
+trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=m2a8azbI3AWfQz-VtSn84H5T-BqjF5PULH6BrgwZzok,49666
+trilogy/dialect/base.py,sha256=0QVHv4F0t3_gRQrZ0woFoUNKu7vaXGo-BG1l47CZUKc,49698
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
-trilogy/dialect/common.py,sha256=tSthIZOXXRPQ4KeMKnDDsH7KlTmf2EVqigVtLyoc4zc,6071
+trilogy/dialect/common.py,sha256=_MarnMWRBn3VcNt3k5VUdFrwH6oHzGdNQquSpHNLq4o,5644
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
 trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
 trilogy/dialect/duckdb.py,sha256=JoUvQ19WvgxoaJkGLM7DPXOd1H0394k3vBiblksQzOI,5631
 trilogy/dialect/enums.py,sha256=FRNYQ5-w-B6-X0yXKNU5g9GowsMlERFogTC5u2nxL_s,4740
-trilogy/dialect/metadata.py,sha256=Vt4-p82bD1ijqeoI2dagUVUbC-KgNNJ2MvDwQIa5mG8,7034
+trilogy/dialect/metadata.py,sha256=p_V-MYPQ2iR6fcTjagnptCIWtsZe4fTfoS_iXpavPzY,7098
 trilogy/dialect/postgres.py,sha256=el2PKwfyvWGk5EZtLudqAH5ewLitY1sFHJiocBSyxyM,3393
 trilogy/dialect/presto.py,sha256=k1IaeilR3nzPC9Hp7jlAdzJ7TsuxB3LQTBQ28MYE7O8,3715
 trilogy/dialect/snowflake.py,sha256=T6_mKfhpDazB1xQxqFLS2AJwzwzBcPYY6_qxRnAtFBs,3326
@@ -103,21 +104,22 @@ trilogy/parsing/common.py,sha256=550-L0444GUuBFdiDWkOg_DxnMXtcJFUMES2R5zlwik,310
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=fZGFNN7PEVJ7_tk0GxBHJQerIJTifbrjoMcHfTR_TYk,81647
-trilogy/parsing/render.py,sha256=HSNntD82GiiwHT-TWPLXAaIMWLYVV5B5zQEsgwrHIBE,19605
-trilogy/parsing/trilogy.lark,sha256=9-SMrKFGZLdBTNheK1szif0VYOIt5m0xhVd8pOFCByU,16267
+trilogy/parsing/parse_engine.py,sha256=-3_-EoiciWGOlylKurSPlG7gYSbScjneBup7ZvjDz-c,81800
+trilogy/parsing/render.py,sha256=tqB3GlGk3bX6AbkJjvADad2QH6n63nw1kgrpjzLX2tI,20520
+trilogy/parsing/trilogy.lark,sha256=rM4WleeyGhoRgU-FOGcaeHOzZcYVxN4f13e_3B4OeLQ,16389
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/std/date.preql,sha256=HWZm4t4HWyxr5geWRsY05RnHBVDMci8z8YA2cu0-OOw,188
 trilogy/std/display.preql,sha256=nm7lox87Xf6lBvXCVCS6x2HskguMKzndEBucJ5pktzk,175
 trilogy/std/geography.preql,sha256=1A9Sq5PPMBnEPPf7f-rPVYxJfsnWpQ8oV_k4Fm3H2dU,675
+trilogy/std/metric.preql,sha256=DRECGhkMyqfit5Fl4Ut9zbWrJuSMI1iO9HikuyoBpE0,421
 trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.96.dist-info/METADATA,sha256=l4yiGzDzMYd4of8nPXBMBKyetasPk4yfIOyENWjEqcU,11023
-pytrilogy-0.0.3.96.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.96.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.96.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.96.dist-info/RECORD,,
+pytrilogy-0.0.3.98.dist-info/METADATA,sha256=8FfhicjKv1X47j0nL557PM_b1TisXXmAz3DG2B-PBdI,11683
+pytrilogy-0.0.3.98.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.98.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.98.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.98.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.96"
+__version__ = "0.0.3.98"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -16,6 +16,7 @@ ENV_CACHE_NAME = ".preql_cache.json"
 
 class MagicConstants(Enum):
     NULL = "null"
+    LINE_SEPARATOR = "\n"
 
 
 NULL_VALUE = MagicConstants.NULL

trilogy/core/enums.py

@@ -82,6 +82,15 @@ class Modifier(Enum):
             return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
+    def __lt__(self, other):
+        order = [
+            Modifier.HIDDEN,
+            Modifier.PARTIAL,
+            Modifier.NULLABLE,
+            Modifier.OPTIONAL,
+        ]
+        return order.index(self) < order.index(other)
+
 
 class JoinType(Enum):
     INNER = "inner"

trilogy/core/exceptions.py

@@ -1,4 +1,15 @@
-from typing import List, Sequence
+from dataclasses import dataclass
+from typing import Any, List, Sequence
+
+from trilogy.core.enums import Modifier
+from trilogy.core.models.core import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+)
 
 
 class UndefinedConceptException(Exception):
@@ -29,7 +40,7 @@ class ModelValidationError(Exception):
         self,
         message,
         children: Sequence["ModelValidationError"] | None = None,
-        **kwargs
+        **kwargs,
     ):
         super().__init__(self, message, **kwargs)
         self.message = message
@@ -40,6 +51,49 @@ class DatasourceModelValidationError(ModelValidationError):
     pass
 
 
+class DatasourceGrainValidationError(DatasourceModelValidationError):
+    pass
+
+
+@dataclass
+class DatasourceColumnBindingData:
+    address: str
+    value: Any
+    value_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    value_modifiers: List[Modifier]
+    actual_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    actual_modifiers: List[Modifier]
+
+    def format_failure(self):
+        return f"Concept {self.address} value '{self.value}' with type {self.value_modifiers} does not conform to expected type {str(self.actual_type)} with modifiers {self.actual_modifiers}"
+
+    def is_modifier_issue(self) -> bool:
+        return len(self.value_modifiers) > 0 and any(
+            [x not in self.actual_modifiers for x in self.value_modifiers]
+        )
+
+    def is_type_issue(self) -> bool:
+        return self.value_type != self.actual_type
+
+
+class DatasourceColumnBindingError(DatasourceModelValidationError):
+    def __init__(
+        self,
+        address: str,
+        errors: list[DatasourceColumnBindingData],
+        message: str | None = None,
+    ):
+        if not message:
+            message = f"Datasource {address} failed validation. Found rows that do not conform to types: {[failure.format_failure() for failure in errors]}"
+        super().__init__(message)
+        self.errors = errors
+        self.dataset_address = address
+
+
 class ConceptModelValidationError(ModelValidationError):
     pass
 

trilogy/core/functions.py

@@ -427,7 +427,7 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
             {DataType.STRING},
         ],
         output_purpose=Purpose.PROPERTY,
-        output_type=DataType.BOOL,
+        output_type=DataType.STRING,
         arg_count=2,
     ),
     FunctionType.SUBSTRING: FunctionConfig(

trilogy/core/models/execute.py

@@ -151,7 +151,6 @@ class CTE(BaseModel):
             ]
         ):
             return False
-
         self.source.datasources = [
             ds_being_inlined,
             *[

trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,4 +1,5 @@
-from typing import List, Optional
+from itertools import combinations
+from typing import Callable, List, Optional
 
 import networkx as nx
 from networkx.algorithms import approximation as ax
@@ -11,7 +12,12 @@ from trilogy.core.graph_models import (
     concept_to_node,
     prune_sources_for_conditions,
 )
-from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildConditional,
+    BuildGrain,
+    BuildWhereClause,
+)
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
@@ -29,7 +35,10 @@ def filter_pseudonyms_for_source(
         if edge in pseudonyms:
             lengths = {}
             for n in edge:
-                lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+                try:
+                    lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+                except nx.NetworkXNoPath:
+                    lengths[n] = 999
             to_remove.add(max(lengths, key=lambda x: lengths.get(x, 0)))
     for node in to_remove:
         ds_graph.remove_node(node)
@@ -84,12 +93,104 @@ def extract_ds_components(
     return graphs
 
 
+def prune_and_merge(
+    G: nx.DiGraph,
+    keep_node_lambda: Callable[[str], bool],
+) -> nx.DiGraph:
+    """
+    Prune nodes of one type and create direct connections between remaining nodes.
+
+    Args:
+        G: NetworkX graph
+        keep_node_type: The node type to keep
+        node_type_attr: Attribute name that stores node type
+
+    Returns:
+        New graph with only keep_node_type nodes and merged connections
+    """
+    # Get nodes to keep
+    nodes_to_keep = [n for n in G.nodes if keep_node_lambda(n)]
+    # Create new graph
+    new_graph = G.subgraph(nodes_to_keep).copy()
+
+    # Find paths between nodes to keep through removed nodes
+    nodes_to_remove = [n for n in G.nodes() if n not in nodes_to_keep]
+
+    for node_pair in combinations(nodes_to_keep, 2):
+        n1, n2 = node_pair
+
+        # Check if there's a path through removed nodes
+        try:
+            path = nx.shortest_path(G, n1, n2)
+            # If path exists and goes through nodes we're removing
+            if len(path) > 2 or any(node in nodes_to_remove for node in path[1:-1]):
+                new_graph.add_edge(n1, n2)
+        except nx.NetworkXNoPath:
+            continue
+
+    return new_graph
+
+
+def reinject_common_join_keys_v2(
+    G: ReferenceGraph,
+    final: nx.DiGraph,
+    nodelist: list[str],
+    synonyms: set[str] = set(),
+) -> bool:
+    # when we've discovered a concept join, for each pair of ds nodes
+    # check if they have more keys in common
+    # and inject those to discovery as join conditions
+    def is_ds_node(n: str) -> bool:
+        return n.startswith("ds~")
+
+    ds_graph = prune_and_merge(final, is_ds_node)
+    injected = False
+    for datasource in ds_graph.nodes:
+        node1 = G.datasources[datasource]
+        neighbors = nx.all_neighbors(ds_graph, datasource)
+        for neighbor in neighbors:
+            node2 = G.datasources[neighbor]
+            common_concepts = set(
+                x.concept.address for x in node1.columns
+            ).intersection(set(x.concept.address for x in node2.columns))
+            concrete_concepts = [
+                x.concept for x in node1.columns if x.concept.address in common_concepts
+            ]
+            reduced = BuildGrain.from_concepts(concrete_concepts).components
+            existing_addresses = set()
+            for concrete in concrete_concepts:
+                logger.info(
+                    f"looking at column {concrete.address} with pseudonyms {concrete.pseudonyms}"
+                )
+                cnode = concept_to_node(concrete.with_default_grain())
+                if cnode in final.nodes:
+                    existing_addresses.add(concrete.address)
+                    continue
+            for concrete in concrete_concepts:
+                if concrete.address in synonyms:
+                    continue
+                if concrete.address not in reduced:
+                    continue
+                # skip anything that is already in the graph pseudonyms
+                if any(x in concrete.pseudonyms for x in existing_addresses):
+                    continue
+                cnode = concept_to_node(concrete.with_default_grain())
+                final.add_edge(datasource, cnode)
+                final.add_edge(neighbor, cnode)
+                logger.info(
+                    f"{LOGGER_PREFIX} reinjecting common join key {cnode} between {datasource} and {neighbor}"
+                )
+                injected = True
+    return injected
+
+
 def determine_induced_minimal_nodes(
     G: ReferenceGraph,
     nodelist: list[str],
     environment: BuildEnvironment,
     filter_downstream: bool,
     accept_partial: bool = False,
+    synonyms: set[str] = set(),
 ) -> nx.DiGraph | None:
     H: nx.Graph = nx.to_undirected(G).copy()
     nodes_to_remove = []
@@ -129,7 +230,7 @@ def determine_induced_minimal_nodes(
         return None
     path_removals = list(x for x in H.nodes if x not in paths)
     if path_removals:
-        logger.debug(f"Removing paths {path_removals} from graph")
+        # logger.debug(f"Removing paths {path_removals} from graph")
         H.remove_nodes_from(path_removals)
     # logger.debug(f"Graph after path removal {H.nodes}")
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
@@ -148,8 +249,10 @@ def determine_induced_minimal_nodes(
                 if not accept_partial:
                     continue
             final.add_edge(*edge)
-    # all concept nodes must have a parent
 
+    reinject_common_join_keys_v2(G, final, nodelist, synonyms)
+
+    # all concept nodes must have a parent
     if not all(
         [
             final.in_degree(node) > 0
@@ -302,6 +405,7 @@ def resolve_weak_components(
                 filter_downstream=filter_downstream,
                 accept_partial=accept_partial,
                 environment=environment,
+                synonyms=synonyms,
             )
 
             if not g or not g.nodes:

trilogy/core/validation/common.py

@@ -2,13 +2,25 @@ from dataclasses import dataclass
 from enum import Enum
 
 from trilogy import Environment
-from trilogy.authoring import ConceptRef
+from trilogy.authoring import (
+    ConceptRef,
+    DataType,
+    Ordering,
+    Purpose,
+)
+from trilogy.constants import MagicConstants
+from trilogy.core.enums import ComparisonOperator, FunctionType
 from trilogy.core.exceptions import ModelValidationError
 from trilogy.core.models.build import (
+    BuildCaseElse,
+    BuildCaseWhen,
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
+    BuildFunction,
+    BuildOrderBy,
+    BuildOrderItem,
 )
 from trilogy.core.models.environment import EnvironmentConceptDict
 from trilogy.core.models.execute import (
@@ -39,6 +51,32 @@ class ValidationType(Enum):
     CONCEPTS = "concepts"
 
 
+def build_order_args(concepts: list[BuildConcept]) -> list[BuildFunction]:
+    order_args = []
+    for concept in concepts:
+        order_args.append(
+            BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=concept,
+                            operator=ComparisonOperator.IS,
+                            right=MagicConstants.NULL,
+                        ),
+                        expr=1,
+                    ),
+                    BuildCaseElse(expr=0),
+                ],
+                output_data_type=DataType.INTEGER,
+                output_purpose=Purpose.PROPERTY,
+                arg_count=2,
+            )
+        )
+
+    return order_args
+
+
 def easy_query(
     concepts: list[BuildConcept],
     datasource: BuildDatasource,
@@ -81,7 +119,6 @@ def easy_query(
         group_to_grain=True,
         base_alias_override=datasource.safe_identifier,
     )
-
     filter_cte = CTE(
         name=f"datasource_{datasource.name}_filter",
         source=QueryDatasource(
@@ -100,6 +137,20 @@ def easy_query(
         grain=cte.grain,
         condition=condition,
         limit=limit,
+        order_by=BuildOrderBy(
+            items=[
+                BuildOrderItem(
+                    expr=BuildFunction(
+                        operator=FunctionType.SUM,
+                        arguments=build_order_args(concepts),
+                        output_data_type=DataType.INTEGER,
+                        output_purpose=Purpose.PROPERTY,
+                        arg_count=len(concepts),
+                    ),
+                    order=Ordering.DESCENDING,
+                )
+            ]
+        ),
     )
 
     return ProcessedQuery(

trilogy/core/validation/concept.py

@@ -1,8 +1,9 @@
 from trilogy import Environment, Executor
-from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.enums import Derivation, Modifier, Purpose
 from trilogy.core.exceptions import (
     ConceptModelValidationError,
-    DatasourceModelValidationError,
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
 )
 from trilogy.core.models.build import (
     BuildConcept,
@@ -25,6 +26,15 @@ def validate_key_concept(
 ):
     results: list[ValidationTest] = []
     seen: dict[str, int] = {}
+
+    count = 0
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            count += 1
+    # if it only has one source, it's a key
+    if count <= 1:
+        return results
+
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [
@@ -69,8 +79,19 @@ def validate_key_concept(
             err = None
             datasource_count: int = seen.get(datasource.name, 0)
             if datasource_count < max_seen and assignment.is_complete:
-                err = DatasourceModelValidationError(
-                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+                err = DatasourceColumnBindingError(
+                    address=datasource.identifier,
+                    errors=[
+                        DatasourceColumnBindingData(
+                            address=concept.address,
+                            value=None,
+                            value_type=concept.datatype,
+                            value_modifiers=[Modifier.PARTIAL],
+                            actual_type=concept.datatype,
+                            actual_modifiers=concept.modifiers,
+                        )
+                    ],
+                    message=f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial.",
                 )
             results.append(
                 ValidationTest(

trilogy/core/validation/datasource.py

@@ -10,9 +10,14 @@ from trilogy.authoring import (
     NumericType,
     StructType,
     TraitDataType,
+    arg_to_datatype,
+)
+from trilogy.core.enums import ComparisonOperator, Modifier
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+    DatasourceModelValidationError,
 )
-from trilogy.core.enums import ComparisonOperator
-from trilogy.core.exceptions import DatasourceModelValidationError
 from trilogy.core.models.build import (
     BuildComparison,
     BuildDatasource,
@@ -64,6 +69,7 @@ def validate_datasource(
     env: Environment,
     build_env: BuildEnvironment,
     exec: Executor | None = None,
+    fix: bool = False,
 ) -> list[ValidationTest]:
     results: list[ValidationTest] = []
     # we might have merged concepts, where both will map out to the same
@@ -109,14 +115,7 @@ def validate_datasource(
             )
         )
         return results
-    failures: list[
-        tuple[
-            str,
-            Any,
-            DataType | ArrayType | StructType | MapType | NumericType | TraitDataType,
-            bool,
-        ]
-    ] = []
+    failures: list[DatasourceColumnBindingData] = []
     cols_with_error = set()
     for row in rows:
         for col in datasource.columns:
@@ -127,26 +126,29 @@ def validate_datasource(
             passed = type_check(rval, col.concept.datatype, col.is_nullable)
             if not passed:
                 failures.append(
-                    (
-                        col.concept.address,
-                        rval,
-                        col.concept.datatype,
-                        col.is_nullable,
+                    DatasourceColumnBindingData(
+                        address=col.concept.address,
+                        value=rval,
+                        value_type=(
+                            arg_to_datatype(rval)
+                            if rval is not None
+                            else col.concept.datatype
+                        ),
+                        value_modifiers=[Modifier.NULLABLE] if rval is None else [],
+                        actual_type=col.concept.datatype,
+                        actual_modifiers=col.concept.modifiers,
                     )
                 )
                 cols_with_error.add(actual_address)
 
-    def format_failure(failure):
-        return f"Concept {failure[0]} value '{failure[1]}' does not conform to expected type {str(failure[2])} (nullable={failure[3]})"
-
     if failures:
         results.append(
             ValidationTest(
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 ran=True,
-                result=DatasourceModelValidationError(
-                    f"Datasource {datasource.name} failed validation. Found rows that do not conform to types: {[format_failure(failure) for failure in failures]}",
+                result=DatasourceColumnBindingError(
+                    address=datasource.identifier, errors=failures
                 ),
             )
         )

trilogy/core/validation/environment.py

@@ -15,9 +15,10 @@ def validate_environment(
     scope: ValidationScope = ValidationScope.ALL,
     targets: list[str] | None = None,
     exec: Executor | None = None,
+    generate_only: bool = False,
 ) -> list[ValidationTest]:
     # avoid mutating the environment for validation
-    generate_only = exec is None
+    generate_only = exec is None or generate_only
     env = env.duplicate()
     grain_check = function_to_concept(
         parent=Function(
@@ -68,4 +69,5 @@ def validate_environment(
                 f"Environment validation failed with the following errors:\n{messages}",
                 children=exceptions,
             )
+
     return results

trilogy/core/validation/fix.py

@@ -0,0 +1,106 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import ConceptDeclarationStatement, Datasource
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+)
+from trilogy.core.validation.environment import validate_environment
+from trilogy.parsing.render import Renderer
+
+
+def rewrite_file_with_errors(
+    statements: list[Any], errors: list[DatasourceColumnBindingError]
+):
+    renderer = Renderer()
+    output = []
+    ds_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    concept_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    for error in errors:
+        if isinstance(error, DatasourceColumnBindingError):
+            for x in error.errors:
+                if error.dataset_address not in ds_error_map:
+                    ds_error_map[error.dataset_address] = []
+                # this is by dataset address
+                if x.is_modifier_issue():
+                    ds_error_map[error.dataset_address].append(x)
+                # this is by column
+                if x.is_type_issue():
+                    concept_error_map[x.address].append(x)
+    for statement in statements:
+        if isinstance(statement, Datasource):
+            if statement.identifier in ds_error_map:
+                error_cols = ds_error_map[statement.identifier]
+                for col in statement.columns:
+                    if col.concept.address in [x.address for x in error_cols]:
+                        error_col = [
+                            x for x in error_cols if x.address == col.concept.address
+                        ][0]
+                        col.modifiers = list(
+                            set(col.modifiers + error_col.value_modifiers)
+                        )
+        elif isinstance(statement, ConceptDeclarationStatement):
+            if statement.concept.address in concept_error_map:
+                error_cols = concept_error_map[statement.concept.address]
+                statement.concept.datatype = error_cols[0].value_type
+        output.append(statement)
+
+    return renderer.render_statement_string(output)
+
+
+DEPTH_CUTOFF = 3
+
+
+def validate_and_rewrite(
+    input: Path | str, exec: Executor | None = None, depth: int = 0
+) -> str | None:
+    if depth > DEPTH_CUTOFF:
+        print(f"Reached depth cutoff of {DEPTH_CUTOFF}, stopping.")
+        return None
+    if isinstance(input, str):
+        raw = input
+        env = Environment()
+    else:
+        with open(input, "r") as f:
+            raw = f.read()
+        env = Environment(working_path=input.parent)
+    if exec:
+        env = exec.environment
+    env, statements = env.parse(raw)
+
+    validation_results = validate_environment(env, exec=exec, generate_only=True)
+
+    errors = [
+        x.result
+        for x in validation_results
+        if isinstance(x.result, DatasourceColumnBindingError)
+    ]
+
+    if not errors:
+        print("No validation errors found")
+        return None
+    print(
+        f"Found {len(errors)} validation errors, attempting to fix, current depth: {depth}..."
+    )
+    for error in errors:
+        for item in error.errors:
+            print(f"- {item.format_failure()}")
+
+    new_text = rewrite_file_with_errors(statements, errors)
+
+    while iteration := validate_and_rewrite(new_text, exec=exec, depth=depth + 1):
+        depth = depth + 1
+        if depth >= DEPTH_CUTOFF:
+            break
+        if iteration:
+            new_text = iteration
+        depth += 1
+    if isinstance(input, Path):
+        with open(input, "w") as f:
+            f.write(new_text)
+        return None
+    else:
+        return new_text

trilogy/dialect/base.py

@@ -761,6 +761,7 @@ class BaseDialect:
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"
+            return str(e.value)
         elif isinstance(e, date):
             return self.FUNCTION_MAP[FunctionType.DATE_LITERAL](e)
         elif isinstance(e, datetime):
@@ -1139,7 +1140,7 @@ class BaseDialect:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join(
                 [
-                    f'{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}'
+                    f"{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}"
                     for x in query.output_values
                     if isinstance(x, (ProcessedQuery, ProcessedCopyStatement))
                 ]

trilogy/dialect/common.py

@@ -10,7 +10,6 @@ from trilogy.core.models.build import (
     BuildParamaterizedConceptReference,
     BuildParenthetical,
 )
-from trilogy.core.models.datasource import RawColumnExpr
 from trilogy.core.models.execute import (
     CTE,
     InstantiatedUnnestJoin,
@@ -65,15 +64,8 @@ def render_join_concept(
     inlined_ctes: set[str],
 ):
     if cte.name in inlined_ctes:
-        ds = cte.source.datasources[0]
-        raw_content = ds.get_alias(concept)
-        if isinstance(raw_content, RawColumnExpr):
-            rval = raw_content.text
-            return rval
-        elif isinstance(raw_content, BuildFunction):
-            rval = render_expr(raw_content, cte=cte)
-            return rval
-        return f"{quote_character}{name}{quote_character}.{quote_character}{raw_content}{quote_character}"
+        base = render_expr(concept, cte)
+        return base
     return f"{quote_character}{name}{quote_character}.{quote_character}{concept.safe_address}{quote_character}"
 
 

trilogy/dialect/metadata.py

@@ -174,7 +174,7 @@ def raw_validation_to_result(
 ) -> Optional[MockResult]:
     """Convert raw validation tests to mock result."""
     if not raw:
-        return None
+        return MockResult([], ["check_type", "expected", "result", "ran", "query"])
     output = []
     for row in raw:
         if row.raw_query and generator and not row.generated_query:

trilogy/executor.py

@@ -410,6 +410,13 @@ class Executor(object):
                 )
                 output.extend(results)
                 continue
+            elif isinstance(statement, ProcessedValidateStatement):
+                validate_result = handle_processed_validate_statement(
+                    statement, self.generator, self.validate_environment
+                )
+                if validate_result:
+                    output.append(validate_result)
+                continue
             if non_interactive:
                 if not isinstance(
                     statement, (ProcessedCopyStatement, ProcessedQueryPersist)

trilogy/parsing/parse_engine.py

@@ -379,14 +379,16 @@ class ParseToObjects(Transformer):
     def start(self, args):
         return args
 
+    def LINE_SEPARATOR(self, args):
+        return MagicConstants.LINE_SEPARATOR
+
     def block(self, args):
         output = args[0]
         if isinstance(output, ConceptDeclarationStatement):
-            if len(args) > 1 and isinstance(args[1], Comment):
-                output.concept.metadata.description = (
-                    output.concept.metadata.description
-                    or args[1].text.split("#")[1].strip()
-                )
+            if len(args) > 1 and args[1] != MagicConstants.LINE_SEPARATOR:
+                comments = [x for x in args[1:] if isinstance(x, Comment)]
+                merged = "\n".join([x.text.split("#")[1].rstrip() for x in comments])
+                output.concept.metadata.description = merged
         # this is a bad plan for now;
         # because a comment after an import statement is very common
         # and it's not intuitive that it modifies the import description
@@ -913,7 +915,7 @@ class ParseToObjects(Transformer):
         return Comment(text=args[0].value)
 
     def PARSE_COMMENT(self, args):
-        return Comment(text=args.value)
+        return Comment(text=args.value.rstrip())
 
     @v_args(meta=True)
     def select_transform(self, meta: Meta, args) -> ConceptTransform:
@@ -1000,7 +1002,7 @@ class ParseToObjects(Transformer):
     def validate_statement(self, meta: Meta, args) -> ValidateStatement:
         if len(args) == 2:
             scope = args[0]
-            targets = args[1]
+            targets = args[1].split(",")
         elif len(args) == 0:
             scope = ValidationScope.ALL
             targets = None
@@ -2293,7 +2295,7 @@ def parse_text(
             raise _create_syntax_error(210, pos, text)
 
         # Handle FROM token error
-        parsed_tokens = [x.value for x in e.token_history] if e.token_history else []
+        parsed_tokens = [x.value for x in e.token_history if x] if e.token_history else []
         if parsed_tokens == ["FROM"]:
             raise _create_syntax_error(101, pos, text)
 

trilogy/parsing/render.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from datetime import date, datetime
 from functools import singledispatchmethod
+from typing import Any
 
 from jinja2 import Template
 
@@ -12,6 +13,7 @@ from trilogy.core.models.author import (
     AlignItem,
     CaseElse,
     CaseWhen,
+    Comment,
     Comparison,
     Concept,
     ConceptRef,
@@ -83,6 +85,23 @@ class Renderer:
     def __init__(self, environment: Environment | None = None):
         self.environment = environment
 
+    def render_statement_string(self, list_of_statements: list[Any]) -> str:
+        new = []
+        last_statement_type = None
+        for stmt in list_of_statements:
+            stmt_type = type(stmt)
+            if last_statement_type is None:
+                pass
+            elif last_statement_type == Comment:
+                new.append("\n")
+            elif stmt_type != last_statement_type:
+                new.append("\n\n")
+            else:
+                new.append("\n")
+            new.append(Renderer().to_string(stmt))
+            last_statement_type = stmt_type
+        return "".join(new)
+
     @singledispatchmethod
     def to_string(self, arg):
         raise NotImplementedError("Cannot render type {}".format(type(arg)))
@@ -269,6 +288,8 @@ class Renderer:
     @to_string.register
     def _(self, arg: "Address"):
         if arg.is_query:
+            if arg.location.startswith("("):
+                return f"query '''{arg.location[1:-1]}'''"
             return f"query '''{arg.location}'''"
         return f"address {arg.location}"
 
@@ -286,7 +307,7 @@ class Renderer:
     def _(self, arg: "ColumnAssignment"):
         if arg.modifiers:
             modifiers = "".join(
-                [self.to_string(modifier) for modifier in arg.modifiers]
+                [self.to_string(modifier) for modifier in sorted(arg.modifiers)]
             )
         else:
             modifiers = ""
@@ -328,7 +349,7 @@ class Renderer:
         else:
             output = f"{concept.purpose.value} {namespace}{concept.name} <- {self.to_string(concept.lineage)};"
         if base_description:
-            output += f" # {base_description}"
+            output += f" #{base_description}"
         return output
 
     @to_string.register
@@ -428,6 +449,10 @@ class Renderer:
     def _(self, arg: "Comparison"):
         return f"{self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)}"
 
+    @to_string.register
+    def _(self, arg: "Comment"):
+        return f"{arg.text}"
+
     @to_string.register
     def _(self, arg: "WindowItem"):
         over = ",".join(self.to_string(c) for c in arg.over)
@@ -521,8 +546,8 @@ class Renderer:
             return f"{self.to_string(arg.arguments[0])}[{self.to_string(arg.arguments[1])}]"
 
         if arg.operator == FunctionType.CASE:
-            inputs = "\n".join(args)
-            return f"CASE {inputs}\nEND"
+            inputs = "\n\t".join(args)
+            return f"CASE\n\t{inputs}\nEND"
         return f"{arg.operator.value}({inputs})"
 
     @to_string.register
@@ -551,8 +576,10 @@ class Renderer:
     def _(self, arg: Modifier):
         if arg == Modifier.PARTIAL:
             return "~"
-        if arg == Modifier.HIDDEN:
+        elif arg == Modifier.HIDDEN:
             return "--"
+        elif arg == Modifier.NULLABLE:
+            return "?"
         return arg.value
 
     @to_string.register

trilogy/parsing/trilogy.lark

@@ -1,5 +1,5 @@
- !start: ( block | show_statement )*
-    block: statement _TERMINATOR PARSE_COMMENT?
+ !start: ( block | show_statement | PARSE_COMMENT )*
+    block: statement _TERMINATOR LINE_SEPARATOR? PARSE_COMMENT*
     ?statement: concept
     | datasource
     | function
@@ -14,9 +14,12 @@
     | rawsql_statement
     | validate_statement
     
-    _TERMINATOR:  ";"i /\s*/
+    _TERMINATOR:  ";"i
     
-    PARSE_COMMENT.1:   /#.*(\n|$)/ |  /\/\/.*\n/  
+    PARSE_COMMENT.1:   /#.*(\n|$)/ |  /\/\/.*(\n|$)/  
+
+    // when whitespace matters - comment placement
+    LINE_SEPARATOR.1: /[ \t\r\f\v]*\n+/
     
     // property display_name string
     concept_declaration: PURPOSE IDENTIFIER data_type concept_nullable_modifier? metadata?

trilogy/std/metric.preql

@@ -0,0 +1,15 @@
+# Length and distance units
+type m numeric;      # meters
+type km numeric;     # kilometers
+type cm numeric;     # centimeters
+type mm numeric;     # millimeters
+
+# Mass units
+type kg numeric;     # kilograms
+type g numeric;      # grams
+type tonne numeric;  # metric tons (1000 kg)
+
+# Force units
+type n numeric;      # newtons
+type kn numeric;     # kilonewtons (1000 N)
+type mn numeric;     # meganewtons (1,000,000 N)