pytrilogy 0.0.3.96__tar.gz → 0.0.3.98__tar.gz

{pytrilogy-0.0.3.96/pytrilogy.egg-info → pytrilogy-0.0.3.98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.96
+Version: 0.0.3.98
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -334,7 +334,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/README.md

@@ -296,7 +296,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.96
+Version: 0.0.3.98
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -334,7 +334,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/pytrilogy.egg-info/SOURCES.txt

@@ -116,6 +116,7 @@ trilogy/core/validation/common.py
 trilogy/core/validation/concept.py
 trilogy/core/validation/datasource.py
 trilogy/core/validation/environment.py
+trilogy/core/validation/fix.py
 trilogy/dialect/__init__.py
 trilogy/dialect/base.py
 trilogy/dialect/bigquery.py
@@ -148,6 +149,7 @@ trilogy/std/__init__.py
 trilogy/std/date.preql
 trilogy/std/display.preql
 trilogy/std/geography.preql
+trilogy/std/metric.preql
 trilogy/std/money.preql
 trilogy/std/net.preql
 trilogy/std/ranking.preql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/tests/test_parsing.py

@@ -321,6 +321,55 @@ def test_the_comments():
     assert rendered == "null"
 
 
+def test_the_comment_multiline():
+    env, parsed = parse_text(
+        """const
+         # comment here?
+           order_id <- 4;  # this is the order id
+        # order ids are important
+           
+           SELECT 
+        # TOOD - add in more columns?
+        order_id  
+        WHERE 
+        # order_id should not be null
+        order_id
+        # in this comp
+          is not 
+        null; # nulls are the worst
+        
+        """
+    )
+    parsed[-1]
+    assert env.concepts["order_id"].metadata.description is not None
+    assert " this is the order id" in env.concepts["order_id"].metadata.description
+    assert " order ids are important" in env.concepts["order_id"].metadata.description
+
+
+def test_the_comment_multiline_enter():
+    # we should not associate it as a description if there is a newline
+    env, parsed = parse_text(
+        """const
+         # comment here?
+           order_id <- 4;  
+        # this is the order id
+        # order ids are important
+           
+           SELECT 
+        # TOOD - add in more columns?
+        order_id  
+        WHERE 
+        # order_id should not be null
+        order_id
+        # in this comp
+          is not 
+        null; # nulls are the worst
+        
+        """
+    )
+    assert env.concepts["order_id"].metadata.description is None
+
+
 def test_purpose_nesting():
     env, parsed = parse_text(
         """key year int;

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.96"
+__version__ = "0.0.3.98"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/constants.py

@@ -16,6 +16,7 @@ ENV_CACHE_NAME = ".preql_cache.json"
 
 class MagicConstants(Enum):
     NULL = "null"
+    LINE_SEPARATOR = "\n"
 
 
 NULL_VALUE = MagicConstants.NULL

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/core/enums.py

@@ -82,6 +82,15 @@ class Modifier(Enum):
             return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
+    def __lt__(self, other):
+        order = [
+            Modifier.HIDDEN,
+            Modifier.PARTIAL,
+            Modifier.NULLABLE,
+            Modifier.OPTIONAL,
+        ]
+        return order.index(self) < order.index(other)
+
 
 class JoinType(Enum):
     INNER = "inner"

pytrilogy-0.0.3.98/trilogy/core/exceptions.py

@@ -0,0 +1,105 @@
+from dataclasses import dataclass
+from typing import Any, List, Sequence
+
+from trilogy.core.enums import Modifier
+from trilogy.core.models.core import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+)
+
+
+class UndefinedConceptException(Exception):
+    def __init__(self, message, suggestions: List[str]):
+        super().__init__(self, message)
+        self.message = message
+        self.suggestions = suggestions
+
+
+class FrozenEnvironmentException(Exception):
+    pass
+
+
+class InvalidSyntaxException(Exception):
+    pass
+
+
+class UnresolvableQueryException(Exception):
+    pass
+
+
+class NoDatasourceException(UnresolvableQueryException):
+    pass
+
+
+class ModelValidationError(Exception):
+    def __init__(
+        self,
+        message,
+        children: Sequence["ModelValidationError"] | None = None,
+        **kwargs,
+    ):
+        super().__init__(self, message, **kwargs)
+        self.message = message
+        self.children = children
+
+
+class DatasourceModelValidationError(ModelValidationError):
+    pass
+
+
+class DatasourceGrainValidationError(DatasourceModelValidationError):
+    pass
+
+
+@dataclass
+class DatasourceColumnBindingData:
+    address: str
+    value: Any
+    value_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    value_modifiers: List[Modifier]
+    actual_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    actual_modifiers: List[Modifier]
+
+    def format_failure(self):
+        return f"Concept {self.address} value '{self.value}' with type {self.value_modifiers} does not conform to expected type {str(self.actual_type)} with modifiers {self.actual_modifiers}"
+
+    def is_modifier_issue(self) -> bool:
+        return len(self.value_modifiers) > 0 and any(
+            [x not in self.actual_modifiers for x in self.value_modifiers]
+        )
+
+    def is_type_issue(self) -> bool:
+        return self.value_type != self.actual_type
+
+
+class DatasourceColumnBindingError(DatasourceModelValidationError):
+    def __init__(
+        self,
+        address: str,
+        errors: list[DatasourceColumnBindingData],
+        message: str | None = None,
+    ):
+        if not message:
+            message = f"Datasource {address} failed validation. Found rows that do not conform to types: {[failure.format_failure() for failure in errors]}"
+        super().__init__(message)
+        self.errors = errors
+        self.dataset_address = address
+
+
+class ConceptModelValidationError(ModelValidationError):
+    pass
+
+
+class AmbiguousRelationshipResolutionException(UnresolvableQueryException):
+    def __init__(self, message, parents: List[set[str]]):
+        super().__init__(self, message)
+        self.message = message
+        self.parents = parents

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/core/functions.py

@@ -427,7 +427,7 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
             {DataType.STRING},
         ],
         output_purpose=Purpose.PROPERTY,
-        output_type=DataType.BOOL,
+        output_type=DataType.STRING,
         arg_count=2,
     ),
     FunctionType.SUBSTRING: FunctionConfig(

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/core/models/execute.py

@@ -151,7 +151,6 @@ class CTE(BaseModel):
             ]
         ):
             return False
-
         self.source.datasources = [
             ds_being_inlined,
             *[

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,4 +1,5 @@
-from typing import List, Optional
+from itertools import combinations
+from typing import Callable, List, Optional
 
 import networkx as nx
 from networkx.algorithms import approximation as ax
@@ -11,7 +12,12 @@ from trilogy.core.graph_models import (
     concept_to_node,
     prune_sources_for_conditions,
 )
-from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildConditional,
+    BuildGrain,
+    BuildWhereClause,
+)
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
@@ -29,7 +35,10 @@ def filter_pseudonyms_for_source(
         if edge in pseudonyms:
             lengths = {}
             for n in edge:
-                lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+                try:
+                    lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+                except nx.NetworkXNoPath:
+                    lengths[n] = 999
             to_remove.add(max(lengths, key=lambda x: lengths.get(x, 0)))
     for node in to_remove:
         ds_graph.remove_node(node)
@@ -84,12 +93,104 @@ def extract_ds_components(
     return graphs
 
 
+def prune_and_merge(
+    G: nx.DiGraph,
+    keep_node_lambda: Callable[[str], bool],
+) -> nx.DiGraph:
+    """
+    Prune nodes of one type and create direct connections between remaining nodes.
+
+    Args:
+        G: NetworkX graph
+        keep_node_type: The node type to keep
+        node_type_attr: Attribute name that stores node type
+
+    Returns:
+        New graph with only keep_node_type nodes and merged connections
+    """
+    # Get nodes to keep
+    nodes_to_keep = [n for n in G.nodes if keep_node_lambda(n)]
+    # Create new graph
+    new_graph = G.subgraph(nodes_to_keep).copy()
+
+    # Find paths between nodes to keep through removed nodes
+    nodes_to_remove = [n for n in G.nodes() if n not in nodes_to_keep]
+
+    for node_pair in combinations(nodes_to_keep, 2):
+        n1, n2 = node_pair
+
+        # Check if there's a path through removed nodes
+        try:
+            path = nx.shortest_path(G, n1, n2)
+            # If path exists and goes through nodes we're removing
+            if len(path) > 2 or any(node in nodes_to_remove for node in path[1:-1]):
+                new_graph.add_edge(n1, n2)
+        except nx.NetworkXNoPath:
+            continue
+
+    return new_graph
+
+
+def reinject_common_join_keys_v2(
+    G: ReferenceGraph,
+    final: nx.DiGraph,
+    nodelist: list[str],
+    synonyms: set[str] = set(),
+) -> bool:
+    # when we've discovered a concept join, for each pair of ds nodes
+    # check if they have more keys in common
+    # and inject those to discovery as join conditions
+    def is_ds_node(n: str) -> bool:
+        return n.startswith("ds~")
+
+    ds_graph = prune_and_merge(final, is_ds_node)
+    injected = False
+    for datasource in ds_graph.nodes:
+        node1 = G.datasources[datasource]
+        neighbors = nx.all_neighbors(ds_graph, datasource)
+        for neighbor in neighbors:
+            node2 = G.datasources[neighbor]
+            common_concepts = set(
+                x.concept.address for x in node1.columns
+            ).intersection(set(x.concept.address for x in node2.columns))
+            concrete_concepts = [
+                x.concept for x in node1.columns if x.concept.address in common_concepts
+            ]
+            reduced = BuildGrain.from_concepts(concrete_concepts).components
+            existing_addresses = set()
+            for concrete in concrete_concepts:
+                logger.info(
+                    f"looking at column {concrete.address} with pseudonyms {concrete.pseudonyms}"
+                )
+                cnode = concept_to_node(concrete.with_default_grain())
+                if cnode in final.nodes:
+                    existing_addresses.add(concrete.address)
+                    continue
+            for concrete in concrete_concepts:
+                if concrete.address in synonyms:
+                    continue
+                if concrete.address not in reduced:
+                    continue
+                # skip anything that is already in the graph pseudonyms
+                if any(x in concrete.pseudonyms for x in existing_addresses):
+                    continue
+                cnode = concept_to_node(concrete.with_default_grain())
+                final.add_edge(datasource, cnode)
+                final.add_edge(neighbor, cnode)
+                logger.info(
+                    f"{LOGGER_PREFIX} reinjecting common join key {cnode} between {datasource} and {neighbor}"
+                )
+                injected = True
+    return injected
+
+
 def determine_induced_minimal_nodes(
     G: ReferenceGraph,
     nodelist: list[str],
     environment: BuildEnvironment,
     filter_downstream: bool,
     accept_partial: bool = False,
+    synonyms: set[str] = set(),
 ) -> nx.DiGraph | None:
     H: nx.Graph = nx.to_undirected(G).copy()
     nodes_to_remove = []
@@ -129,7 +230,7 @@ def determine_induced_minimal_nodes(
         return None
     path_removals = list(x for x in H.nodes if x not in paths)
     if path_removals:
-        logger.debug(f"Removing paths {path_removals} from graph")
+        # logger.debug(f"Removing paths {path_removals} from graph")
         H.remove_nodes_from(path_removals)
     # logger.debug(f"Graph after path removal {H.nodes}")
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
@@ -148,8 +249,10 @@ def determine_induced_minimal_nodes(
                 if not accept_partial:
                     continue
             final.add_edge(*edge)
-    # all concept nodes must have a parent
 
+    reinject_common_join_keys_v2(G, final, nodelist, synonyms)
+
+    # all concept nodes must have a parent
     if not all(
         [
             final.in_degree(node) > 0
@@ -302,6 +405,7 @@ def resolve_weak_components(
                 filter_downstream=filter_downstream,
                 accept_partial=accept_partial,
                 environment=environment,
+                synonyms=synonyms,
             )
 
             if not g or not g.nodes:

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/core/validation/common.py

@@ -2,13 +2,25 @@ from dataclasses import dataclass
 from enum import Enum
 
 from trilogy import Environment
-from trilogy.authoring import ConceptRef
+from trilogy.authoring import (
+    ConceptRef,
+    DataType,
+    Ordering,
+    Purpose,
+)
+from trilogy.constants import MagicConstants
+from trilogy.core.enums import ComparisonOperator, FunctionType
 from trilogy.core.exceptions import ModelValidationError
 from trilogy.core.models.build import (
+    BuildCaseElse,
+    BuildCaseWhen,
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
+    BuildFunction,
+    BuildOrderBy,
+    BuildOrderItem,
 )
 from trilogy.core.models.environment import EnvironmentConceptDict
 from trilogy.core.models.execute import (
@@ -39,6 +51,32 @@ class ValidationType(Enum):
     CONCEPTS = "concepts"
 
 
+def build_order_args(concepts: list[BuildConcept]) -> list[BuildFunction]:
+    order_args = []
+    for concept in concepts:
+        order_args.append(
+            BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=concept,
+                            operator=ComparisonOperator.IS,
+                            right=MagicConstants.NULL,
+                        ),
+                        expr=1,
+                    ),
+                    BuildCaseElse(expr=0),
+                ],
+                output_data_type=DataType.INTEGER,
+                output_purpose=Purpose.PROPERTY,
+                arg_count=2,
+            )
+        )
+
+    return order_args
+
+
 def easy_query(
     concepts: list[BuildConcept],
     datasource: BuildDatasource,
@@ -81,7 +119,6 @@ def easy_query(
         group_to_grain=True,
         base_alias_override=datasource.safe_identifier,
     )
-
     filter_cte = CTE(
         name=f"datasource_{datasource.name}_filter",
         source=QueryDatasource(
@@ -100,6 +137,20 @@ def easy_query(
         grain=cte.grain,
         condition=condition,
         limit=limit,
+        order_by=BuildOrderBy(
+            items=[
+                BuildOrderItem(
+                    expr=BuildFunction(
+                        operator=FunctionType.SUM,
+                        arguments=build_order_args(concepts),
+                        output_data_type=DataType.INTEGER,
+                        output_purpose=Purpose.PROPERTY,
+                        arg_count=len(concepts),
+                    ),
+                    order=Ordering.DESCENDING,
+                )
+            ]
+        ),
     )
 
     return ProcessedQuery(

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.98}/trilogy/core/validation/concept.py

@@ -1,8 +1,9 @@
 from trilogy import Environment, Executor
-from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.enums import Derivation, Modifier, Purpose
 from trilogy.core.exceptions import (
     ConceptModelValidationError,
-    DatasourceModelValidationError,
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
 )
 from trilogy.core.models.build import (
     BuildConcept,
@@ -25,6 +26,15 @@ def validate_key_concept(
 ):
     results: list[ValidationTest] = []
     seen: dict[str, int] = {}
+
+    count = 0
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            count += 1
+    # if it only has one source, it's a key
+    if count <= 1:
+        return results
+
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [
@@ -69,8 +79,19 @@ def validate_key_concept(
             err = None
             datasource_count: int = seen.get(datasource.name, 0)
             if datasource_count < max_seen and assignment.is_complete:
-                err = DatasourceModelValidationError(
-                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+                err = DatasourceColumnBindingError(
+                    address=datasource.identifier,
+                    errors=[
+                        DatasourceColumnBindingData(
+                            address=concept.address,
+                            value=None,
+                            value_type=concept.datatype,
+                            value_modifiers=[Modifier.PARTIAL],
+                            actual_type=concept.datatype,
+                            actual_modifiers=concept.modifiers,
+                        )
+                    ],
+                    message=f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial.",
                 )
             results.append(
                 ValidationTest(