pytrilogy 0.0.2.54__tar.gz → 0.0.2.56__tar.gz

{pytrilogy-0.0.2.54/pytrilogy.egg-info → pytrilogy-0.0.2.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.54
+Version: 0.0.2.56
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.54
+Version: 0.0.2.56
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/tests/test_select.py

@@ -1,5 +1,5 @@
 # from trilogy.compiler import compile
-from trilogy import Dialects
+from trilogy import Dialects, Environment
 from trilogy.core.models import Grain, SelectStatement
 from trilogy.core.query_processor import process_query
 from trilogy.dialect.bigquery import BigqueryDialect
@@ -176,3 +176,35 @@ select id + 2 as three;
 
     result = Dialects.DUCK_DB.default_executor(environment=env).execute_text(q1)[-1]
     assert result.fetchone().three == 3
+
+
+def test_select_from_components():
+    env = Environment()
+    q1 = """
+
+key id int;
+property id.class int;
+property id.name string;
+
+select
+    class,
+    upper(id.name)-> upper_name,
+    count(id) ->class_id_count,
+;
+"""
+    env, statements = env.parse(q1)
+
+    select: SelectStatement = statements[-1]
+
+    assert select.grain.components == {"local.class", "local.upper_name"}
+    assert select.local_concepts["local.class_id_count"].grain.components == {
+        "local.class",
+        "local.upper_name",
+    }
+
+    # SelectStatement.from_inputs(
+    #     environment=env,
+    #     selection=[SelectItem(concept=env.concepts["id"]),
+    #                SelectItem(concept=env.concepts["id.class"])],
+    #     input_components=[],
+    # )

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.2.54"
+__version__ = "0.0.2.56"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/environment_helpers.py

@@ -11,12 +11,23 @@ from trilogy.core.models import (
 )
 from trilogy.parsing.common import Meta, arg_to_datatype, process_function_args
 
+FUNCTION_DESCRIPTION_MAPS = {
+    FunctionType.DATE: "The date part of a timestamp/date. Integer, 0-31 depending on month.",
+    FunctionType.MONTH: "The month part of a timestamp/date. Integer, 1-12.",
+    FunctionType.YEAR: "The year part of a timestamp/date. Integer.",
+    FunctionType.QUARTER: "The quarter part of a timestamp/date. Integer, 1-4.",
+    FunctionType.DAY_OF_WEEK: "The day of the week part of a timestamp/date. Integer, 0-6.",
+    FunctionType.HOUR: "The hour part of a timestamp. Integer, 0-23.",
+    FunctionType.MINUTE: "The minute part of a timestamp. Integer, 0-59.",
+    FunctionType.SECOND: "The second part of a timestamp. Integer, 0-59.",
+}
+
 
 def generate_date_concepts(concept: Concept, environment: Environment):
     if concept.metadata and concept.metadata.description:
         base_description = concept.metadata.description
     else:
-        base_description = f"a {concept.datatype.value}"
+        base_description = f"a {concept.address}"
     if concept.metadata and concept.metadata.line_number:
         base_line_number = concept.metadata.line_number
     else:
@@ -54,7 +65,7 @@ def generate_date_concepts(concept: Concept, environment: Environment):
                 concept.address,
             ),
             metadata=Metadata(
-                description=f"Auto-derived. Integer format. The {ftype.value} derived from {concept.name}, {base_description}",
+                description=f"Auto-derived from {base_description}. {FUNCTION_DESCRIPTION_MAPS.get(ftype, ftype.value)}. ",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
@@ -68,7 +79,7 @@ def generate_datetime_concepts(concept: Concept, environment: Environment):
     if concept.metadata and concept.metadata.description:
         base_description = concept.metadata.description
     else:
-        base_description = f"a {concept.datatype.value}"
+        base_description = concept.address
     if concept.metadata and concept.metadata.line_number:
         base_line_number = concept.metadata.line_number
     else:
@@ -105,7 +116,7 @@ def generate_datetime_concepts(concept: Concept, environment: Environment):
                 concept.address,
             ),
             metadata=Metadata(
-                description=f"Auto-derived. Integer format. The {ftype.value} derived from {concept.name}, {base_description}",
+                description=f"Auto-derived from {base_description}. {FUNCTION_DESCRIPTION_MAPS.get(ftype, ftype.value)}.",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
@@ -147,7 +158,7 @@ def generate_key_concepts(concept: Concept, environment: Environment):
                 concept.address,
             },
             metadata=Metadata(
-                description=f"Auto-derived. Integer format. The {ftype.value} derived from {concept.name}, {base_description}",
+                description=f"Auto-derived integer. The {ftype.value} of {concept.address}, {base_description}",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/models.py

@@ -620,7 +620,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
             )
         final_grain = self.grain or grain
         keys = self.keys if self.keys else None
-        if self.is_aggregate and isinstance(new_lineage, Function):
+        if self.is_aggregate and isinstance(new_lineage, Function) and grain.components:
             grain_components = [environment.concepts[c] for c in grain.components]
             new_lineage = AggregateWrapper(function=new_lineage, by=grain_components)
             final_grain = grain
@@ -852,7 +852,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
 
 class ConceptRef(BaseModel):
     address: str
-    line_no: int
+    line_no: int | None = None
 
     def hydrate(self, environment: Environment) -> Concept:
         return environment.concepts.__getitem__(self.address, self.line_no)
@@ -1015,6 +1015,7 @@ class EnvironmentConceptDict(dict):
     def raise_undefined(
         self, key: str, line_no: int | None = None, file: Path | str | None = None
     ) -> Never:
+
         matches = self._find_similar_concepts(key)
         message = f"Undefined concept: {key}."
         if matches:
@@ -1660,6 +1661,96 @@ class SelectStatement(HasUUID, Mergeable, Namespaced, SelectTypeMixin, BaseModel
     ] = Field(default_factory=EnvironmentConceptDict)
     grain: Grain = Field(default_factory=Grain)
 
+    @classmethod
+    def from_inputs(
+        cls,
+        environment: Environment,
+        selection: List[SelectItem],
+        order_by: OrderBy | None = None,
+        limit: int | None = None,
+        meta: Metadata | None = None,
+        where_clause: WhereClause | None = None,
+        having_clause: HavingClause | None = None,
+    ) -> "SelectStatement":
+
+        output = SelectStatement(
+            selection=selection,
+            where_clause=where_clause,
+            having_clause=having_clause,
+            limit=limit,
+            order_by=order_by,
+            meta=meta or Metadata(),
+        )
+        for parse_pass in [
+            1,
+            2,
+        ]:
+            # the first pass will result in all concepts being defined
+            # the second will get grains appropriately
+            # eg if someone does sum(x)->a, b+c -> z - we don't know if Z is a key to group by or an aggregate
+            # until after the first pass, and so don't know the grain of a
+
+            if parse_pass == 1:
+                grain = Grain.from_concepts(
+                    [
+                        x.content
+                        for x in output.selection
+                        if isinstance(x.content, Concept)
+                    ],
+                    where_clause=output.where_clause,
+                )
+            if parse_pass == 2:
+                grain = Grain.from_concepts(
+                    output.output_components, where_clause=output.where_clause
+                )
+            output.grain = grain
+            pass_grain = Grain() if parse_pass == 1 else grain
+            for item in selection:
+                # we don't know the grain of an aggregate at assignment time
+                # so rebuild at this point in the tree
+                # TODO: simplify
+                if isinstance(item.content, ConceptTransform):
+                    new_concept = item.content.output.with_select_context(
+                        output.local_concepts,
+                        # the first pass grain will be incorrect
+                        pass_grain,
+                        environment=environment,
+                    )
+                    output.local_concepts[new_concept.address] = new_concept
+                    item.content.output = new_concept
+                    if parse_pass == 2 and CONFIG.select_as_definition:
+                        environment.add_concept(new_concept)
+                elif isinstance(item.content, UndefinedConcept):
+                    environment.concepts.raise_undefined(
+                        item.content.address,
+                        line_no=item.content.metadata.line_number,
+                        file=environment.env_file_path,
+                    )
+                elif isinstance(item.content, Concept):
+                    # Sometimes cached values here don't have the latest info
+                    # but we can't just use environment, as it might not have the right grain.
+                    item.content = item.content.with_select_context(
+                        output.local_concepts,
+                        pass_grain,
+                        environment=environment,
+                    )
+                    output.local_concepts[item.content.address] = item.content
+
+        if order_by:
+            output.order_by = order_by.with_select_context(
+                local_concepts=output.local_concepts,
+                grain=output.grain,
+                environment=environment,
+            )
+        if output.having_clause:
+            output.having_clause = output.having_clause.with_select_context(
+                local_concepts=output.local_concepts,
+                grain=output.grain,
+                environment=environment,
+            )
+        output.validate_syntax(environment)
+        return output
+
     def validate_syntax(self, environment: Environment):
         if self.where_clause:
             for x in self.where_clause.concept_arguments:
@@ -3264,6 +3355,7 @@ class Environment(BaseModel):
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
     # TODO: support freezing environments to avoid mutation
     frozen: bool = False
+    env_file_path: Path | None = None
 
     def freeze(self):
         self.frozen = True
@@ -3317,7 +3409,11 @@ class Environment(BaseModel):
             path = Path(path)
         with open(path, "r") as f:
             read = f.read()
-        return Environment(working_path=Path(path).parent).parse(read)[0]
+        return Environment(working_path=path.parent, env_file_path=path).parse(read)[0]
+
+    @classmethod
+    def from_string(cls, input: str) -> "Environment":
+        return Environment().parse(input)[0]
 
     @classmethod
     def from_cache(cls, path) -> Optional["Environment"]:

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/processing/concept_strategies_v3.py

@@ -866,6 +866,7 @@ def _search_concepts(
         )
         if complete == ValidationResult.INCOMPLETE_CONDITION:
             cond_dict = {str(node): node.preexisting_conditions for node in stack}
+            logger.error(f"Have {cond_dict} and need {str(conditions)}")
             raise SyntaxError(f"Have {cond_dict} and need {str(conditions)}")
         # early exit if we have a complete stack with one node
         # we can only early exit if we have a complete stack

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/processing/node_generators/filter_node.py

@@ -60,7 +60,7 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
-        # conditions=conditions,
+        conditions=conditions,
     )
 
     flattened_existence = [x for y in parent_existence_concepts for x in y]
@@ -194,6 +194,9 @@ def gen_filter_node(
         history=history,
         conditions=conditions,
     )
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} returning filter node and enrich node with {enrich_node.output_concepts} and {enrich_node.input_concepts}"
+    )
     return MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[
@@ -206,4 +209,5 @@ def gen_filter_node(
             filter_node,
             enrich_node,
         ],
+        preexisting_conditions=conditions.conditional if conditions else None,
     )

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/processing/nodes/base_node.py

@@ -210,6 +210,8 @@ class StrategyNode:
         return self
 
     def add_condition(self, condition: Conditional | Comparison | Parenthetical):
+        if self.conditions and condition == self.conditions:
+            return self
         if self.conditions:
             self.conditions = Conditional(
                 left=self.conditions, right=condition, operator=BooleanOperator.AND

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/processing/nodes/merge_node.py

@@ -224,9 +224,6 @@ class MergeNode(StrategyNode):
                 f"{self.logging_prefix}{LOGGER_PREFIX} Final joins is not null {final_joins} but is empty, skipping join generation"
             )
             return []
-
-        for join in joins:
-            logger.info(f"{self.logging_prefix}{LOGGER_PREFIX} final join {str(join)}")
         return joins
 
     def _resolve(self) -> QueryDatasource:

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/processing/utility.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Set, Tuple
 
 import networkx as nx
 
+from trilogy.constants import logger
 from trilogy.core.enums import BooleanOperator, FunctionClass, Granularity, Purpose
 from trilogy.core.models import (
     CTE,
@@ -160,6 +161,8 @@ def resolve_join_order_v2(
                 final_join_type = JoinType.LEFT_OUTER
             elif any([x == JoinType.FULL for x in join_types]):
                 final_join_type = JoinType.FULL
+            logger.info("JOIN DEBUG")
+            logger.info(joinkeys)
             output.append(
                 JoinOrderOutput(
                     # left=left_candidate,
@@ -306,11 +309,37 @@ def resolve_instantiated_concept(
     )
 
 
+def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
+    left_keys = set()
+    right_keys = set()
+    for pair in input:
+        if pair.left.purpose == Purpose.KEY:
+            left_keys.add(pair.left.address)
+        if pair.right.purpose == Purpose.KEY:
+            right_keys.add(pair.right.address)
+    final: list[ConceptPair] = []
+    for pair in input:
+        if (
+            pair.left.purpose == Purpose.PROPERTY
+            and pair.left.keys
+            and pair.left.keys.issubset(left_keys)
+        ):
+            continue
+        if (
+            pair.right.purpose == Purpose.PROPERTY
+            and pair.right.keys
+            and pair.right.keys.issubset(right_keys)
+        ):
+            continue
+        final.append(pair)
+    return final
+
+
 def get_node_joins(
     datasources: List[QueryDatasource | Datasource],
     environment: Environment,
     # concepts:List[Concept],
-):
+) -> List[BaseJoin]:
     graph = nx.Graph()
     partials: dict[str, list[str]] = {}
     ds_node_map: dict[str, QueryDatasource | Datasource] = {}
@@ -337,19 +366,21 @@ def get_node_joins(
             join_type=j.type,
             # preserve empty field for maps
             concepts=[] if not j.keys else None,
-            concept_pairs=[
-                ConceptPair(
-                    left=resolve_instantiated_concept(
-                        concept_map[concept], ds_node_map[k]
-                    ),
-                    right=resolve_instantiated_concept(
-                        concept_map[concept], ds_node_map[j.right]
-                    ),
-                    existing_datasource=ds_node_map[k],
-                )
-                for k, v in j.keys.items()
-                for concept in v
-            ],
+            concept_pairs=reduce_concept_pairs(
+                [
+                    ConceptPair(
+                        left=resolve_instantiated_concept(
+                            concept_map[concept], ds_node_map[k]
+                        ),
+                        right=resolve_instantiated_concept(
+                            concept_map[concept], ds_node_map[j.right]
+                        ),
+                        existing_datasource=ds_node_map[k],
+                    )
+                    for k, v in j.keys.items()
+                    for concept in v
+                ]
+            ),
         )
         for j in joins
     ]

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/core/query_processor.py

@@ -359,7 +359,7 @@ def get_query_node(
         environment.concepts[k] = v
     graph = generate_graph(environment)
     logger.info(
-        f"{LOGGER_PREFIX} getting source datasource for query with filtering {statement.where_clause_category} and output {[str(c) for c in statement.output_components]}"
+        f"{LOGGER_PREFIX} getting source datasource for query with filtering {statement.where_clause_category} and grain {statement.grain}"
     )
     if not statement.output_components:
         raise ValueError(f"Statement has no output components {statement}")

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/parsing/parse_engine.py

@@ -17,7 +17,6 @@ from lark.tree import Meta
 from pydantic import ValidationError
 
 from trilogy.constants import (
-    CONFIG,
     DEFAULT_NAMESPACE,
     NULL_VALUE,
     MagicConstants,
@@ -108,7 +107,6 @@ from trilogy.core.models import (
     StructType,
     SubselectComparison,
     TupleWrapper,
-    UndefinedConcept,
     WhereClause,
     Window,
     WindowItem,
@@ -1030,81 +1028,15 @@ class ParseToObjects(Transformer):
         if not select_items:
             raise ValueError("Malformed select, missing select items")
 
-        output = SelectStatement(
+        return SelectStatement.from_inputs(
+            environment=self.environment,
             selection=select_items,
+            order_by=order_by,
             where_clause=where,
             having_clause=having,
             limit=limit,
-            order_by=order_by,
             meta=Metadata(line_number=meta.line),
         )
-        for parse_pass in [
-            1,
-            2,
-        ]:
-            # the first pass will result in all concepts being defined
-            # the second will get grains appropriately
-            # eg if someone does sum(x)->a, b+c -> z - we don't know if Z is a key to group by or an aggregate
-            # until after the first pass, and so don't know the grain of a
-
-            if parse_pass == 1:
-                grain = Grain.from_concepts(
-                    [
-                        x.content
-                        for x in output.selection
-                        if isinstance(x.content, Concept)
-                    ],
-                    where_clause=output.where_clause,
-                )
-            if parse_pass == 2:
-                grain = Grain.from_concepts(
-                    output.output_components, where_clause=output.where_clause
-                )
-            output.grain = grain
-            for item in select_items:
-                # we don't know the grain of an aggregate at assignment time
-                # so rebuild at this point in the tree
-                # TODO: simplify
-                if isinstance(item.content, ConceptTransform):
-                    new_concept = item.content.output.with_select_context(
-                        output.local_concepts,
-                        output.grain,
-                        environment=self.environment,
-                    )
-                    output.local_concepts[new_concept.address] = new_concept
-                    item.content.output = new_concept
-                    if parse_pass == 2 and CONFIG.select_as_definition:
-                        self.environment.add_concept(new_concept)
-                elif isinstance(item.content, UndefinedConcept):
-                    self.environment.concepts.raise_undefined(
-                        item.content.address,
-                        line_no=item.content.metadata.line_number,
-                        file=self.token_address,
-                    )
-                elif isinstance(item.content, Concept):
-                    # Sometimes cached values here don't have the latest info
-                    # but we can't just use environment, as it might not have the right grain.
-                    item.content = item.content.with_select_context(
-                        output.local_concepts,
-                        output.grain,
-                        environment=self.environment,
-                    )
-                    output.local_concepts[item.content.address] = item.content
-
-        if order_by:
-            output.order_by = order_by.with_select_context(
-                local_concepts=output.local_concepts,
-                grain=output.grain,
-                environment=self.environment,
-            )
-        if output.having_clause:
-            output.having_clause = output.having_clause.with_select_context(
-                local_concepts=output.local_concepts,
-                grain=output.grain,
-                environment=self.environment,
-            )
-        output.validate_syntax(self.environment)
-        return output
 
     @v_args(meta=True)
     def address(self, meta: Meta, args):

{pytrilogy-0.0.2.54 → pytrilogy-0.0.2.56}/trilogy/parsing/render.py

@@ -18,6 +18,7 @@ from trilogy.core.models import (
     Concept,
     ConceptDeclarationStatement,
     ConceptDerivation,
+    ConceptRef,
     ConceptTransform,
     Conditional,
     CopyStatement,
@@ -256,8 +257,18 @@ class Renderer:
             namespace = ""
         if not concept.lineage:
             if concept.purpose == Purpose.PROPERTY and concept.keys:
-                keys = ",".join([self.to_string(key) for key in concept.keys])
-                output = f"{concept.purpose.value} <{keys}>.{namespace}{concept.name} {self.to_string(concept.datatype)};"
+                if len(concept.keys) == 1:
+                    output = f"{concept.purpose.value} {self.to_string(ConceptRef(address=list(concept.keys)[0]))}.{namespace}{concept.name} {self.to_string(concept.datatype)};"
+                else:
+                    keys = ",".join(
+                        sorted(
+                            list(
+                                self.to_string(ConceptRef(address=x))
+                                for x in concept.keys
+                            )
+                        )
+                    )
+                    output = f"{concept.purpose.value} <{keys}>.{namespace}{concept.name} {self.to_string(concept.datatype)};"
             else:
                 output = f"{concept.purpose.value} {namespace}{concept.name} {self.to_string(concept.datatype)};"
         else:
@@ -377,8 +388,16 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "FilterItem"):
+
         return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
 
+    @to_string.register
+    def _(self, arg: "ConceptRef"):
+        ns, base = arg.address.rsplit(".", 1)
+        if ns == DEFAULT_NAMESPACE:
+            return base
+        return arg.address
+
     @to_string.register
     def _(self, arg: "ImportStatement"):
         path: str = str(arg.path).replace("\\", ".")