pytrilogy 0.0.3.43__tar.gz → 0.0.3.45__tar.gz

{pytrilogy-0.0.3.43/pytrilogy.egg-info → pytrilogy-0.0.3.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.43
+Version: 0.0.3.45
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.43
+Version: 0.0.3.45
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/tests/test_models.py

@@ -213,9 +213,6 @@ def test_comparison():
     with raises(SyntaxError):
         Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
 
-    with raises(SyntaxError):
-        Comparison(left=1, right=1, operator=ComparisonOperator.IN)
-
     # this should not error
     Comparison(left=1, right=[1, 2, 3], operator=ComparisonOperator.IN)
 

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/tests/test_parsing_failures.py

@@ -77,10 +77,11 @@ SELECT
     1+2->scalar
 ;
 """
-    with raises(ParseError):
+    with raises(ParseError) as e:
         env, parsed = parse_text(
             x, parse_config=Parsing(strict_name_shadow_enforcement=True)
         )
+        assert "abc" in str(e)
     x = """
 key scalar int;    
 property scalar.int_array list<int>;

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.43"
+__version__ = "0.0.3.45"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/core/models/author.py

@@ -629,18 +629,8 @@ class Comparison(ConceptArgs, Mergeable, DataTyped, Namespaced, BaseModel):
                 )
         elif self.operator in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
             right_type = arg_to_datatype(self.right)
-            if not any(
-                [
-                    isinstance(self.right, ConceptRef),
-                    right_type in (DataType.LIST,),
-                    isinstance(right_type, (ListType, ListWrapper, TupleWrapper)),
-                ]
-            ):
-                raise SyntaxError(
-                    f"Cannot use {self.operator.value} with non-list, non-tuple, non-concept object {self.right} in {str(self)}"
-                )
 
-            elif isinstance(right_type, ListType) and not is_compatible_datatype(
+            if isinstance(right_type, ListType) and not is_compatible_datatype(
                 arg_to_datatype(self.left), right_type.value_data_type
             ):
                 raise SyntaxError(
@@ -1916,7 +1906,7 @@ class AggregateWrapper(Mergeable, DataTyped, ConceptArgs, Namespaced, BaseModel)
 
 
 class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
-    content: ConceptRef
+    content: FuncArgs
     where: "WhereClause"
 
     @field_validator("content", mode="before")
@@ -1932,13 +1922,21 @@ class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "FilterItem":
         return FilterItem.model_construct(
-            content=self.content.with_merge(source, target, modifiers),
+            content=(
+                self.content.with_merge(source, target, modifiers)
+                if isinstance(self.content, Mergeable)
+                else self.content
+            ),
             where=self.where.with_merge(source, target, modifiers),
         )
 
     def with_namespace(self, namespace: str) -> "FilterItem":
         return FilterItem.model_construct(
-            content=self.content.with_namespace(namespace),
+            content=(
+                self.content.with_namespace(namespace)
+                if isinstance(self.content, Namespaced)
+                else self.content
+            ),
             where=self.where.with_namespace(namespace),
         )
 

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/core/models/build.py

@@ -1496,22 +1496,43 @@ class Factory:
     ):
         return base
 
+    def instantiate_concept(
+        self,
+        arg: (
+            AggregateWrapper
+            | FunctionCallWrapper
+            | WindowItem
+            | FilterItem
+            | Function
+            | ListWrapper[Any]
+            | MapWrapper[Any, Any]
+            | int
+            | float
+            | str
+        ),
+    ) -> tuple[Concept, BuildConcept]:
+        from trilogy.parsing.common import arbitrary_to_concept
+
+        new = arbitrary_to_concept(
+            arg,
+            environment=self.environment,
+        )
+        built = self.build(new)
+        self.local_concepts[new.address] = built
+        return new, built
+
     @build.register
     def _(self, base: None) -> None:
         return base
 
     @build.register
     def _(self, base: Function) -> BuildFunction | BuildAggregateWrapper:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         raw_args: list[Concept | FuncArgs] = []
         for arg in base.arguments:
             # to do proper discovery, we need to inject virtual intermediate ocncepts
             if isinstance(arg, (AggregateWrapper, FilterItem, WindowItem)):
-                narg = arbitrary_to_concept(
-                    arg,
-                    environment=self.environment,
-                )
+                narg, _ = self.instantiate_concept(arg)
                 raw_args.append(narg)
             else:
                 raw_args.append(arg)
@@ -1532,24 +1553,16 @@ class Factory:
                 for x in arguments:
                     if isinstance(x, (ConceptRef, Concept)):
                         final_args.append(x)
-                    elif isinstance(x, (AggregateWrapper, FilterItem, WindowItem)):
-                        newx = arbitrary_to_concept(
-                            x,
-                            environment=self.environment,
-                        )
-                        final_args.append(newx)
                     else:
                         # constants, etc, can be ignored for group
                         continue
-                group_base = arbitrary_to_concept(
+                _, rval = self.instantiate_concept(
                     AggregateWrapper(
                         function=group_base.lineage.function,
                         by=final_args,
-                    ),
-                    environment=self.environment,
+                    )
                 )
 
-                rval = self.build(group_base)
                 return BuildFunction.model_construct(
                     operator=base.operator,
                     arguments=[rval, *[self.build(c) for c in raw_args[1:]]],
@@ -1580,20 +1593,13 @@ class Factory:
 
     @build.register
     def _(self, base: CaseWhen) -> BuildCaseWhen:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         comparison = base.comparison
         if isinstance(comparison, (AggregateWrapper, FilterItem, WindowItem)):
-            comparison = arbitrary_to_concept(
-                comparison,
-                environment=self.environment,
-            )
+            comparison, _ = self.instantiate_concept(comparison)
         expr: Concept | FuncArgs = base.expr
         if isinstance(expr, (AggregateWrapper, FilterItem, WindowItem)):
-            expr = arbitrary_to_concept(
-                expr,
-                environment=self.environment,
-            )
+            expr, _ = self.instantiate_concept(expr)
         return BuildCaseWhen.model_construct(
             comparison=self.build(comparison),
             expr=self.build(expr),
@@ -1615,6 +1621,7 @@ class Factory:
         else:
             build_lineage = None
         derivation = Concept.calculate_derivation(build_lineage, base.purpose)
+
         granularity = Concept.calculate_granularity(
             derivation, final_grain, build_lineage
         )
@@ -1675,16 +1682,12 @@ class Factory:
 
     @build.register
     def _(self, base: OrderItem) -> BuildOrderItem:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         bexpr: Any
         if isinstance(base.expr, (AggregateWrapper, WindowItem, FilterItem)) or (
             isinstance(base.expr, Function) and base.expr.operator == FunctionType.GROUP
         ):
-            bexpr = arbitrary_to_concept(
-                base.expr,
-                environment=self.environment,
-            )
+            bexpr, _ = self.instantiate_concept(base.expr)
         else:
             bexpr = base.expr
         return BuildOrderItem.model_construct(
@@ -1707,15 +1710,10 @@ class Factory:
 
     @build.register
     def _(self, base: WindowItem) -> BuildWindowItem:
-        # to do proper discovery, we need to inject virtual intermediate ocncepts
-        from trilogy.parsing.common import arbitrary_to_concept
 
         content: Concept | FuncArgs = base.content
         if isinstance(content, (AggregateWrapper, FilterItem, WindowItem)):
-            content = arbitrary_to_concept(
-                content,
-                environment=self.environment,
-            )
+            content, _ = self.instantiate_concept(content)
         final_by = []
         for x in base.order_by:
             if (
@@ -1743,30 +1741,26 @@ class Factory:
 
     @build.register
     def _(self, base: SubselectComparison) -> BuildSubselectComparison:
+        right: Any = base.right
+        if isinstance(base.right, (AggregateWrapper, WindowItem, FilterItem, Function)):
+            right_c, _ = self.instantiate_concept(base.right)
+            right = right_c
         return BuildSubselectComparison.model_construct(
-            left=(self.build(base.left)),
-            right=(self.build(base.right)),
+            left=self.build(base.left),
+            right=self.build(right),
             operator=base.operator,
         )
 
     @build.register
     def _(self, base: Comparison) -> BuildComparison:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         left = base.left
         if isinstance(left, (AggregateWrapper, WindowItem, FilterItem)):
-            left_c = arbitrary_to_concept(
-                left,
-                environment=self.environment,
-            )
+            left_c, _ = self.instantiate_concept(left)
             left = left_c  # type: ignore
         right = base.right
         if isinstance(right, (AggregateWrapper, WindowItem, FilterItem)):
-            right_c = arbitrary_to_concept(
-                right,
-                environment=self.environment,
-            )
-
+            right_c, _ = self.instantiate_concept(right)
             right = right_c  # type: ignore
         return BuildComparison.model_construct(
             left=(self.build(left)),
@@ -1826,6 +1820,13 @@ class Factory:
 
     @build.register
     def _(self, base: FilterItem) -> BuildFilterItem:
+        if isinstance(
+            base.content, (Function, AggregateWrapper, WindowItem, FilterItem)
+        ):
+            _, built = self.instantiate_concept(base.content)
+            return BuildFilterItem.model_construct(
+                content=built, where=self.build(base.where)
+            )
         return BuildFilterItem.model_construct(
             content=self.build(base.content), where=self.build(base.where)
         )
@@ -1969,6 +1970,10 @@ class Factory:
             new.datasources[k] = self.build(d)
         for k, a in base.alias_origin_lookup.items():
             new.alias_origin_lookup[k] = self.build(a)
+        # add in anything that was built as a side-effect
+        for bk, bv in self.local_concepts.items():
+            if bk not in new.concepts:
+                new.concepts[bk] = bv
         new.gen_concept_list_caches()
         return new
 

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/core/processing/concept_strategies_v3.py

@@ -748,6 +748,7 @@ def search_concepts(
     accept_partial: bool = False,
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
+    logger.error(f"starting search for {mandatory_list}")
     hist = history.get_history(
         search=mandatory_list, accept_partial=accept_partial, conditions=conditions
     )
@@ -1094,6 +1095,7 @@ def _search_concepts(
     logger.error(
         f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Could not resolve concepts {[c.address for c in mandatory_list]}, network outcome was {complete}, missing {all_mandatory - found},"
     )
+
     return None
 
 

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/core/processing/node_generators/common.py

@@ -178,7 +178,7 @@ def gen_enrichment_node(
             for x in extra_required
         ):
             log_lambda(
-                f"{str(type(base_node).__name__)} returning property optimized enrichment node"
+                f"{str(type(base_node).__name__)} returning property optimized enrichment node for {extra_required[0].keys}"
             )
             return gen_property_enrichment_node(
                 base_node,

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/core/processing/node_generators/window_node.py

@@ -32,7 +32,9 @@ def resolve_window_parent_concepts(
     if concept.lineage.order_by:
         for item in concept.lineage.order_by:
             base += item.concept_arguments
-
+    if concept.grain:
+        for gitem in concept.grain.components:
+            base.append(environment.concepts[gitem])
     return concept.lineage.content, unique(base, "address")
 
 
@@ -131,20 +133,24 @@ def gen_window_node(
     )
     _window_node.rebuild_cache()
     _window_node.resolve()
+
     window_node = StrategyNode(
         input_concepts=[concept] + additional_outputs + parent_concepts + targets,
         output_concepts=[concept] + additional_outputs + parent_concepts + targets,
         environment=environment,
         parents=[_window_node],
         preexisting_conditions=conditions.conditional if conditions else None,
-        # hidden_concepts=[
-        #     x.address for x in parent_concepts if x.address not in local_optional
-        # ],
+        grain=BuildGrain.from_concepts(
+            concepts=[concept] + additional_outputs + parent_concepts + targets,
+            environment=environment,
+        ),
     )
     if not non_equivalent_optional:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no optional concepts, returning window node"
         )
+        # prune outputs if we don't need join keys
+        window_node.set_output_concepts([concept] + additional_outputs + targets)
         return window_node
 
     missing_optional = [

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/core/processing/nodes/__init__.py

@@ -1,5 +1,6 @@
 from pydantic import BaseModel, ConfigDict, Field
 
+from trilogy.core.exceptions import UnresolvableQueryException
 from trilogy.core.models.build import BuildConcept, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.models.environment import Environment
@@ -18,7 +19,7 @@ class History(BaseModel):
     base_environment: Environment
     history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     select_history: dict[str, StrategyNode | None] = Field(default_factory=dict)
-    started: set[str] = Field(default_factory=set)
+    started: dict[str, int] = Field(default_factory=dict)
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     def _concepts_to_lookup(
@@ -27,13 +28,10 @@ class History(BaseModel):
         accept_partial: bool,
         conditions: BuildWhereClause | None = None,
     ) -> str:
+        base = sorted([c.address for c in search])
         if conditions:
-            return (
-                "-".join([c.address for c in search])
-                + str(accept_partial)
-                + str(conditions)
-            )
-        return "-".join([c.address for c in search]) + str(accept_partial)
+            return "-".join(base) + str(accept_partial) + str(conditions)
+        return "-".join(base) + str(accept_partial)
 
     def search_to_history(
         self,
@@ -80,13 +78,19 @@ class History(BaseModel):
         accept_partial: bool = False,
         conditions: BuildWhereClause | None = None,
     ):
-        self.started.add(
-            self._concepts_to_lookup(
-                search,
-                accept_partial=accept_partial,
-                conditions=conditions,
-            )
+        key = self._concepts_to_lookup(
+            search,
+            accept_partial=accept_partial,
+            conditions=conditions,
         )
+        if key in self.started:
+            self.started[key] += 1
+        else:
+            self.started[key] = 1
+        if self.started[key] > 5:
+            raise UnresolvableQueryException(
+                f"Was unable to resolve datasources to serve this query from model; unresolvable set was {search}. You may be querying unrelated concepts."
+            )
 
     def log_end(
         self,
@@ -94,13 +98,13 @@ class History(BaseModel):
         accept_partial: bool = False,
         conditions: BuildWhereClause | None = None,
     ):
-        self.started.discard(
-            self._concepts_to_lookup(
-                search,
-                accept_partial=accept_partial,
-                conditions=conditions,
-            )
+        key = self._concepts_to_lookup(
+            search,
+            accept_partial=accept_partial,
+            conditions=conditions,
         )
+        if key in self.started:
+            del self.started[key]
 
     def check_started(
         self,

{pytrilogy-0.0.3.43 → pytrilogy-0.0.3.45}/trilogy/parsing/common.py

@@ -458,7 +458,25 @@ def filter_item_to_concept(
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
-    cparent = environment.concepts[parent.content.address]
+    if isinstance(parent.content, ConceptRef):
+        cparent = environment.concepts[parent.content.address]
+    elif isinstance(
+        parent.content,
+        (
+            FilterItem,
+            AggregateWrapper,
+            FunctionCallWrapper,
+            WindowItem,
+            Function,
+            ListWrapper,
+            MapWrapper,
+        ),
+    ):
+        cparent = arbitrary_to_concept(parent.content, environment, namespace=namespace)
+    else:
+        raise NotImplementedError(
+            f"Filter item with non ref content {parent.content} not yet supported"
+        )
     modifiers = get_upstream_modifiers(
         cparent.concept_arguments, environment=environment
     )
@@ -494,24 +512,6 @@ def window_item_to_concept(
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
-    # if isinstance(
-    #     parent.content,
-    #     (
-    #         AggregateWrapper
-    #         | FunctionCallWrapper
-    #         | WindowItem
-    #         | FilterItem
-    #         | Function
-    #         | ListWrapper
-    #         | MapWrapper
-    #     ),
-    # ):
-    #     new_parent = arbitrary_to_concept(
-    #         parent.content, environment=environment, namespace=namespace
-    #     )
-    #     environment.add_concept(new_parent)
-    #     parent = parent.model_copy(update={"content": new_parent.reference})
-
     if not isinstance(parent.content, ConceptRef):
         raise NotImplementedError(
             f"Window function wiht non ref content {parent.content} not yet supported"
@@ -523,16 +523,26 @@ def window_item_to_concept(
         local_purpose, keys = get_purpose_and_keys(None, (bcontent,), environment)
     else:
         local_purpose = Purpose.PROPERTY
-        keys = {
-            bcontent.address,
-        }
+        keys = Grain.from_concepts(
+            [bcontent.address] + [y.address for y in parent.over], environment
+        ).components
 
+    # when including the order by in discovery grain
     if parent.order_by:
         grain_components = parent.over + [bcontent.output]
         for item in parent.order_by:
-            grain_components += item.concept_arguments
+            # confirm that it's not just an aggregate at the grain of the stuff we're already keying of of
+            # in which case we can ignore contributions
+            if (
+                isinstance(item.expr, AggregateWrapper)
+                and set([x.address for x in item.expr.by]) == keys
+            ):
+                continue
+            else:
+                grain_components += item.concept_arguments
     else:
         grain_components = parent.over + [bcontent.output]
+
     final_grain = Grain.from_concepts(grain_components, environment)
     modifiers = get_upstream_modifiers(bcontent.concept_arguments, environment)
     datatype = parent.content.datatype
@@ -651,7 +661,9 @@ def rowset_concept(
     orig_concept = environment.concepts[orig_address.address]
     name = orig_concept.name
     if isinstance(orig_concept.lineage, FilterItem):
-        if orig_concept.lineage.where == rowset.select.where_clause:
+        if orig_concept.lineage.where == rowset.select.where_clause and isinstance(
+            orig_concept.lineage.content, (ConceptRef, Concept)
+        ):
             name = environment.concepts[orig_concept.lineage.content.address].name
     base_namespace = (
         f"{rowset.name}.{orig_concept.namespace}"
@@ -761,7 +773,10 @@ def arbitrary_to_concept(
         )
     elif isinstance(parent, FilterItem):
         if not name:
-            name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{parent.content.name}_{string_to_hash(str(parent))}"
+            if isinstance(parent.content, ConceptRef):
+                name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{parent.content.name}_{string_to_hash(str(parent))}"
+            else:
+                name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{string_to_hash(str(parent))}"
         return filter_item_to_concept(
             parent,
             name,