pytrilogy 0.0.2.10__tar.gz → 0.0.2.12__tar.gz

{pytrilogy-0.0.2.10/pytrilogy.egg-info → pytrilogy-0.0.2.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.10
+Version: 0.0.2.12
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.10
+Version: 0.0.2.12
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/tests/test_models.py

@@ -12,6 +12,7 @@ from trilogy.core.models import (
     Comparison,
     Join,
     JoinKey,
+    Concept,
 )
 
 
@@ -69,6 +70,23 @@ def test_concept(test_environment, test_environment_graph):
     )
 
 
+def test_concept_filter(test_environment, test_environment_graph):
+    test_concept: Concept = list(test_environment.concepts.values())[0]
+    new = test_concept.with_filter(
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    )
+    new2 = test_concept.with_filter(
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    )
+
+    assert new.name == new2.name != test_concept.name
+
+    new3 = new.with_filter(
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    )
+    assert new3 == new
+
+
 def test_conditional(test_environment, test_environment_graph):
     test_concept = list(test_environment.concepts.values())[-1]
 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/tests/test_parsing.py

@@ -492,3 +492,27 @@ select x;
     results = Dialects.DUCK_DB.default_executor().generate_sql(text)[0]
 
     assert "abcdef as test" in results, results
+
+
+def test_filter_concise():
+
+    text = """
+key x int;
+key y int;
+
+datasource test (
+x:x,
+y:y)
+grain(x)
+address `abc:def`
+;
+
+auto filtered_test <- x ? y > 10;
+
+select filtered_test;
+"""
+    env, parsed = parse_text(text)
+
+    results = Dialects.DUCK_DB.default_executor().generate_sql(text)[0]
+
+    assert "filtered_test" in results, results

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.10"
+__version__ = "0.0.2.12"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/enums.py

@@ -229,7 +229,6 @@ class ComparisonOperator(Enum):
     ILIKE = "ilike"
     CONTAINS = "contains"
     ELSE = "else"
-    BETWEEN = "between"
 
     @classmethod
     def _missing_(cls, value):

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/environment_helpers.py

@@ -1,6 +1,15 @@
-from trilogy.core.models import DataType, Concept, Environment, Function, Metadata
+from trilogy.core.models import (
+    DataType,
+    Concept,
+    Environment,
+    Function,
+    Metadata,
+    StructType,
+)
+from trilogy.core.functions import AttrAccess
 from trilogy.core.enums import Purpose, FunctionType, ConceptSource
 from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.parsing.common import process_function_args, arg_to_datatype, Meta
 
 
 def generate_date_concepts(concept: Concept, environment: Environment):
@@ -142,15 +151,44 @@ def generate_key_concepts(concept: Concept, environment: Environment):
         environment.add_concept(new_concept, add_derived=False)
 
 
-def generate_related_concepts(concept: Concept, environment: Environment):
+def generate_related_concepts(
+    concept: Concept,
+    environment: Environment,
+    meta: Meta | None = None,
+    add_derived: bool = False,
+):
     """Auto populate common derived concepts on types"""
-    if concept.purpose == Purpose.KEY:
+    if concept.purpose == Purpose.KEY and add_derived:
         generate_key_concepts(concept, environment)
-    if concept.datatype == DataType.DATE:
+
+    # datatype types
+    if concept.datatype == DataType.DATE and add_derived:
         generate_date_concepts(concept, environment)
-    elif concept.datatype == DataType.DATETIME:
+    elif concept.datatype == DataType.DATETIME and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
-    elif concept.datatype == DataType.TIMESTAMP:
+    elif concept.datatype == DataType.TIMESTAMP and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
+
+    if isinstance(concept.datatype, StructType):
+        for key, value in concept.datatype.fields_map.items():
+            args = process_function_args(
+                [concept, key], meta=meta, environment=environment
+            )
+            auto = Concept(
+                name=key,
+                datatype=arg_to_datatype(value),
+                purpose=Purpose.PROPERTY,
+                namespace=(
+                    environment.namespace + "." + concept.name
+                    if environment.namespace
+                    and environment.namespace != DEFAULT_NAMESPACE
+                    else concept.name
+                ),
+                lineage=AttrAccess(args),
+            )
+            environment.add_concept(auto, meta=meta)
+            if isinstance(value, Concept):
+                environment.merge_concept(auto, value, modifiers=[])
+                assert value.pseudonyms is not None

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/models.py

@@ -300,7 +300,7 @@ class MapType(BaseModel):
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str]
 
     @property
     def data_type(self):
@@ -801,15 +801,18 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     ) -> "Concept":
         from trilogy.utility import string_to_hash
 
-        name = string_to_hash(self.name + str(condition))
+        if self.lineage and isinstance(self.lineage, FilterItem):
+            if self.lineage.where.conditional == condition:
+                return self
+        hash = string_to_hash(self.name + str(condition))
         new = Concept(
-            name=f"{self.name}_{name}",
+            name=f"{self.name}_filter_{hash}",
             datatype=self.datatype,
             purpose=self.purpose,
             metadata=self.metadata,
             lineage=FilterItem(content=self, where=WhereClause(conditional=condition)),
-            keys=None,
-            grain=(self.grain if self.purpose == Purpose.PROPERTY else Grain()),
+            keys=(self.keys if self.purpose == Purpose.PROPERTY else None),
+            grain=self.grain if self.grain else Grain(components=[]),
             namespace=self.namespace,
             modifiers=self.modifiers,
             pseudonyms=self.pseudonyms,
@@ -842,6 +845,16 @@ class Grain(Mergeable, BaseModel):
         v2 = sorted(final, key=lambda x: x.name)
         return v2
 
+    def with_filter(
+        self,
+        condition: "Conditional | Comparison | Parenthetical",
+        environment: Environment | None = None,
+    ) -> "Grain":
+        return Grain(
+            components=[c.with_filter(condition, environment) for c in self.components],
+            nested=self.nested,
+        )
+
     @property
     def components_copy(self) -> List[Concept]:
         return [*self.components]
@@ -1680,6 +1693,9 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
                 )
             ):
                 output.append(item)
+        # TODO: explore implicit filtering more
+        # if self.where_clause.conditional and self.where_clause_category == SelectFiltering.IMPLICIT:
+        #     output =[x.with_filter(self.where_clause.conditional) for x in output]
         return Grain(
             components=unique(output, "address"), where_clause=self.where_clause
         )
@@ -2103,16 +2119,19 @@ class Datasource(Namespaced, BaseModel):
 
 
 class UnnestJoin(BaseModel):
-    concept: Concept
+    concepts: list[Concept]
+    parent: Function
     alias: str = "unnest"
     rendering_required: bool = True
 
     def __hash__(self):
-        return (self.alias + self.concept.address).__hash__()
+        return (
+            self.alias + "".join([str(s.address) for s in self.concepts])
+        ).__hash__()
 
 
 class InstantiatedUnnestJoin(BaseModel):
-    concept: Concept
+    concept_to_unnest: Concept
     alias: str = "unnest"
 
 
@@ -2252,6 +2271,7 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
+
         return v
 
     @field_validator("input_concepts")
@@ -2271,8 +2291,13 @@ class QueryDatasource(BaseModel):
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
                 continue
+            concept: Concept
             for concept in values[key]:
-                if concept.address not in v and CONFIG.validate_missing:
+                if (
+                    concept.address not in v
+                    and not any(x in v for x in concept.pseudonyms)
+                    and CONFIG.validate_missing
+                ):
                     raise SyntaxError(
                         f"Missing source map for {concept.address} on {key}, have {v}"
                     )
@@ -2517,7 +2542,7 @@ class CTE(BaseModel):
                     )
                 ]
                 for join in self.joins:
-                    if isinstance(join, UnnestJoin) and join.concept == concept:
+                    if isinstance(join, UnnestJoin) and concept in join.concepts:
                         join.rendering_required = False
 
                 self.parent_ctes = [
@@ -2980,8 +3005,8 @@ class EnvironmentDatasourceDict(dict):
         except KeyError:
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key)
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1])
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1])
             raise
 
     def values(self) -> ValuesView[Datasource]:  # type: ignore
@@ -3011,8 +3036,8 @@ class EnvironmentConceptDict(dict):
             return super(EnvironmentConceptDict, self).__getitem__(key)
 
         except KeyError:
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1], line_no)
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1], line_no)
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key, line_no)
             if not self.fail_on_missing:
@@ -3277,10 +3302,9 @@ class Environment(BaseModel):
             self.concepts[concept.name] = concept
         else:
             self.concepts[concept.address] = concept
-        if add_derived:
-            from trilogy.core.environment_helpers import generate_related_concepts
+        from trilogy.core.environment_helpers import generate_related_concepts
 
-            generate_related_concepts(concept, self)
+        generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
         self.gen_concept_list_caches()
         return concept
 
@@ -3407,14 +3431,6 @@ class Comparison(
             raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
-        if self.operator == ComparisonOperator.BETWEEN:
-            if (
-                not isinstance(self.right, ComparisonOperator)
-                and self.right.operator == BooleanOperator.AND
-            ):
-                raise SyntaxError(
-                    f"Between operator must have two operands with and, not {self.right}"
-                )
 
     def __add__(self, other):
         if other is None:
@@ -4074,8 +4090,13 @@ class RowsetDerivationStatement(Namespaced, BaseModel):
         output: list[Concept] = []
         orig: dict[str, Concept] = {}
         for orig_concept in self.select.output_components:
+            name = orig_concept.name
+            if isinstance(orig_concept.lineage, FilterItem):
+                if orig_concept.lineage.where == self.select.where_clause:
+                    name = orig_concept.lineage.content.name
+
             new_concept = Concept(
-                name=orig_concept.name,
+                name=name,
                 datatype=orig_concept.datatype,
                 purpose=orig_concept.purpose,
                 lineage=RowsetItem(

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/optimization.py

@@ -10,6 +10,7 @@ from trilogy.core.optimizations import (
     OptimizationRule,
     InlineConstant,
     PredicatePushdown,
+    PredicatePushdownRemove,
     InlineDatasource,
 )
 
@@ -34,6 +35,31 @@ MAX_OPTIMIZATION_LOOPS = 100
 #     return parent
 
 
+def reorder_ctes(
+    input: list[CTE],
+):
+    import networkx as nx
+
+    # Create a directed graph
+    G = nx.DiGraph()
+    mapping: dict[str, CTE] = {}
+    for cte in input:
+        mapping[cte.name] = cte
+        for parent in cte.parent_ctes:
+            G.add_edge(parent.name, cte.name)
+    # Perform topological sort (only works for DAGs)
+    try:
+        topological_order = list(nx.topological_sort(G))
+        if not topological_order:
+            return input
+        return [mapping[x] for x in topological_order]
+    except nx.NetworkXUnfeasible as e:
+        print(
+            "The graph is not a DAG (contains cycles) and cannot be topologically sorted."
+        )
+        raise e
+
+
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
@@ -169,20 +195,22 @@ def optimize_ctes(
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
-
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdownRemove())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False
         while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
             actions_taken = False
             # assume we go through all CTEs once
-            look_at = [root_cte, *input]
+            look_at = [root_cte, *reversed(input)]
             inverse_map = gen_inverse_map(look_at)
             for cte in look_at:
                 opt = rule.optimize(cte, inverse_map)
                 actions_taken = actions_taken or opt
             complete = not actions_taken
             loops += 1
+            input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
         logger.info(f"finished checking for {type(rule).__name__} in {loops} loops")
 
-    return filter_irrelevant_ctes(input, root_cte)
+    return reorder_ctes(filter_irrelevant_ctes(input, root_cte))

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/optimizations/__init__.py

@@ -1,6 +1,6 @@
 from .inline_constant import InlineConstant
 from .inline_datasource import InlineDatasource
-from .predicate_pushdown import PredicatePushdown
+from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 from .base_optimization import OptimizationRule
 
 __all__ = [
@@ -8,4 +8,5 @@ __all__ = [
     "InlineConstant",
     "InlineDatasource",
     "PredicatePushdown",
+    "PredicatePushdownRemove",
 ]

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/optimizations/predicate_pushdown.py

@@ -114,48 +114,6 @@ class PredicatePushdown(OptimizationRule):
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
-
-        parent_filter_status = {
-            parent.name: is_child_of(cte.condition, parent.condition)
-            for parent in cte.parent_ctes
-        }
-        # flatten existnce argument tuples to a list
-
-        flattened_existence = [
-            x.address for y in cte.condition.existence_arguments for x in y
-        ]
-
-        existence_only = [
-            parent.name
-            for parent in cte.parent_ctes
-            if all([x.address in flattened_existence for x in parent.output_columns])
-            and len(flattened_existence) > 0
-        ]
-        if all(
-            [
-                value
-                for key, value in parent_filter_status.items()
-                if key not in existence_only
-            ]
-        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
-            self.log(
-                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
-            )
-            cte.condition = None
-            # remove any "parent" CTEs that provided only existence inputs
-            if existence_only:
-                original = [y.name for y in cte.parent_ctes]
-                cte.parent_ctes = [
-                    x for x in cte.parent_ctes if x.name not in existence_only
-                ]
-                self.log(
-                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
-                )
-            return True
-        else:
-            self.log(
-                f"Could not remove filter from {cte.name}, as not all parents have the same filter: {parent_filter_status}"
-            )
         if self.complete.get(cte.name):
             self.debug("Have done this CTE before")
             return False
@@ -197,3 +155,63 @@ class PredicatePushdown(OptimizationRule):
 
         self.complete[cte.name] = True
         return optimized
+
+
+class PredicatePushdownRemove(OptimizationRule):
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        optimized = False
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+
+        parent_filter_status = {
+            parent.name: is_child_of(cte.condition, parent.condition)
+            for parent in cte.parent_ctes
+        }
+        # flatten existnce argument tuples to a list
+
+        flattened_existence = [
+            x.address for y in cte.condition.existence_arguments for x in y
+        ]
+
+        existence_only = [
+            parent.name
+            for parent in cte.parent_ctes
+            if all([x.address in flattened_existence for x in parent.output_columns])
+            and len(flattened_existence) > 0
+        ]
+        if all(
+            [
+                value
+                for key, value in parent_filter_status.items()
+                if key not in existence_only
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log(
+                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
+            )
+            cte.condition = None
+            # remove any "parent" CTEs that provided only existence inputs
+            if existence_only:
+                original = [y.name for y in cte.parent_ctes]
+                cte.parent_ctes = [
+                    x for x in cte.parent_ctes if x.name not in existence_only
+                ]
+                self.log(
+                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
+                )
+            return True
+
+        self.complete[cte.name] = True
+        return optimized

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/processing/concept_strategies_v3.py

@@ -180,7 +180,10 @@ def generate_candidates_restrictive(
     local_candidates = [
         x
         for x in list(candidates)
-        if x.address not in exhausted and x.granularity != Granularity.SINGLE_ROW
+        if x.address not in exhausted
+        and x.granularity != Granularity.SINGLE_ROW
+        and x.address not in priority_concept.pseudonyms
+        and priority_concept.address not in x.pseudonyms
     ]
     combos: list[list[Concept]] = []
     grain_check = Grain(components=[*local_candidates]).components_copy
@@ -191,6 +194,7 @@ def generate_candidates_restrictive(
         ):
             combos.append(local_candidates)
     combos.append(grain_check)
+    # combos.append(local_candidates)
     # append the empty set for sourcing concept by itself last
     combos.append([])
     return combos
@@ -607,7 +611,7 @@ def _search_concepts(
         if len(stack) == 1:
             output = stack[0]
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]} and {output.resolve().source_map}"
             )
             return output
 
@@ -645,6 +649,7 @@ def _search_concepts(
             depth=depth,
             source_concepts=search_concepts,
             history=history,
+            search_conditions=conditions,
         )
 
         if expanded:
@@ -656,8 +661,7 @@ def _search_concepts(
                 if x.address not in [y.address for y in mandatory_list]
                 and x not in ex_resolve.grain.components
             ]
-            expanded.output_concepts = mandatory_list
-            expanded.rebuild_cache()
+            expanded.set_output_concepts(mandatory_list)
 
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.12}/trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,7 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
@@ -31,12 +32,17 @@ def gen_basic_node(
     )
 
     local_optional_redundant = [x for x in local_optional if x in parent_concepts]
-    attempts = [(parent_concepts, [concept] + local_optional_redundant)]
-    from itertools import combinations
+    attempts: List[tuple[list[Concept], list[Concept]]] = [
+        (parent_concepts, [concept] + local_optional_redundant)
+    ]
+    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
 
     if local_optional:
-        for combo in range(1, len(local_optional) + 1):
-            combos = combinations(local_optional, combo)
+        for combo in range(1, len(non_equivalent_optional) + 1):
+            combos = combinations(non_equivalent_optional, combo)
             for optional_set in combos:
                 attempts.append(
                     (
@@ -64,13 +70,10 @@ def gen_basic_node(
                 continue
             if all(x in source.partial_concepts for source in sources):
                 partials.append(x)
-        outputs = parent_node.output_concepts + [concept]
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in outputs]}"
-        )
-        # parents.resolve()
 
         parent_node.add_output_concept(concept)
+        for x in equivalent_optional:
+            parent_node.add_output_concept(x)
 
         parent_node.remove_output_concepts(
             [
@@ -79,6 +82,9 @@ def gen_basic_node(
                 if x.address not in [y.address for y in basic_output]
             ]
         )
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in parent_node.output_concepts]}"
+        )
         return parent_node
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"