pytrilogy 0.0.2.11__tar.gz → 0.0.2.13__tar.gz

{pytrilogy-0.0.2.11/pytrilogy.egg-info → pytrilogy-0.0.2.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.11
+Version: 0.0.2.13
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.11
+Version: 0.0.2.13
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.11"
+__version__ = "0.0.2.13"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/constants.py

@@ -28,6 +28,11 @@ class Optimizations:
     direct_return: bool = True
 
 
+@dataclass
+class Comments:
+    pass
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/enums.py

@@ -62,6 +62,8 @@ class Modifier(Enum):
         strval = str(value)
         if strval == "~":
             return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
 
@@ -229,7 +231,6 @@ class ComparisonOperator(Enum):
     ILIKE = "ilike"
     CONTAINS = "contains"
     ELSE = "else"
-    BETWEEN = "between"
 
     @classmethod
     def _missing_(cls, value):
@@ -274,6 +275,7 @@ class SourceType(Enum):
     CONSTANT = "constant"
     ROWSET = "rowset"
     MERGE = "merge"
+    BASIC = "basic"
 
 
 class ShowCategory(Enum):

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/environment_helpers.py

@@ -1,6 +1,15 @@
-from trilogy.core.models import DataType, Concept, Environment, Function, Metadata
+from trilogy.core.models import (
+    DataType,
+    Concept,
+    Environment,
+    Function,
+    Metadata,
+    StructType,
+)
+from trilogy.core.functions import AttrAccess
 from trilogy.core.enums import Purpose, FunctionType, ConceptSource
 from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.parsing.common import process_function_args, arg_to_datatype, Meta
 
 
 def generate_date_concepts(concept: Concept, environment: Environment):
@@ -142,15 +151,44 @@ def generate_key_concepts(concept: Concept, environment: Environment):
         environment.add_concept(new_concept, add_derived=False)
 
 
-def generate_related_concepts(concept: Concept, environment: Environment):
+def generate_related_concepts(
+    concept: Concept,
+    environment: Environment,
+    meta: Meta | None = None,
+    add_derived: bool = False,
+):
     """Auto populate common derived concepts on types"""
-    if concept.purpose == Purpose.KEY:
+    if concept.purpose == Purpose.KEY and add_derived:
         generate_key_concepts(concept, environment)
-    if concept.datatype == DataType.DATE:
+
+    # datatype types
+    if concept.datatype == DataType.DATE and add_derived:
         generate_date_concepts(concept, environment)
-    elif concept.datatype == DataType.DATETIME:
+    elif concept.datatype == DataType.DATETIME and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
-    elif concept.datatype == DataType.TIMESTAMP:
+    elif concept.datatype == DataType.TIMESTAMP and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
+
+    if isinstance(concept.datatype, StructType):
+        for key, value in concept.datatype.fields_map.items():
+            args = process_function_args(
+                [concept, key], meta=meta, environment=environment
+            )
+            auto = Concept(
+                name=key,
+                datatype=arg_to_datatype(value),
+                purpose=Purpose.PROPERTY,
+                namespace=(
+                    environment.namespace + "." + concept.name
+                    if environment.namespace
+                    and environment.namespace != DEFAULT_NAMESPACE
+                    else concept.name
+                ),
+                lineage=AttrAccess(args),
+            )
+            environment.add_concept(auto, meta=meta)
+            if isinstance(value, Concept):
+                environment.merge_concept(auto, value, modifiers=[])
+                assert value.pseudonyms is not None

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/models.py

@@ -300,7 +300,7 @@ class MapType(BaseModel):
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str]
 
     @property
     def data_type(self):
@@ -2119,16 +2119,19 @@ class Datasource(Namespaced, BaseModel):
 
 
 class UnnestJoin(BaseModel):
-    concept: Concept
+    concepts: list[Concept]
+    parent: Function
     alias: str = "unnest"
     rendering_required: bool = True
 
     def __hash__(self):
-        return (self.alias + self.concept.address).__hash__()
+        return (
+            self.alias + "".join([str(s.address) for s in self.concepts])
+        ).__hash__()
 
 
 class InstantiatedUnnestJoin(BaseModel):
-    concept: Concept
+    concept_to_unnest: Concept
     alias: str = "unnest"
 
 
@@ -2261,6 +2264,7 @@ class QueryDatasource(BaseModel):
     @field_validator("joins")
     @classmethod
     def validate_joins(cls, v):
+        unique_pairs = set()
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
@@ -2268,6 +2272,16 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
+            pairing = "".join(
+                sorted(
+                    [join.left_datasource.identifier, join.right_datasource.identifier]
+                )
+            )
+            if pairing in unique_pairs:
+                raise SyntaxError(
+                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
+                )
+            unique_pairs.add(pairing)
         return v
 
     @field_validator("input_concepts")
@@ -2287,8 +2301,13 @@ class QueryDatasource(BaseModel):
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
                 continue
+            concept: Concept
             for concept in values[key]:
-                if concept.address not in v and CONFIG.validate_missing:
+                if (
+                    concept.address not in v
+                    and not any(x in v for x in concept.pseudonyms)
+                    and CONFIG.validate_missing
+                ):
                     raise SyntaxError(
                         f"Missing source map for {concept.address} on {key}, have {v}"
                     )
@@ -2377,6 +2396,11 @@ class QueryDatasource(BaseModel):
                 final_source_map[key] = other.source_map[key]
         for k, v in final_source_map.items():
             final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
+        self_hidden = self.hidden_concepts or []
+        other_hidden = other.hidden_concepts or []
+        hidden = [
+            x for x in self_hidden if x.address in [y.address for y in other_hidden]
+        ]
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2400,9 +2424,7 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
-            hidden_concepts=unique(
-                self.hidden_concepts + other.hidden_concepts, "address"
-            ),
+            hidden_concepts=hidden,
         )
 
         return qds
@@ -2533,7 +2555,7 @@ class CTE(BaseModel):
                     )
                 ]
                 for join in self.joins:
-                    if isinstance(join, UnnestJoin) and join.concept == concept:
+                    if isinstance(join, UnnestJoin) and concept in join.concepts:
                         join.rendering_required = False
 
                 self.parent_ctes = [
@@ -2548,6 +2570,7 @@ class CTE(BaseModel):
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}."
+        base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
         if self.joins:
@@ -2556,6 +2579,11 @@ class CTE(BaseModel):
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
+        base += f"\n-- Source Map: {self.source_map}."
+        base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
+        if self.hidden_concepts:
+            base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
+
         return base
 
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
@@ -2623,6 +2651,10 @@ class CTE(BaseModel):
                 f" {self.name} {other.name} conditions {self.condition} {other.condition}"
             )
             raise ValueError(error)
+        mutually_hidden = []
+        for concept in self.hidden_concepts:
+            if concept in other.hidden_concepts:
+                mutually_hidden.append(concept)
         self.partial_concepts = unique(
             self.partial_concepts + other.partial_concepts, "address"
         )
@@ -2645,9 +2677,7 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
-        self.hidden_concepts = unique(
-            self.hidden_concepts + other.hidden_concepts, "address"
-        )
+        self.hidden_concepts = mutually_hidden
         self.existence_source_map = {
             **self.existence_source_map,
             **other.existence_source_map,
@@ -2996,13 +3026,16 @@ class EnvironmentDatasourceDict(dict):
         except KeyError:
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key)
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1])
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1])
             raise
 
     def values(self) -> ValuesView[Datasource]:  # type: ignore
         return super().values()
 
+    def items(self) -> ItemsView[str, Datasource]:  # type: ignore
+        return super().items()
+
 
 class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:
@@ -3027,8 +3060,8 @@ class EnvironmentConceptDict(dict):
             return super(EnvironmentConceptDict, self).__getitem__(key)
 
         except KeyError:
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1], line_no)
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1], line_no)
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key, line_no)
             if not self.fail_on_missing:
@@ -3293,10 +3326,9 @@ class Environment(BaseModel):
             self.concepts[concept.name] = concept
         else:
             self.concepts[concept.address] = concept
-        if add_derived:
-            from trilogy.core.environment_helpers import generate_related_concepts
+        from trilogy.core.environment_helpers import generate_related_concepts
 
-            generate_related_concepts(concept, self)
+        generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
         self.gen_concept_list_caches()
         return concept
 
@@ -3423,14 +3455,6 @@ class Comparison(
             raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
-        if self.operator == ComparisonOperator.BETWEEN:
-            if (
-                not isinstance(self.right, ComparisonOperator)
-                and self.right.operator == BooleanOperator.AND
-            ):
-                raise SyntaxError(
-                    f"Between operator must have two operands with and, not {self.right}"
-                )
 
     def __add__(self, other):
         if other is None:

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/optimization.py

@@ -10,6 +10,7 @@ from trilogy.core.optimizations import (
     OptimizationRule,
     InlineConstant,
     PredicatePushdown,
+    PredicatePushdownRemove,
     InlineDatasource,
 )
 
@@ -34,6 +35,31 @@ MAX_OPTIMIZATION_LOOPS = 100
 #     return parent
 
 
+def reorder_ctes(
+    input: list[CTE],
+):
+    import networkx as nx
+
+    # Create a directed graph
+    G = nx.DiGraph()
+    mapping: dict[str, CTE] = {}
+    for cte in input:
+        mapping[cte.name] = cte
+        for parent in cte.parent_ctes:
+            G.add_edge(parent.name, cte.name)
+    # Perform topological sort (only works for DAGs)
+    try:
+        topological_order = list(nx.topological_sort(G))
+        if not topological_order:
+            return input
+        return [mapping[x] for x in topological_order]
+    except nx.NetworkXUnfeasible as e:
+        print(
+            "The graph is not a DAG (contains cycles) and cannot be topologically sorted."
+        )
+        raise e
+
+
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
@@ -169,20 +195,22 @@ def optimize_ctes(
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
-
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdownRemove())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False
         while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
             actions_taken = False
             # assume we go through all CTEs once
-            look_at = [root_cte, *input]
+            look_at = [root_cte, *reversed(input)]
             inverse_map = gen_inverse_map(look_at)
             for cte in look_at:
                 opt = rule.optimize(cte, inverse_map)
                 actions_taken = actions_taken or opt
             complete = not actions_taken
             loops += 1
+            input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
         logger.info(f"finished checking for {type(rule).__name__} in {loops} loops")
 
-    return filter_irrelevant_ctes(input, root_cte)
+    return reorder_ctes(filter_irrelevant_ctes(input, root_cte))

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/optimizations/__init__.py

@@ -1,6 +1,6 @@
 from .inline_constant import InlineConstant
 from .inline_datasource import InlineDatasource
-from .predicate_pushdown import PredicatePushdown
+from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 from .base_optimization import OptimizationRule
 
 __all__ = [
@@ -8,4 +8,5 @@ __all__ = [
     "InlineConstant",
     "InlineDatasource",
     "PredicatePushdown",
+    "PredicatePushdownRemove",
 ]

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/optimizations/predicate_pushdown.py

@@ -114,48 +114,6 @@ class PredicatePushdown(OptimizationRule):
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
-
-        parent_filter_status = {
-            parent.name: is_child_of(cte.condition, parent.condition)
-            for parent in cte.parent_ctes
-        }
-        # flatten existnce argument tuples to a list
-
-        flattened_existence = [
-            x.address for y in cte.condition.existence_arguments for x in y
-        ]
-
-        existence_only = [
-            parent.name
-            for parent in cte.parent_ctes
-            if all([x.address in flattened_existence for x in parent.output_columns])
-            and len(flattened_existence) > 0
-        ]
-        if all(
-            [
-                value
-                for key, value in parent_filter_status.items()
-                if key not in existence_only
-            ]
-        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
-            self.log(
-                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
-            )
-            cte.condition = None
-            # remove any "parent" CTEs that provided only existence inputs
-            if existence_only:
-                original = [y.name for y in cte.parent_ctes]
-                cte.parent_ctes = [
-                    x for x in cte.parent_ctes if x.name not in existence_only
-                ]
-                self.log(
-                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
-                )
-            return True
-        else:
-            self.log(
-                f"Could not remove filter from {cte.name}, as not all parents have the same filter: {parent_filter_status}"
-            )
         if self.complete.get(cte.name):
             self.debug("Have done this CTE before")
             return False
@@ -197,3 +155,63 @@ class PredicatePushdown(OptimizationRule):
 
         self.complete[cte.name] = True
         return optimized
+
+
+class PredicatePushdownRemove(OptimizationRule):
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        optimized = False
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+
+        parent_filter_status = {
+            parent.name: is_child_of(cte.condition, parent.condition)
+            for parent in cte.parent_ctes
+        }
+        # flatten existnce argument tuples to a list
+
+        flattened_existence = [
+            x.address for y in cte.condition.existence_arguments for x in y
+        ]
+
+        existence_only = [
+            parent.name
+            for parent in cte.parent_ctes
+            if all([x.address in flattened_existence for x in parent.output_columns])
+            and len(flattened_existence) > 0
+        ]
+        if all(
+            [
+                value
+                for key, value in parent_filter_status.items()
+                if key not in existence_only
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log(
+                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
+            )
+            cte.condition = None
+            # remove any "parent" CTEs that provided only existence inputs
+            if existence_only:
+                original = [y.name for y in cte.parent_ctes]
+                cte.parent_ctes = [
+                    x for x in cte.parent_ctes if x.name not in existence_only
+                ]
+                self.log(
+                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
+                )
+            return True
+
+        self.complete[cte.name] = True
+        return optimized

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/processing/concept_strategies_v3.py

@@ -180,7 +180,10 @@ def generate_candidates_restrictive(
     local_candidates = [
         x
         for x in list(candidates)
-        if x.address not in exhausted and x.granularity != Granularity.SINGLE_ROW
+        if x.address not in exhausted
+        and x.granularity != Granularity.SINGLE_ROW
+        and x.address not in priority_concept.pseudonyms
+        and priority_concept.address not in x.pseudonyms
     ]
     combos: list[list[Concept]] = []
     grain_check = Grain(components=[*local_candidates]).components_copy
@@ -608,7 +611,7 @@ def _search_concepts(
         if len(stack) == 1:
             output = stack[0]
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]} and {output.resolve().source_map}"
             )
             return output
 
@@ -658,8 +661,7 @@ def _search_concepts(
                 if x.address not in [y.address for y in mandatory_list]
                 and x not in ex_resolve.grain.components
             ]
-            expanded.output_concepts = mandatory_list
-            expanded.rebuild_cache()
+            expanded.set_output_concepts(mandatory_list)
 
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"

{pytrilogy-0.0.2.11 → pytrilogy-0.0.2.13}/trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,8 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from trilogy.core.enums import SourceType
+from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
@@ -31,12 +33,21 @@ def gen_basic_node(
     )
 
     local_optional_redundant = [x for x in local_optional if x in parent_concepts]
-    attempts = [(parent_concepts, [concept] + local_optional_redundant)]
-    from itertools import combinations
+    attempts: List[tuple[list[Concept], list[Concept]]] = [
+        (parent_concepts, [concept] + local_optional_redundant)
+    ]
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if x.lineage == concept.lineage and x.address != concept.address
+    ]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
 
     if local_optional:
-        for combo in range(1, len(local_optional) + 1):
-            combos = combinations(local_optional, combo)
+        for combo in range(1, len(non_equivalent_optional) + 1):
+            combos = combinations(non_equivalent_optional, combo)
             for optional_set in combos:
                 attempts.append(
                     (
@@ -55,8 +66,10 @@ def gen_basic_node(
             depth=depth + 1,
             history=history,
         )
+
         if not parent_node:
             continue
+        parent_node.source_type = SourceType.BASIC
         parents: List[StrategyNode] = [parent_node]
         for x in basic_output:
             sources = [p for p in parents if x in p.output_concepts]
@@ -64,13 +77,10 @@ def gen_basic_node(
                 continue
             if all(x in source.partial_concepts for source in sources):
                 partials.append(x)
-        outputs = parent_node.output_concepts + [concept]
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in outputs]}"
-        )
-        # parents.resolve()
 
         parent_node.add_output_concept(concept)
+        for x in equivalent_optional:
+            parent_node.add_output_concept(x)
 
         parent_node.remove_output_concepts(
             [
@@ -79,6 +89,9 @@ def gen_basic_node(
                 if x.address not in [y.address for y in basic_output]
             ]
         )
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in parent_node.output_concepts]}"
+        )
         return parent_node
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"