pytrilogy 0.0.3.54__tar.gz → 0.0.3.56__tar.gz

{pytrilogy-0.0.3.54/pytrilogy.egg-info → pytrilogy-0.0.3.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.54
+Version: 0.0.3.56
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.54
+Version: 0.0.3.56
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/pytrilogy.egg-info/SOURCES.txt

@@ -71,6 +71,10 @@ trilogy/core/optimizations/inline_datasource.py
 trilogy/core/optimizations/predicate_pushdown.py
 trilogy/core/processing/__init__.py
 trilogy/core/processing/concept_strategies_v3.py
+trilogy/core/processing/discovery_loop.py
+trilogy/core/processing/discovery_node_factory.py
+trilogy/core/processing/discovery_utility.py
+trilogy/core/processing/discovery_validation.py
 trilogy/core/processing/graph_utils.py
 trilogy/core/processing/utility.py
 trilogy/core/processing/node_generators/__init__.py
@@ -81,6 +85,7 @@ trilogy/core/processing/node_generators/group_node.py
 trilogy/core/processing/node_generators/group_to_node.py
 trilogy/core/processing/node_generators/multiselect_node.py
 trilogy/core/processing/node_generators/node_merge_node.py
+trilogy/core/processing/node_generators/recursive_node.py
 trilogy/core/processing/node_generators/rowset_node.py
 trilogy/core/processing/node_generators/select_merge_node.py
 trilogy/core/processing/node_generators/select_node.py
@@ -95,6 +100,7 @@ trilogy/core/processing/nodes/base_node.py
 trilogy/core/processing/nodes/filter_node.py
 trilogy/core/processing/nodes/group_node.py
 trilogy/core/processing/nodes/merge_node.py
+trilogy/core/processing/nodes/recursive_node.py
 trilogy/core/processing/nodes/select_node_v2.py
 trilogy/core/processing/nodes/union_node.py
 trilogy/core/processing/nodes/unnest_node.py
@@ -136,4 +142,5 @@ trilogy/std/date.preql
 trilogy/std/display.preql
 trilogy/std/geography.preql
 trilogy/std/money.preql
+trilogy/std/net.preql
 trilogy/std/report.preql

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.54"
+__version__ = "0.0.3.56"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/constants.py

@@ -7,6 +7,8 @@ logger = getLogger("trilogy")
 
 DEFAULT_NAMESPACE = "local"
 
+RECURSIVE_GATING_CONCEPT = "_terminal"
+
 VIRTUAL_CONCEPT_PREFIX = "_virt"
 
 ENV_CACHE_NAME = ".preql_cache.json"

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/core/enums.py

@@ -42,6 +42,7 @@ class Purpose(Enum):
 
 class Derivation(Enum):
     BASIC = "basic"
+    GROUP_TO = "group_to"
     WINDOW = "window"
     AGGREGATE = "aggregate"
     FILTER = "filter"
@@ -51,6 +52,7 @@ class Derivation(Enum):
     ROOT = "root"
     ROWSET = "rowset"
     MULTISELECT = "multiselect"
+    RECURSIVE = "recursive"
 
 
 class Granularity(Enum):
@@ -117,6 +119,7 @@ class FunctionType(Enum):
 
     # structural
     UNNEST = "unnest"
+    RECURSE_EDGE = "recurse_edge"
 
     UNION = "union"
 
@@ -233,6 +236,8 @@ class FunctionClass(Enum):
 
     ONE_TO_MANY = [FunctionType.UNNEST]
 
+    RECURSIVE = [FunctionType.RECURSE_EDGE]
+
 
 class Boolean(Enum):
     TRUE = "true"
@@ -333,6 +338,7 @@ class SourceType(Enum):
     MERGE = "merge"
     BASIC = "basic"
     UNION = "union"
+    RECURSIVE = "recursive"
 
 
 class ShowCategory(Enum):

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/core/functions.py

@@ -190,6 +190,9 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
         output_type_function=get_unnest_output_type,
         arg_count=1,
     ),
+    FunctionType.RECURSE_EDGE: FunctionConfig(
+        arg_count=2,
+    ),
     FunctionType.GROUP: FunctionConfig(
         arg_count=-1,
         output_type_function=lambda args: get_output_type_at_index(args, 0),

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/core/models/author.py

@@ -1164,12 +1164,24 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
             and lineage.operator == FunctionType.UNNEST
         ):
             return Derivation.UNNEST
+        elif (
+            lineage
+            and isinstance(lineage, (BuildFunction, Function))
+            and lineage.operator == FunctionType.RECURSE_EDGE
+        ):
+            return Derivation.RECURSIVE
         elif (
             lineage
             and isinstance(lineage, (BuildFunction, Function))
             and lineage.operator == FunctionType.UNION
         ):
             return Derivation.UNION
+        elif (
+            lineage
+            and isinstance(lineage, (BuildFunction, Function))
+            and lineage.operator == FunctionType.GROUP
+        ):
+            return Derivation.GROUP_TO
         elif (
             lineage
             and isinstance(lineage, (BuildFunction, Function))
@@ -1189,10 +1201,6 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
             return Derivation.CONSTANT
         return Derivation.ROOT
 
-    # @property
-    # def derivation(self) -> Derivation:
-    #     return self.calculate_derivation(self.lineage, self.purpose)
-
     @classmethod
     def calculate_granularity(cls, derivation: Derivation, grain: Grain, lineage):
         from trilogy.core.models.build import BuildFunction

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/core/models/execute.py

@@ -12,9 +12,16 @@ from pydantic import (
     model_validator,
 )
 
-from trilogy.constants import CONFIG, logger
+from trilogy.constants import (
+    CONFIG,
+    DEFAULT_NAMESPACE,
+    RECURSIVE_GATING_CONCEPT,
+    MagicConstants,
+    logger,
+)
 from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.enums import (
+    ComparisonOperator,
     Derivation,
     FunctionType,
     Granularity,
@@ -24,16 +31,20 @@ from trilogy.core.enums import (
     SourceType,
 )
 from trilogy.core.models.build import (
+    BuildCaseElse,
+    BuildCaseWhen,
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
+    BuildExpr,
     BuildFunction,
     BuildGrain,
     BuildOrderBy,
     BuildParamaterizedConceptReference,
     BuildParenthetical,
     BuildRowsetItem,
+    DataType,
     LooseBuildConceptList,
 )
 from trilogy.core.models.datasource import Address
@@ -841,6 +852,195 @@ class QueryDatasource(BaseModel):
         return self.identifier
 
 
+class AliasedExpression(BaseModel):
+    expr: BuildExpr
+    alias: str
+
+
+class RecursiveCTE(CTE):
+
+    def generate_loop_functions(
+        self,
+        recursive_derived: BuildConcept,
+        left_recurse_concept: BuildConcept,
+        right_recurse_concept: BuildConcept,
+    ) -> tuple[BuildConcept, BuildConcept, BuildConcept]:
+
+        join_gate = BuildConcept(
+            name=RECURSIVE_GATING_CONCEPT,
+            namespace=DEFAULT_NAMESPACE,
+            grain=recursive_derived.grain,
+            build_is_aggregate=False,
+            datatype=DataType.BOOL,
+            purpose=Purpose.KEY,
+            derivation=Derivation.BASIC,
+            lineage=BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=right_recurse_concept,
+                            right=MagicConstants.NULL,
+                            operator=ComparisonOperator.IS,
+                        ),
+                        expr=True,
+                    ),
+                    BuildCaseElse(expr=False),
+                ],
+                output_datatype=DataType.BOOL,
+                output_purpose=Purpose.KEY,
+            ),
+        )
+        bottom_join_gate = BuildConcept(
+            name=f"{RECURSIVE_GATING_CONCEPT}_two",
+            namespace=DEFAULT_NAMESPACE,
+            grain=recursive_derived.grain,
+            build_is_aggregate=False,
+            datatype=DataType.BOOL,
+            purpose=Purpose.KEY,
+            derivation=Derivation.BASIC,
+            lineage=BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=right_recurse_concept,
+                            right=MagicConstants.NULL,
+                            operator=ComparisonOperator.IS,
+                        ),
+                        expr=True,
+                    ),
+                    BuildCaseElse(expr=False),
+                ],
+                output_datatype=DataType.BOOL,
+                output_purpose=Purpose.KEY,
+            ),
+        )
+        bottom_derivation = BuildConcept(
+            name=recursive_derived.name + "_bottom",
+            namespace=recursive_derived.namespace,
+            grain=recursive_derived.grain,
+            build_is_aggregate=False,
+            datatype=recursive_derived.datatype,
+            purpose=recursive_derived.purpose,
+            derivation=Derivation.RECURSIVE,
+            lineage=BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=right_recurse_concept,
+                            right=MagicConstants.NULL,
+                            operator=ComparisonOperator.IS,
+                        ),
+                        expr=recursive_derived,
+                    ),
+                    BuildCaseElse(expr=right_recurse_concept),
+                ],
+                output_datatype=recursive_derived.datatype,
+                output_purpose=recursive_derived.purpose,
+            ),
+        )
+        return bottom_derivation, join_gate, bottom_join_gate
+
+    @property
+    def internal_ctes(self) -> List[CTE]:
+        filtered_output = [
+            x for x in self.output_columns if x.name != RECURSIVE_GATING_CONCEPT
+        ]
+        recursive_derived = [
+            x for x in self.output_columns if x.derivation == Derivation.RECURSIVE
+        ][0]
+        if not isinstance(recursive_derived.lineage, BuildFunction):
+            raise SyntaxError(
+                "Recursive CTEs must have a function lineage, found"
+                f" {recursive_derived.lineage}"
+            )
+        left_recurse_concept = recursive_derived.lineage.concept_arguments[0]
+        right_recurse_concept = recursive_derived.lineage.concept_arguments[1]
+        parent_ctes: List[CTE | UnionCTE]
+        if self.parent_ctes:
+            base = self.parent_ctes[0]
+            loop_input_cte = base
+            parent_ctes = [base]
+            parent_identifier = base.identifier
+        else:
+            raise SyntaxError("Recursive CTEs must have a parent CTE currently")
+        bottom_derivation, join_gate, bottom_join_gate = self.generate_loop_functions(
+            recursive_derived, left_recurse_concept, right_recurse_concept
+        )
+        base_output = [*filtered_output, join_gate]
+        bottom_output = []
+        for x in filtered_output:
+            if x.address == recursive_derived.address:
+                bottom_output.append(bottom_derivation)
+            else:
+                bottom_output.append(x)
+
+        bottom_output = [*bottom_output, bottom_join_gate]
+        top = CTE(
+            name=self.name,
+            source=self.source,
+            output_columns=base_output,
+            source_map=self.source_map,
+            grain=self.grain,
+            existence_source_map=self.existence_source_map,
+            parent_ctes=self.parent_ctes,
+            joins=self.joins,
+            condition=self.condition,
+            partial_concepts=self.partial_concepts,
+            hidden_concepts=self.hidden_concepts,
+            nullable_concepts=self.nullable_concepts,
+            join_derived_concepts=self.join_derived_concepts,
+            group_to_grain=self.group_to_grain,
+            order_by=self.order_by,
+            limit=self.limit,
+        )
+        top_cte_array: list[CTE | UnionCTE] = [top]
+        bottom_source_map = {
+            left_recurse_concept.address: [top.identifier],
+            right_recurse_concept.address: [parent_identifier],
+            # recursive_derived.address: self.source_map[recursive_derived.address],
+            join_gate.address: [top.identifier],
+            recursive_derived.address: [top.identifier],
+        }
+        bottom = CTE(
+            name=self.name,
+            source=self.source,
+            output_columns=bottom_output,
+            source_map=bottom_source_map,
+            grain=self.grain,
+            existence_source_map=self.existence_source_map,
+            parent_ctes=top_cte_array + parent_ctes,
+            joins=[
+                Join(
+                    right_cte=loop_input_cte,
+                    jointype=JoinType.INNER,
+                    joinkey_pairs=[
+                        CTEConceptPair(
+                            left=recursive_derived,
+                            right=left_recurse_concept,
+                            existing_datasource=loop_input_cte.source,
+                            modifiers=[],
+                            cte=top,
+                        )
+                    ],
+                    condition=BuildComparison(
+                        left=join_gate, right=True, operator=ComparisonOperator.IS_NOT
+                    ),
+                )
+            ],
+            partial_concepts=self.partial_concepts,
+            hidden_concepts=self.hidden_concepts,
+            nullable_concepts=self.nullable_concepts,
+            join_derived_concepts=self.join_derived_concepts,
+            group_to_grain=self.group_to_grain,
+            order_by=self.order_by,
+            limit=self.limit,
+        )
+        return [top, bottom]
+
+
 class UnionCTE(BaseModel):
     name: str
     source: QueryDatasource
@@ -891,6 +1091,10 @@ class UnionCTE(BaseModel):
     def condition(self, value):
         raise NotImplementedError
 
+    @property
+    def identifier(self) -> str:
+        return self.name
+
     @property
     def safe_identifier(self):
         return self.name
@@ -906,12 +1110,13 @@ class UnionCTE(BaseModel):
 
 
 class Join(BaseModel):
-    right_cte: CTE
+    right_cte: CTE | UnionCTE
     jointype: JoinType
     left_cte: CTE | None = None
     joinkey_pairs: List[CTEConceptPair] | None = None
     inlined_ctes: set[str] = Field(default_factory=set)
     quote: str | None = None
+    condition: BuildConditional | BuildComparison | BuildParenthetical | None = None
 
     def inline_cte(self, cte: CTE):
         self.inlined_ctes.add(cte.name)

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/core/optimization.py

@@ -3,7 +3,7 @@ from trilogy.core.enums import BooleanOperator, Derivation
 from trilogy.core.models.build import (
     BuildConditional,
 )
-from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
 from trilogy.core.optimizations import (
     InlineDatasource,
     OptimizationRule,
@@ -123,7 +123,7 @@ def gen_inverse_map(input: list[CTE | UnionCTE]) -> dict[str, list[CTE | UnionCT
 SENSITIVE_DERIVATIONS = [
     Derivation.UNNEST,
     Derivation.WINDOW,
-    # Derivation.AGGREGATE,
+    Derivation.RECURSIVE,
 ]
 
 
@@ -133,7 +133,7 @@ def is_direct_return_eligible(cte: CTE | UnionCTE) -> CTE | UnionCTE | None:
     if len(cte.parent_ctes) != 1:
         return None
     direct_parent = cte.parent_ctes[0]
-    if isinstance(direct_parent, UnionCTE):
+    if isinstance(direct_parent, (UnionCTE, RecursiveCTE)):
         return None
 
     output_addresses = set([x.address for x in cte.output_columns])

{pytrilogy-0.0.3.54 → pytrilogy-0.0.3.56}/trilogy/core/optimizations/inline_datasource.py

@@ -1,13 +1,8 @@
 from collections import defaultdict
 
 from trilogy.constants import CONFIG
-
-# from trilogy.core.models.datasource import Datasource
 from trilogy.core.models.build import BuildDatasource
-from trilogy.core.models.execute import (
-    CTE,
-    UnionCTE,
-)
+from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 
 
@@ -24,7 +19,8 @@ class InlineDatasource(OptimizationRule):
             return any(
                 self.optimize(x, inverse_map=inverse_map) for x in cte.internal_ctes
             )
-
+        if isinstance(cte, RecursiveCTE):
+            return False
         if not cte.parent_ctes:
             return False
 
@@ -36,6 +32,8 @@ class InlineDatasource(OptimizationRule):
         for parent_cte in cte.parent_ctes:
             if isinstance(parent_cte, UnionCTE):
                 continue
+            if isinstance(parent_cte, RecursiveCTE):
+                continue
             if not parent_cte.is_root_datasource:
                 self.debug(f"Cannot inline: parent {parent_cte.name} is not root")
                 continue