pytrilogy 0.0.1.103__tar.gz → 0.0.1.105__tar.gz

{pytrilogy-0.0.1.103/pytrilogy.egg-info → pytrilogy-0.0.1.105}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.103
+Version: 0.0.1.105
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.103
+Version: 0.0.1.105
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105}/trilogy/__init__.py

@@ -3,6 +3,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.1.103"
+__version__ = "0.0.1.105"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment"]

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105}/trilogy/core/models.py

@@ -113,15 +113,27 @@ NAMESPACED_TYPES = Union[
 
 
 class Namespaced(ABC):
-    pass
 
     def with_namespace(self, namespace: str):
         raise NotImplementedError
 
 
-class SelectGrain(ABC):
-    pass
+class ConceptArgs(ABC):
+
+    @property
+    def concept_arguments(self) -> List["Concept"]:
+        raise NotImplementedError
+
+    @property
+    def existence_arguments(self) -> List["Concept"]:
+        return []
+
+    @property
+    def row_arguments(self) -> List["Concept"]:
+        return self.concept_arguments
 
+
+class SelectGrain(ABC):
     def with_select_grain(self, grain: Grain):
         raise NotImplementedError
 
@@ -355,7 +367,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
         return f"{self.namespace}.{self.name}"
 
@@ -436,7 +448,8 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
-    def with_default_grain(self) -> "Concept":
+    @cached_property
+    def _with_default_grain(self) -> "Concept":
         if self.purpose == Purpose.KEY:
             # we need to make this abstract
             grain = Grain(components=[self.with_grain(Grain())], nested=True)
@@ -473,6 +486,9 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
+    def with_default_grain(self) -> "Concept":
+        return self._with_default_grain
+
     @property
     def sources(self) -> List["Concept"]:
         if self.lineage:
@@ -610,7 +626,7 @@ class Grain(BaseModel):
             [c.name == ALL_ROWS_CONCEPT for c in self.components]
         )
 
-    @property
+    @cached_property
     def set(self):
         return set([c.address for c in self.components_copy])
 
@@ -1391,16 +1407,11 @@ class MultiSelectStatement(Namespaced, BaseModel):
         return output
 
     def find_source(self, concept: Concept, cte: CTE) -> Concept:
-        all = []
         for x in self.align.items:
             if concept.name == x.alias:
                 for c in x.concepts:
                     if c.address in cte.output_lcl:
-                        all.append(c)
-
-        if len(all) == 1:
-            return all[0]
-
+                        return c
         raise SyntaxError(
             f"Could not find upstream map for multiselect {str(concept)} on cte ({cte})"
         )
@@ -1585,7 +1596,7 @@ class Datasource(Namespaced, BaseModel):
             columns=[c.with_namespace(namespace) for c in self.columns],
         )
 
-    @property
+    @cached_property
     def concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns]
 
@@ -1780,7 +1791,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info=ValidationInfo):
+    def validate_source_map(cls, v, info: ValidationInfo):
         values = info.data
         expected = {c.address for c in values["output_concepts"]}.union(
             c.address for c in values["input_concepts"]
@@ -2288,8 +2299,8 @@ class EnvironmentConceptDict(dict):
 
 class ImportStatement(BaseModel):
     alias: str
-    path: str
-    # environment: "Environment" | None = None
+    path: Path
+    environment: Union["Environment", None] = None
     # TODO: this might result in a lot of duplication
     # environment:"Environment"
 
@@ -2324,6 +2335,9 @@ class Environment(BaseModel):
     version: str = Field(default_factory=get_version)
     cte_name_map: Dict[str, str] = Field(default_factory=dict)
 
+    materialized_concepts: List[Concept] = Field(default_factory=list)
+    _parse_count: int = 0
+
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
         with open(path, "r") as f:
@@ -2349,20 +2363,14 @@ class Environment(BaseModel):
             f.write(self.model_dump_json())
         return ppath
 
-    @property
-    def materialized_concepts(self) -> List[Concept]:
-        output = []
-        for concept in self.concepts.values():
-            found = False
-            # basic concepts are effectively materialized
-            # and can be found via join paths
-            for datasource in self.datasources.values():
-                if concept.address in [x.address for x in datasource.output_concepts]:
-                    found = True
-                    break
-            if found:
-                output.append(concept)
-        return output
+    def gen_materialized_concepts(self) -> None:
+        concrete_addresses = set()
+        for datasource in self.datasources.values():
+            for concept in datasource.output_concepts:
+                concrete_addresses.add(concept.address)
+        self.materialized_concepts = [
+            c for c in self.concepts.values() if c.address in concrete_addresses
+        ]
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -2392,12 +2400,61 @@ class Environment(BaseModel):
 
     def add_import(self, alias: str, environment: Environment):
         self.imports[alias] = ImportStatement(
-            alias=alias, path=str(environment.working_path)
+            alias=alias, path=Path(environment.working_path)
         )
         for key, concept in environment.concepts.items():
             self.concepts[f"{alias}.{key}"] = concept.with_namespace(alias)
         for key, datasource in environment.datasources.items():
             self.datasources[f"{alias}.{key}"] = datasource.with_namespace(alias)
+        self.gen_materialized_concepts()
+        return self
+
+    def add_file_import(self, path: str, alias: str, env: Environment | None = None):
+        from trilogy.parsing.parse_engine import ParseToObjects, PARSER
+
+        apath = path.split(".")
+        apath[-1] = apath[-1] + ".preql"
+
+        target: Path = Path(self.working_path, *apath)
+        if env:
+            self.imports[alias] = ImportStatement(
+                alias=alias, path=target, environment=env
+            )
+
+        elif alias in self.imports:
+            current = self.imports[alias]
+            env = self.imports[alias].environment
+            if current.path != target:
+                raise ImportError(
+                    f"Attempted to import {target} with alias {alias} but {alias} is already imported from {current.path}"
+                )
+        else:
+            try:
+                with open(target, "r", encoding="utf-8") as f:
+                    text = f.read()
+                nparser = ParseToObjects(
+                    visit_tokens=True,
+                    text=text,
+                    environment=Environment(
+                        working_path=target.parent,
+                    ),
+                    parse_address=str(target),
+                )
+                nparser.transform(PARSER.parse(text))
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import file {target.parent}, parsing error: {e}"
+                )
+            env = nparser.environment
+        if env:
+            for _, concept in env.concepts.items():
+                self.add_concept(concept.with_namespace(alias))
+
+            for _, datasource in env.datasources.items():
+                self.add_datasource(datasource.with_namespace(alias))
+        imps = ImportStatement(alias=alias, path=target, environment=env)
+        self.imports[alias] = imps
+        return imps
 
     def parse(
         self, input: str, namespace: str | None = None, persist: bool = False
@@ -2448,21 +2505,22 @@ class Environment(BaseModel):
             from trilogy.core.environment_helpers import generate_related_concepts
 
             generate_related_concepts(concept, self)
+        self.gen_materialized_concepts()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
+        meta: Meta | None = None,
     ):
-        if datasource.namespace == DEFAULT_NAMESPACE:
-            self.datasources[datasource.name] = datasource
-            return datasource
-        if not datasource.namespace:
+        if not datasource.namespace or datasource.namespace == DEFAULT_NAMESPACE:
             self.datasources[datasource.name] = datasource
+            self.gen_materialized_concepts()
             return datasource
         self.datasources[datasource.namespace + "." + datasource.identifier] = (
             datasource
         )
+        self.gen_materialized_concepts()
         return datasource
 
 
@@ -2495,7 +2553,7 @@ class LazyEnvironment(Environment):
         return super().__getattribute__(name)
 
 
-class Comparison(Namespaced, SelectGrain, BaseModel):
+class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     left: Union[
         int,
         str,
@@ -2547,7 +2605,7 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
         return f"{str(self.left)} {self.operator.value} {str(self.right)}"
 
     def with_namespace(self, namespace: str):
-        return Comparison(
+        return self.__class__(
             left=(
                 self.left.with_namespace(namespace)
                 if isinstance(self.left, Namespaced)
@@ -2562,7 +2620,7 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
         )
 
     def with_select_grain(self, grain: Grain):
-        return Comparison(
+        return self.__class__(
             left=(
                 self.left.with_select_grain(grain)
                 if isinstance(self.left, SelectGrain)
@@ -2581,7 +2639,9 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
         output: List[Concept] = []
         if isinstance(self.left, (Concept,)):
             output += [self.left]
-        if isinstance(self.left, (Conditional, Parenthetical)):
+        if isinstance(
+            self.left, (Comparison, SubselectComparison, Conditional, Parenthetical)
+        ):
             output += self.left.input
         if isinstance(self.left, FilterItem):
             output += self.left.concept_arguments
@@ -2590,7 +2650,9 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
 
         if isinstance(self.right, (Concept,)):
             output += [self.right]
-        if isinstance(self.right, (Conditional, Parenthetical)):
+        if isinstance(
+            self.right, (Comparison, SubselectComparison, Conditional, Parenthetical)
+        ):
             output += self.right.input
         if isinstance(self.right, FilterItem):
             output += self.right.concept_arguments
@@ -2607,8 +2669,31 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
         return output
 
 
+class SubselectComparison(Comparison):
+
+    @property
+    def row_arguments(self) -> List[Concept]:
+        return get_concept_arguments(self.left)
+
+    @property
+    def existence_arguments(self) -> List[Concept]:
+        return get_concept_arguments(self.right)
+
+    def with_select_grain(self, grain: Grain):
+        # there's no need to pass the select grain through to a subselect comparison
+        return self.__class__(
+            left=(
+                self.left.with_select_grain(grain)
+                if isinstance(self.left, SelectGrain)
+                else self.left
+            ),
+            right=self.right,
+            operator=self.operator,
+        )
+
+
 class CaseWhen(Namespaced, SelectGrain, BaseModel):
-    comparison: Conditional | Comparison
+    comparison: Conditional | SubselectComparison | Comparison
     expr: "Expr"
 
     @property
@@ -2675,7 +2760,7 @@ class CaseElse(Namespaced, SelectGrain, BaseModel):
         )
 
 
-class Conditional(Namespaced, SelectGrain, BaseModel):
+class Conditional(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     left: Union[
         int,
         str,
@@ -2770,6 +2855,32 @@ class Conditional(Namespaced, SelectGrain, BaseModel):
         output += get_concept_arguments(self.right)
         return output
 
+    @property
+    def row_arguments(self) -> List[Concept]:
+        output = []
+        if isinstance(self.left, ConceptArgs):
+            output += self.left.row_arguments
+        else:
+            output += get_concept_arguments(self.left)
+        if isinstance(self.right, ConceptArgs):
+            output += self.right.row_arguments
+        else:
+            output += get_concept_arguments(self.right)
+        return output
+
+    @property
+    def existence_arguments(self) -> List[Concept]:
+        output = []
+        if isinstance(self.left, ConceptArgs):
+            output += self.left.existence_arguments
+        else:
+            output += get_concept_arguments(self.left)
+        if isinstance(self.right, ConceptArgs):
+            output += self.right.existence_arguments
+        else:
+            output += get_concept_arguments(self.right)
+        return output
+
 
 class AggregateWrapper(Namespaced, SelectGrain, BaseModel):
     function: Function
@@ -2813,8 +2924,8 @@ class AggregateWrapper(Namespaced, SelectGrain, BaseModel):
         return AggregateWrapper(function=self.function.with_select_grain(grain), by=by)
 
 
-class WhereClause(Namespaced, SelectGrain, BaseModel):
-    conditional: Union[Comparison, Conditional, "Parenthetical"]
+class WhereClause(ConceptArgs, Namespaced, SelectGrain, BaseModel):
+    conditional: Union[SubselectComparison, Comparison, Conditional, "Parenthetical"]
 
     @property
     def input(self) -> List[Concept]:
@@ -2824,6 +2935,14 @@ class WhereClause(Namespaced, SelectGrain, BaseModel):
     def concept_arguments(self) -> List[Concept]:
         return self.conditional.concept_arguments
 
+    @property
+    def row_arguments(self) -> List[Concept]:
+        return self.conditional.row_arguments
+
+    @property
+    def existence_arguments(self) -> List[Concept]:
+        return self.conditional.existence_arguments
+
     def with_namespace(self, namespace: str) -> WhereClause:
         return WhereClause(conditional=self.conditional.with_namespace(namespace))
 
@@ -3011,7 +3130,7 @@ class RowsetItem(Namespaced, BaseModel):
         return [self.content]
 
 
-class Parenthetical(Namespaced, SelectGrain, BaseModel):
+class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     content: "Expr"
 
     def __str__(self):
@@ -3055,6 +3174,18 @@ class Parenthetical(Namespaced, SelectGrain, BaseModel):
             base.append(x)
         return base
 
+    @property
+    def row_arguments(self) -> List[Concept]:
+        if isinstance(self.content, ConceptArgs):
+            return self.content.row_arguments
+        return self.concept_arguments
+
+    @property
+    def existence_arguments(self) -> List[Concept]:
+        if isinstance(self.content, ConceptArgs):
+            return self.content.existence_arguments
+        return self.concept_arguments
+
     @property
     def input(self):
         base = []

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105}/trilogy/core/processing/concept_strategies_v3.py

@@ -23,7 +23,6 @@ from trilogy.core.processing.node_generators import (
     gen_window_node,
     gen_group_node,
     gen_basic_node,
-    gen_select_node,
     gen_unnest_node,
     gen_merge_node,
     gen_group_to_node,
@@ -208,7 +207,8 @@ def generate_node(
     history: History | None = None,
 ) -> StrategyNode | None:
     # first check in case there is a materialized_concept
-    candidate = gen_select_node(
+    history = history or History()
+    candidate = history.gen_select_node(
         concept,
         local_optional,
         environment,
@@ -218,6 +218,7 @@ def generate_node(
         accept_partial=accept_partial,
         accept_partial_optional=False,
     )
+
     if candidate:
         return candidate
 
@@ -316,11 +317,12 @@ def generate_node(
         return gen_basic_node(
             concept, local_optional, environment, g, depth + 1, source_concepts, history
         )
+
     elif concept.derivation == PurposeLineage.ROOT:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional {[x.address for x in local_optional]}"
         )
-        return gen_select_node(
+        return history.gen_select_node(
             concept,
             local_optional,
             environment,
@@ -328,6 +330,7 @@ def generate_node(
             depth + 1,
             fail_if_not_found=False,
             accept_partial=accept_partial,
+            accept_partial_optional=True,
         )
     else:
         raise ValueError(f"Unknown derivation {concept.derivation}")

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105}/trilogy/core/processing/node_generators/common.py

@@ -45,21 +45,33 @@ def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
     return unique(concept.lineage.concept_arguments, "address")
 
 
-def resolve_filter_parent_concepts(concept: Concept) -> Tuple[Concept, List[Concept]]:
+def resolve_filter_parent_concepts(
+    concept: Concept,
+) -> Tuple[Concept, List[Concept], List[Concept]]:
     if not isinstance(concept.lineage, FilterItem):
-        raise ValueError
+        raise ValueError(
+            f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
+        )
     direct_parent = concept.lineage.content
-    base = [direct_parent]
-    base += concept.lineage.where.concept_arguments
+    base_existence = []
+    base_rows = [direct_parent]
+    base_rows += concept.lineage.where.row_arguments
+    base_existence += concept.lineage.where.existence_arguments
     if direct_parent.grain:
-        base += direct_parent.grain.components_copy
+        base_rows += direct_parent.grain.components_copy
     if (
         isinstance(direct_parent, Concept)
         and direct_parent.purpose == Purpose.PROPERTY
         and direct_parent.keys
     ):
-        base += direct_parent.keys
-    return concept.lineage.content, unique(base, "address")
+        base_rows += direct_parent.keys
+    if concept.lineage.where.existence_arguments:
+        return (
+            concept.lineage.content,
+            unique(base_rows, "address"),
+            unique(base_existence, "address"),
+        )
+    return concept.lineage.content, unique(base_rows, "address"), []
 
 
 def gen_property_enrichment_node(

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105}/trilogy/core/processing/node_generators/filter_node.py

@@ -11,7 +11,7 @@ from trilogy.core.processing.node_generators.common import (
     resolve_filter_parent_concepts,
 )
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding
+from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
@@ -26,35 +26,64 @@ def gen_filter_node(
     source_concepts,
     history: History | None = None,
 ) -> MergeNode | FilterNode | None:
-    immediate_parent, parent_concepts = resolve_filter_parent_concepts(concept)
+    immediate_parent, parent_row_concepts, parent_existence_concepts = (
+        resolve_filter_parent_concepts(concept)
+    )
 
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} fetching filter node parents")
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} fetching filter node row parents {[x.address for x in parent_row_concepts]}"
+    )
+    core_parents = []
     parent = source_concepts(
-        mandatory_list=parent_concepts,
+        mandatory_list=parent_row_concepts,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
     )
+
     if not parent:
         return None
+    core_parents.append(parent)
+    if parent_existence_concepts:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in parent_existence_concepts]}"
+        )
+        parent_existence = source_concepts(
+            mandatory_list=parent_existence_concepts,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not parent_existence:
+            return None
+        core_parents.append(parent_existence)
+
     filter_node = FilterNode(
-        input_concepts=[immediate_parent] + parent_concepts,
-        output_concepts=[concept, immediate_parent] + parent_concepts,
+        input_concepts=unique(
+            [immediate_parent] + parent_row_concepts + parent_existence_concepts,
+            "address",
+        ),
+        output_concepts=[concept, immediate_parent] + parent_row_concepts,
         environment=environment,
         g=g,
-        parents=[parent],
+        parents=core_parents,
     )
-    if not local_optional:
+    if not local_optional or all(
+        [x.address in [y.address for y in parent_row_concepts] for x in local_optional]
+    ):
         return filter_node
     enrich_node = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
-        mandatory_list=[immediate_parent] + parent_concepts + local_optional,
+        mandatory_list=[immediate_parent] + parent_row_concepts + local_optional,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
     )
+    if not enrich_node:
+        return filter_node
     x = MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[
@@ -73,7 +102,7 @@ def gen_filter_node(
                 left_node=enrich_node,
                 right_node=filter_node,
                 concepts=concept_to_relevant_joins(
-                    [immediate_parent] + parent_concepts
+                    [immediate_parent] + parent_row_concepts
                 ),
                 join_type=JoinType.LEFT_OUTER,
                 filter_to_mutual=False,

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.105}/trilogy/core/processing/node_generators/merge_node.py

@@ -87,8 +87,18 @@ def gen_merge_node(
 ) -> Optional[MergeNode]:
     join_candidates: List[PathInfo] = []
     # anchor on datasources
+    final_all_concepts = []
+    # implicit_upstream = {}
+    for x in all_concepts:
+        # if x.derivation in (PurposeLineage.AGGREGATE, PurposeLineage.BASIC):
+        #     final_all_concepts +=resolve_function_parent_concepts(x)
+        # elif x.derivation == PurposeLineage.FILTER:
+        #     final_all_concepts +=resolve_filter_parent_concepts(x)
+        # else:
+        #     final_all_concepts.append(x)
+        final_all_concepts.append(x)
     for datasource in environment.datasources.values():
-        path = identify_ds_join_paths(all_concepts, g, datasource, accept_partial)
+        path = identify_ds_join_paths(final_all_concepts, g, datasource, accept_partial)
         if path and path.reduced_concepts:
             join_candidates.append(path)
     join_candidates.sort(key=lambda x: sum([len(v) for v in x.paths.values()]))