pytrilogy 0.0.1.102__tar.gz → 0.0.1.104__tar.gz

{pytrilogy-0.0.1.102/pytrilogy.egg-info → pytrilogy-0.0.1.104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.102
+Version: 0.0.1.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -27,7 +27,7 @@ Requires-Dist: sqlalchemy-bigquery; extra == "bigquery"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
 
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/README.md

@@ -1,4 +1,4 @@
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.102
+Version: 0.0.1.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -27,7 +27,7 @@ Requires-Dist: sqlalchemy-bigquery; extra == "bigquery"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
 
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/tests/test_models.py

@@ -1,4 +1,4 @@
-from trilogy.core.enums import BooleanOperator, Purpose, JoinType
+from trilogy.core.enums import BooleanOperator, Purpose, JoinType, ComparisonOperator
 from trilogy.core.models import (
     CTE,
     Grain,
@@ -9,6 +9,7 @@ from trilogy.core.models import (
     Address,
     UndefinedConcept,
     BaseJoin,
+    Comparison,
 )
 
 
@@ -73,11 +74,17 @@ def test_conditional(test_environment, test_environment_graph):
     condition_b = Conditional(
         left=test_concept, right=test_concept, operator=BooleanOperator.AND
     )
-
     merged = condition_a + condition_b
-    assert merged.left == condition_a
-    assert merged.right == condition_b
-    assert merged.operator == BooleanOperator.AND
+    assert merged == condition_a
+
+    test_concept_two = list(test_environment.concepts.values())[-2]
+    condition_c = Conditional(
+        left=test_concept, right=test_concept_two, operator=BooleanOperator.AND
+    )
+    merged_two = condition_a + condition_c
+    assert merged_two.left == condition_a
+    assert merged_two.right == condition_c
+    assert merged_two.operator == BooleanOperator.AND
 
 
 def test_grain(test_environment):
@@ -177,3 +184,10 @@ def test_base_join(test_environment: Environment):
         exc3 = exc4
         pass
     assert isinstance(exc3, SyntaxError)
+
+
+def test_comparison():
+    try:
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    except Exception as exc:
+        assert isinstance(exc, SyntaxError)

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/trilogy/__init__.py

@@ -3,6 +3,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.1.102"
+__version__ = "0.0.1.104"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment"]

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/trilogy/core/env_processor.py

@@ -1,4 +1,8 @@
-from trilogy.core.graph_models import ReferenceGraph, concept_to_node, datasource_to_node
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    datasource_to_node,
+)
 from trilogy.core.models import Environment
 from trilogy.core.enums import PurposeLineage
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/trilogy/core/models.py

@@ -338,7 +338,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
 
     def __eq__(self, other: object):
         if isinstance(other, str):
-            if self.address == str:
+            if self.address == other:
                 return True
         if not isinstance(other, Concept):
             return False
@@ -355,7 +355,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
         return f"{self.namespace}.{self.name}"
 
@@ -436,7 +436,8 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
-    def with_default_grain(self) -> "Concept":
+    @cached_property
+    def _with_default_grain(self) -> "Concept":
         if self.purpose == Purpose.KEY:
             # we need to make this abstract
             grain = Grain(components=[self.with_grain(Grain())], nested=True)
@@ -473,6 +474,9 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
+    def with_default_grain(self) -> "Concept":
+        return self._with_default_grain
+
     @property
     def sources(self) -> List["Concept"]:
         if self.lineage:
@@ -610,7 +614,7 @@ class Grain(BaseModel):
             [c.name == ALL_ROWS_CONCEPT for c in self.components]
         )
 
-    @property
+    @cached_property
     def set(self):
         return set([c.address for c in self.components_copy])
 
@@ -1585,7 +1589,7 @@ class Datasource(Namespaced, BaseModel):
             columns=[c.with_namespace(namespace) for c in self.columns],
         )
 
-    @property
+    @cached_property
     def concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns]
 
@@ -1780,7 +1784,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info=ValidationInfo):
+    def validate_source_map(cls, v, info: ValidationInfo):
         values = info.data
         expected = {c.address for c in values["output_concepts"]}.union(
             c.address for c in values["input_concepts"]
@@ -1887,7 +1891,9 @@ class QueryDatasource(BaseModel):
                 else None
             ),
             source_type=self.source_type,
-            partial_concepts=self.partial_concepts + other.partial_concepts,
+            partial_concepts=unique(
+                self.partial_concepts + other.partial_concepts, "address"
+            ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
         )
@@ -2286,8 +2292,8 @@ class EnvironmentConceptDict(dict):
 
 class ImportStatement(BaseModel):
     alias: str
-    path: str
-    # environment: "Environment" | None = None
+    path: Path
+    environment: Union["Environment", None] = None
     # TODO: this might result in a lot of duplication
     # environment:"Environment"
 
@@ -2322,6 +2328,9 @@ class Environment(BaseModel):
     version: str = Field(default_factory=get_version)
     cte_name_map: Dict[str, str] = Field(default_factory=dict)
 
+    materialized_concepts: List[Concept] = Field(default_factory=list)
+    _parse_count: int = 0
+
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
         with open(path, "r") as f:
@@ -2347,20 +2356,14 @@ class Environment(BaseModel):
             f.write(self.model_dump_json())
         return ppath
 
-    @property
-    def materialized_concepts(self) -> List[Concept]:
-        output = []
-        for concept in self.concepts.values():
-            found = False
-            # basic concepts are effectively materialized
-            # and can be found via join paths
-            for datasource in self.datasources.values():
-                if concept.address in [x.address for x in datasource.output_concepts]:
-                    found = True
-                    break
-            if found:
-                output.append(concept)
-        return output
+    def gen_materialized_concepts(self) -> None:
+        concrete_addresses = set()
+        for datasource in self.datasources.values():
+            for concept in datasource.output_concepts:
+                concrete_addresses.add(concept.address)
+        self.materialized_concepts = [
+            c for c in self.concepts.values() if c.address in concrete_addresses
+        ]
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -2390,12 +2393,61 @@ class Environment(BaseModel):
 
     def add_import(self, alias: str, environment: Environment):
         self.imports[alias] = ImportStatement(
-            alias=alias, path=str(environment.working_path)
+            alias=alias, path=Path(environment.working_path)
         )
         for key, concept in environment.concepts.items():
             self.concepts[f"{alias}.{key}"] = concept.with_namespace(alias)
         for key, datasource in environment.datasources.items():
             self.datasources[f"{alias}.{key}"] = datasource.with_namespace(alias)
+        self.gen_materialized_concepts()
+        return self
+
+    def add_file_import(self, path: str, alias: str, env: Environment | None = None):
+        from trilogy.parsing.parse_engine import ParseToObjects, PARSER
+
+        apath = path.split(".")
+        apath[-1] = apath[-1] + ".preql"
+
+        target: Path = Path(self.working_path, *apath)
+        if env:
+            self.imports[alias] = ImportStatement(
+                alias=alias, path=target, environment=env
+            )
+
+        elif alias in self.imports:
+            current = self.imports[alias]
+            env = self.imports[alias].environment
+            if current.path != target:
+                raise ImportError(
+                    f"Attempted to import {target} with alias {alias} but {alias} is already imported from {current.path}"
+                )
+        else:
+            try:
+                with open(target, "r", encoding="utf-8") as f:
+                    text = f.read()
+                nparser = ParseToObjects(
+                    visit_tokens=True,
+                    text=text,
+                    environment=Environment(
+                        working_path=target.parent,
+                    ),
+                    parse_address=str(target),
+                )
+                nparser.transform(PARSER.parse(text))
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import file {target.parent}, parsing error: {e}"
+                )
+            env = nparser.environment
+        if env:
+            for _, concept in env.concepts.items():
+                self.add_concept(concept.with_namespace(alias))
+
+            for _, datasource in env.datasources.items():
+                self.add_datasource(datasource.with_namespace(alias))
+        imps = ImportStatement(alias=alias, path=target, environment=env)
+        self.imports[alias] = imps
+        return imps
 
     def parse(
         self, input: str, namespace: str | None = None, persist: bool = False
@@ -2446,21 +2498,22 @@ class Environment(BaseModel):
             from trilogy.core.environment_helpers import generate_related_concepts
 
             generate_related_concepts(concept, self)
+        self.gen_materialized_concepts()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
+        meta: Meta | None = None,
     ):
-        if datasource.namespace == DEFAULT_NAMESPACE:
-            self.datasources[datasource.name] = datasource
-            return datasource
-        if not datasource.namespace:
+        if not datasource.namespace or datasource.namespace == DEFAULT_NAMESPACE:
             self.datasources[datasource.name] = datasource
+            self.gen_materialized_concepts()
             return datasource
         self.datasources[datasource.namespace + "." + datasource.identifier] = (
             datasource
         )
+        self.gen_materialized_concepts()
         return datasource
 
 
@@ -2530,7 +2583,7 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
 
     def __post_init__(self):
         if arg_to_datatype(self.left) != arg_to_datatype(self.right):
-            raise ValueError(
+            raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
 
@@ -2704,6 +2757,8 @@ class Conditional(Namespaced, SelectGrain, BaseModel):
     def __add__(self, other) -> "Conditional":
         if other is None:
             return self
+        elif str(other) == str(self):
+            return self
         elif isinstance(other, (Comparison, Conditional, Parenthetical)):
             return Conditional(left=self, right=other, operator=BooleanOperator.AND)
         raise ValueError(f"Cannot add {self.__class__} and {type(other)}")

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/trilogy/core/processing/concept_strategies_v3.py

@@ -23,7 +23,6 @@ from trilogy.core.processing.node_generators import (
     gen_window_node,
     gen_group_node,
     gen_basic_node,
-    gen_select_node,
     gen_unnest_node,
     gen_merge_node,
     gen_group_to_node,
@@ -208,7 +207,8 @@ def generate_node(
     history: History | None = None,
 ) -> StrategyNode | None:
     # first check in case there is a materialized_concept
-    candidate = gen_select_node(
+    history = history or History()
+    candidate = history.gen_select_node(
         concept,
         local_optional,
         environment,
@@ -218,6 +218,7 @@ def generate_node(
         accept_partial=accept_partial,
         accept_partial_optional=False,
     )
+
     if candidate:
         return candidate
 
@@ -318,9 +319,9 @@ def generate_node(
         )
     elif concept.derivation == PurposeLineage.ROOT:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional {[x.address for x in local_optional]}"
         )
-        return gen_select_node(
+        return history.gen_select_node(
             concept,
             local_optional,
             environment,
@@ -328,6 +329,7 @@ def generate_node(
             depth + 1,
             fail_if_not_found=False,
             accept_partial=accept_partial,
+            accept_partial_optional=True,
         )
     else:
         raise ValueError(f"Unknown derivation {concept.derivation}")

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/trilogy/core/processing/node_generators/filter_node.py

@@ -55,6 +55,8 @@ def gen_filter_node(
         depth=depth + 1,
         history=history,
     )
+    if not enrich_node:
+        return filter_node
     x = MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.104}/trilogy/core/processing/node_generators/rowset_node.py

@@ -35,26 +35,30 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
+    if where := select.where_clause:
+        targets = select.output_components + where.conditional.concept_arguments
+    else:
+        targets = select.output_components
     node: StrategyNode = source_concepts(
-        mandatory_list=select.output_components,
+        mandatory_list=targets,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
     )
-    node.conditions = select.where_clause.conditional if select.where_clause else None
-    # rebuild any cached info with the new condition clause
-    node.rebuild_cache()
     if not node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
         )
         return None
+    node.conditions = select.where_clause.conditional if select.where_clause else None
+    # rebuild any cached info with the new condition clause
+    node.rebuild_cache()
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [
         x
         for x in rowset.derived_concepts
-        if x.address == concept.address or x.address in enrichment
+        # if x.address == concept.address or x.address in enrichment
     ]
     additional_relevant = [
         x for x in select.output_components if x.address in enrichment
@@ -68,7 +72,7 @@ def gen_rowset_node(
         for item in additional_relevant:
             node.partial_concepts.append(item)
 
-    # assume grain to be outoput of select
+    # assume grain to be output of select
     # but don't include anything aggregate at this point
     assert node.resolution_cache
     node.resolution_cache.grain = concept_list_to_grain(