pytrilogy 0.0.2.19__tar.gz → 0.0.2.21__tar.gz

{pytrilogy-0.0.2.19/pytrilogy.egg-info → pytrilogy-0.0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.19
+Version: 0.0.2.21
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.19
+Version: 0.0.2.21
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/tests/test_models.py

@@ -100,12 +100,16 @@ def test_conditional(test_environment, test_environment_graph):
     merged = condition_a + condition_b
     assert merged == condition_a
 
-    test_concept_two = list(test_environment.concepts.values())[-2]
+    test_concept_two = [
+        x
+        for x in test_environment.concepts.values()
+        if x.address != test_concept.address
+    ].pop()
     condition_c = Conditional(
         left=test_concept, right=test_concept_two, operator=BooleanOperator.AND
     )
     merged_two = condition_a + condition_c
-    assert merged_two.left == condition_a
+    assert merged_two.left == condition_a, f"{str(merged_two.left), str(condition_a)}"
     assert merged_two.right == condition_c
     assert merged_two.operator == BooleanOperator.AND
 

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.19"
+__version__ = "0.0.2.21"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/core/constants.py

@@ -1,3 +1,4 @@
 CONSTANT_DATASET: str = "preql_internal_constant_dataset"
 ALL_ROWS_CONCEPT = "all_rows"
 INTERNAL_NAMESPACE = "__preql_internal"
+PERSISTED_CONCEPT_PREFIX = "__pre_persist"

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/core/enums.py

@@ -12,6 +12,7 @@ class UnnestMode(Enum):
 
 class ConceptSource(Enum):
     MANUAL = "manual"
+    PERSIST_STATEMENT = "persist_statement"
     AUTO_DERIVED = "auto_derived"
 
 

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/core/models.py

@@ -44,6 +44,7 @@ from trilogy.core.constants import (
     ALL_ROWS_CONCEPT,
     INTERNAL_NAMESPACE,
     CONSTANT_DATASET,
+    PERSISTED_CONCEPT_PREFIX,
 )
 from trilogy.core.enums import (
     InfiniteFunctionArgs,
@@ -559,19 +560,16 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
-        if not self._address_cache:
-            self._address_cache = f"{self.namespace}.{self.name}"
-        return self._address_cache
-
-    @address.setter
-    def address(self, address: str) -> None:
-        self._address_cache = address
+        return f"{self.namespace}.{self.name}"
 
     def set_name(self, name: str):
         self.name = name
-        self.address = f"{self.namespace}.{self.name}"
+        try:
+            del self.address
+        except AttributeError:
+            pass
 
     @property
     def output(self) -> "Concept":
@@ -915,10 +913,10 @@ class Grain(Mergeable, BaseModel):
         )
 
     @cached_property
-    def set(self):
+    def set(self) -> set[str]:
         base = []
         for x in self.components_copy:
-            if x.derivation == PurposeLineage.ROWSET:
+            if isinstance(x.lineage, RowsetItem):
                 base.append(x.lineage.content.address)
             else:
                 base.append(x.address)
@@ -3134,6 +3132,12 @@ class EnvironmentConceptDict(dict):
     def values(self) -> ValuesView[Concept]:  # type: ignore
         return super().values()
 
+    def get(self, key: str, default: Concept | None = None) -> Concept | None:  # type: ignore
+        try:
+            return self.__getitem__(key)
+        except UndefinedConceptException:
+            return default
+
     def __getitem__(
         self, key, line_no: int | None = None
     ) -> Concept | UndefinedConcept:
@@ -3260,34 +3264,62 @@ class Environment(BaseModel):
         for datasource in self.datasources.values():
             for concept in datasource.output_concepts:
                 concrete_addresses.add(concept.address)
-        current_mat = [x.address for x in self.materialized_concepts]
-        self.materialized_concepts = [
-            c for c in self.concepts.values() if c.address in concrete_addresses
-        ]
-        # include aliased concepts
-        self.materialized_concepts += [
-            c
-            for c in self.alias_origin_lookup.values()
-            if c.address in concrete_addresses
-        ]
-        new = [
-            x.address
-            for x in self.materialized_concepts
-            if x.address not in current_mat
-        ]
-        if new:
-            logger.debug(f"Environment added new materialized concepts {new}")
+        self.materialized_concepts = unique(
+            [c for c in self.concepts.values() if c.address in concrete_addresses]
+            + [
+                c
+                for c in self.alias_origin_lookup.values()
+                if c.address in concrete_addresses
+            ],
+            "address",
+        )
 
-    def validate_concept(self, lookup: str, meta: Meta | None = None):
+    def validate_concept(self, new_concept: Concept, meta: Meta | None = None):
+        lookup = new_concept.address
         existing: Concept = self.concepts.get(lookup)  # type: ignore
         if not existing:
             return
-        elif existing and self.environment_config.allow_duplicate_declaration:
+
+        def handle_persist():
+            deriv_lookup = (
+                f"{existing.namespace}.{PERSISTED_CONCEPT_PREFIX}_{existing.name}"
+            )
+
+            alt_source = self.alias_origin_lookup.get(deriv_lookup)
+            if not alt_source:
+                return None
+            # if the new concept binding has no lineage
+            # nothing to cause us to think a persist binding
+            # needs to be invalidated
+            if not new_concept.lineage:
+                return existing
+            if str(alt_source.lineage) == str(new_concept.lineage):
+                logger.info(
+                    f"Persisted concept {existing.address} matched redeclaration, keeping current persistence binding."
+                )
+                return existing
+            logger.warning(
+                f"Persisted concept {existing.address} lineage {str(alt_source.lineage)} did not match redeclaration {str(new_concept.lineage)}, overwriting and invalidating persist binding."
+            )
+            for k, datasource in self.datasources.items():
+                if existing.address in datasource.output_concepts:
+                    datasource.columns = [
+                        x
+                        for x in datasource.columns
+                        if x.concept.address != existing.address
+                    ]
+            return None
+
+        if existing and self.environment_config.allow_duplicate_declaration:
+            if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
+                return handle_persist()
             return
         elif existing.metadata:
+            if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
+                return handle_persist()
             # if the existing concept is auto derived, we can overwrite it
             if existing.metadata.concept_source == ConceptSource.AUTO_DERIVED:
-                return
+                return None
         elif meta and existing.metadata:
             raise ValueError(
                 f"Assignment to concept '{lookup}' on line {meta.line} is a duplicate"
@@ -3400,39 +3432,58 @@ class Environment(BaseModel):
         meta: Meta | None = None,
         force: bool = False,
         add_derived: bool = True,
+        _ignore_cache: bool = False,
     ):
         if not force:
-            self.validate_concept(concept.address, meta=meta)
+            existing = self.validate_concept(concept, meta=meta)
+            if existing:
+                concept = existing
         if concept.namespace == DEFAULT_NAMESPACE:
             self.concepts[concept.name] = concept
-        else:
-            self.concepts[concept.address] = concept
+        self.concepts[concept.address] = concept
         from trilogy.core.environment_helpers import generate_related_concepts
 
         generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
-        self.gen_concept_list_caches()
+        if not _ignore_cache:
+            self.gen_concept_list_caches()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
         meta: Meta | None = None,
+        _ignore_cache: bool = False,
     ):
-
         self.datasources[datasource.env_label] = datasource
-        for column in datasource.columns:
-            current_concept = column.concept
+        for current_concept in datasource.output_concepts:
             current_derivation = current_concept.derivation
+            # TODO: refine this section;
+            # too hacky for maintainability
             if current_derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT):
-                new_concept = current_concept.model_copy(deep=True)
-                new_concept.set_name("_pre_persist_" + current_concept.name)
-                # remove the associated lineage
-                current_concept.lineage = None
-                self.add_concept(new_concept, meta=meta, force=True)
-                self.add_concept(current_concept, meta=meta, force=True)
-                self.merge_concept(new_concept, current_concept, [])
-
-        self.gen_concept_list_caches()
+                persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + current_concept.name
+                # override the current concept source to reflect that it's now coming from a datasource
+                if (
+                    current_concept.metadata.concept_source
+                    != ConceptSource.PERSIST_STATEMENT
+                ):
+                    new_concept = current_concept.model_copy(deep=True)
+                    new_concept.set_name(persisted)
+                    self.add_concept(
+                        new_concept, meta=meta, force=True, _ignore_cache=True
+                    )
+                    current_concept.metadata.concept_source = (
+                        ConceptSource.PERSIST_STATEMENT
+                    )
+                    # remove the associated lineage
+                    # to make this a root for discovery purposes
+                    # as it now "exists" in a table
+                    current_concept.lineage = None
+                    self.add_concept(
+                        current_concept, meta=meta, force=True, _ignore_cache=True
+                    )
+                    self.merge_concept(new_concept, current_concept, [])
+        if not _ignore_cache:
+            self.gen_concept_list_caches()
         return datasource
 
     def delete_datasource(

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/core/processing/node_generators/select_node.py

@@ -39,9 +39,10 @@ def gen_select_node(
         ]
     )
     if materialized_lcl != all_lcl:
+        missing = all_lcl.difference(materialized_lcl)
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
-            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}) "
+            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}, missing {missing}) "
         )
         if fail_if_not_found:
             raise NoDatasourceException(f"No datasource exists for {concept}")

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/core/query_processor.py

@@ -11,6 +11,7 @@ from trilogy.core.models import (
     Concept,
     Environment,
     PersistStatement,
+    ConceptDeclarationStatement,
     SelectStatement,
     MultiSelectStatement,
     CTE,
@@ -393,6 +394,8 @@ def process_auto(
         return process_persist(environment, statement, hooks)
     elif isinstance(statement, SelectStatement):
         return process_query(environment, statement, hooks)
+    elif isinstance(statement, ConceptDeclarationStatement):
+        return None
     raise ValueError(f"Do not know how to process {type(statement)}")
 
 

{pytrilogy-0.0.2.19 → pytrilogy-0.0.2.21}/trilogy/parsing/parse_engine.py

@@ -814,14 +814,19 @@ class ParseToObjects(Transformer):
                 raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
         for _, concept in nparser.environment.concepts.items():
-            self.environment.add_concept(concept.with_namespace(alias))
+            self.environment.add_concept(
+                concept.with_namespace(alias), _ignore_cache=True
+            )
 
         for _, datasource in nparser.environment.datasources.items():
-            self.environment.add_datasource(datasource.with_namespace(alias))
+            self.environment.add_datasource(
+                datasource.with_namespace(alias), _ignore_cache=True
+            )
         imps = ImportStatement(
             alias=alias, path=Path(args[0]), environment=nparser.environment
         )
         self.environment.imports[alias] = imps
+        self.environment.gen_concept_list_caches()
         return imps
 
     @v_args(meta=True)