pytrilogy 0.0.2.20__py3-none-any.whl → 0.0.2.21__py3-none-any.whl

{pytrilogy-0.0.2.20.dist-info → pytrilogy-0.0.2.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.20
+Version: 0.0.2.21
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.20.dist-info → pytrilogy-0.0.2.21.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=jkM9h2qEkIh-R8xZca7g_Qzl5cC8w48aOmkcJUjjTAs,291
+trilogy/__init__.py,sha256=OCXbyl49qGgFMeTRpc5o4WNKO8b9oOs7BpkoN7kt4sg,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=pZkOneh_65f9Ua6NICu1bHAFAbmQxmiXRXS7tsmCWbQ,1235
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -7,8 +7,8 @@ trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=A9VC0lbP5eo9sndm2TzA-nNJRRmvbjE918ZiEXtcQ_c,6043
+trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
+trilogy/core/enums.py,sha256=IHi-Z656detnp2MKMeCphsnVL7-uaogehulJT3JA64I,6087
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
@@ -16,9 +16,9 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ShFTStIKbgI-3EZIU0xTumI78AC5QlvARwnBM53P2O0,10677
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=O1_HxOQ0muMbjWVcOH1mmEFG3IM7taaPgfgCnZv21W8,150676
+trilogy/core/models.py,sha256=2YoDJz0cK5EljGtWCt1uFtcV56VWS9v8RfHw1nvaFrc,153416
 trilogy/core/optimization.py,sha256=od_60A9F8J8Nj24MHgrxl4vwRwmBFH13TMdoMQvgVKs,7717
-trilogy/core/query_processor.py,sha256=kXuBsIaRHu1s7zB_rAnT_gRe4-VgRSrPE1TnVJXFLtc,16447
+trilogy/core/query_processor.py,sha256=D4-FKc-wP6c98-35DPcBE7-Gp6vbyj9DTZrubgtqACA,16561
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
@@ -38,7 +38,7 @@ trilogy/core/processing/node_generators/multiselect_node.py,sha256=_KO9lqzHQoy4V
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=4aoSkynWYcKAxeN4fU5jnCdxausa5rNgFokoVhPXI80,13511
 trilogy/core/processing/node_generators/rowset_node.py,sha256=gU_ybfYXO9tZqHjUSABIioVpb8AWtITpegj3IGSf2GI,4587
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=MKjlXqFBSin6cTnS6n5lEcNBJsMvSefDIXOwYNVbM0s,10371
-trilogy/core/processing/node_generators/select_node.py,sha256=vUg3gXHGvagdbniIAE7DdqJcQ0V1VAfHtTrw3edYPso,1734
+trilogy/core/processing/node_generators/select_node.py,sha256=nwXHQF6C-aQUIelx9dyxN2pK3muL-4-6RIqnqQqNwtw,1808
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cZ26CN338CBnd6asML1OBUtNcDzmNlFpY0Vnade4yrc,2256
 trilogy/core/processing/node_generators/window_node.py,sha256=jy3FF8uN0VA7yyrBeR40B9CAqR_5qBP4PiS6Gr-f-7w,2590
 trilogy/core/processing/nodes/__init__.py,sha256=qS5EJDRwwIrCEfS7ibCA2ESE0RPzsAIii1UWd_wNsHA,4760
@@ -75,9 +75,9 @@ trilogy/parsing/render.py,sha256=8yxerPAi4AhlhPBlAfbYbOM3F9rz6HzpWVEWPtK2VEg,123
 trilogy/parsing/trilogy.lark,sha256=0JAvQBACFNL-X61I0tB_0QPZgsguZgerfHBv903oKh0,11623
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.20.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.20.dist-info/METADATA,sha256=pmZ2uTW1EBclZPel8B698DU3sdTmutM5vnu25tNTHfM,8132
-pytrilogy-0.0.2.20.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-pytrilogy-0.0.2.20.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.20.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.20.dist-info/RECORD,,
+pytrilogy-0.0.2.21.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.21.dist-info/METADATA,sha256=zUKc5pQLdUDjCdk3crvgw6wTA_caPxs0qbHGdm_kD8E,8132
+pytrilogy-0.0.2.21.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+pytrilogy-0.0.2.21.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.21.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.21.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.20"
+__version__ = "0.0.2.21"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/constants.py

@@ -1,3 +1,4 @@
 CONSTANT_DATASET: str = "preql_internal_constant_dataset"
 ALL_ROWS_CONCEPT = "all_rows"
 INTERNAL_NAMESPACE = "__preql_internal"
+PERSISTED_CONCEPT_PREFIX = "__pre_persist"

trilogy/core/enums.py

@@ -12,6 +12,7 @@ class UnnestMode(Enum):
 
 class ConceptSource(Enum):
     MANUAL = "manual"
+    PERSIST_STATEMENT = "persist_statement"
     AUTO_DERIVED = "auto_derived"
 
 

trilogy/core/models.py

@@ -44,6 +44,7 @@ from trilogy.core.constants import (
     ALL_ROWS_CONCEPT,
     INTERNAL_NAMESPACE,
     CONSTANT_DATASET,
+    PERSISTED_CONCEPT_PREFIX,
 )
 from trilogy.core.enums import (
     InfiniteFunctionArgs,
@@ -912,10 +913,10 @@ class Grain(Mergeable, BaseModel):
         )
 
     @cached_property
-    def set(self):
+    def set(self) -> set[str]:
         base = []
         for x in self.components_copy:
-            if x.derivation == PurposeLineage.ROWSET:
+            if isinstance(x.lineage, RowsetItem):
                 base.append(x.lineage.content.address)
             else:
                 base.append(x.address)
@@ -3131,6 +3132,12 @@ class EnvironmentConceptDict(dict):
     def values(self) -> ValuesView[Concept]:  # type: ignore
         return super().values()
 
+    def get(self, key: str, default: Concept | None = None) -> Concept | None:  # type: ignore
+        try:
+            return self.__getitem__(key)
+        except UndefinedConceptException:
+            return default
+
     def __getitem__(
         self, key, line_no: int | None = None
     ) -> Concept | UndefinedConcept:
@@ -3257,24 +3264,62 @@ class Environment(BaseModel):
         for datasource in self.datasources.values():
             for concept in datasource.output_concepts:
                 concrete_addresses.add(concept.address)
-        self.materialized_concepts = [
-            c for c in self.concepts.values() if c.address in concrete_addresses
-        ] + [
-            c
-            for c in self.alias_origin_lookup.values()
-            if c.address in concrete_addresses
-        ]
+        self.materialized_concepts = unique(
+            [c for c in self.concepts.values() if c.address in concrete_addresses]
+            + [
+                c
+                for c in self.alias_origin_lookup.values()
+                if c.address in concrete_addresses
+            ],
+            "address",
+        )
 
-    def validate_concept(self, lookup: str, meta: Meta | None = None):
+    def validate_concept(self, new_concept: Concept, meta: Meta | None = None):
+        lookup = new_concept.address
         existing: Concept = self.concepts.get(lookup)  # type: ignore
         if not existing:
             return
-        elif existing and self.environment_config.allow_duplicate_declaration:
+
+        def handle_persist():
+            deriv_lookup = (
+                f"{existing.namespace}.{PERSISTED_CONCEPT_PREFIX}_{existing.name}"
+            )
+
+            alt_source = self.alias_origin_lookup.get(deriv_lookup)
+            if not alt_source:
+                return None
+            # if the new concept binding has no lineage
+            # nothing to cause us to think a persist binding
+            # needs to be invalidated
+            if not new_concept.lineage:
+                return existing
+            if str(alt_source.lineage) == str(new_concept.lineage):
+                logger.info(
+                    f"Persisted concept {existing.address} matched redeclaration, keeping current persistence binding."
+                )
+                return existing
+            logger.warning(
+                f"Persisted concept {existing.address} lineage {str(alt_source.lineage)} did not match redeclaration {str(new_concept.lineage)}, overwriting and invalidating persist binding."
+            )
+            for k, datasource in self.datasources.items():
+                if existing.address in datasource.output_concepts:
+                    datasource.columns = [
+                        x
+                        for x in datasource.columns
+                        if x.concept.address != existing.address
+                    ]
+            return None
+
+        if existing and self.environment_config.allow_duplicate_declaration:
+            if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
+                return handle_persist()
             return
         elif existing.metadata:
+            if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
+                return handle_persist()
             # if the existing concept is auto derived, we can overwrite it
             if existing.metadata.concept_source == ConceptSource.AUTO_DERIVED:
-                return
+                return None
         elif meta and existing.metadata:
             raise ValueError(
                 f"Assignment to concept '{lookup}' on line {meta.line} is a duplicate"
@@ -3390,11 +3435,12 @@ class Environment(BaseModel):
         _ignore_cache: bool = False,
     ):
         if not force:
-            self.validate_concept(concept.address, meta=meta)
+            existing = self.validate_concept(concept, meta=meta)
+            if existing:
+                concept = existing
         if concept.namespace == DEFAULT_NAMESPACE:
             self.concepts[concept.name] = concept
-        else:
-            self.concepts[concept.address] = concept
+        self.concepts[concept.address] = concept
         from trilogy.core.environment_helpers import generate_related_concepts
 
         generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
@@ -3408,21 +3454,34 @@ class Environment(BaseModel):
         meta: Meta | None = None,
         _ignore_cache: bool = False,
     ):
-
         self.datasources[datasource.env_label] = datasource
-        for column in datasource.columns:
-            current_concept = column.concept
+        for current_concept in datasource.output_concepts:
             current_derivation = current_concept.derivation
+            # TODO: refine this section;
+            # too hacky for maintainability
             if current_derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT):
-                new_concept = current_concept.model_copy(deep=True)
-                new_concept.set_name("_pre_persist_" + current_concept.name)
-                # remove the associated lineage
-                current_concept.lineage = None
-                self.add_concept(new_concept, meta=meta, force=True, _ignore_cache=True)
-                self.add_concept(
-                    current_concept, meta=meta, force=True, _ignore_cache=True
-                )
-                self.merge_concept(new_concept, current_concept, [])
+                persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + current_concept.name
+                # override the current concept source to reflect that it's now coming from a datasource
+                if (
+                    current_concept.metadata.concept_source
+                    != ConceptSource.PERSIST_STATEMENT
+                ):
+                    new_concept = current_concept.model_copy(deep=True)
+                    new_concept.set_name(persisted)
+                    self.add_concept(
+                        new_concept, meta=meta, force=True, _ignore_cache=True
+                    )
+                    current_concept.metadata.concept_source = (
+                        ConceptSource.PERSIST_STATEMENT
+                    )
+                    # remove the associated lineage
+                    # to make this a root for discovery purposes
+                    # as it now "exists" in a table
+                    current_concept.lineage = None
+                    self.add_concept(
+                        current_concept, meta=meta, force=True, _ignore_cache=True
+                    )
+                    self.merge_concept(new_concept, current_concept, [])
         if not _ignore_cache:
             self.gen_concept_list_caches()
         return datasource

trilogy/core/processing/node_generators/select_node.py

@@ -39,9 +39,10 @@ def gen_select_node(
         ]
     )
     if materialized_lcl != all_lcl:
+        missing = all_lcl.difference(materialized_lcl)
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
-            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}) "
+            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}, missing {missing}) "
         )
         if fail_if_not_found:
             raise NoDatasourceException(f"No datasource exists for {concept}")

trilogy/core/query_processor.py

@@ -11,6 +11,7 @@ from trilogy.core.models import (
     Concept,
     Environment,
     PersistStatement,
+    ConceptDeclarationStatement,
     SelectStatement,
     MultiSelectStatement,
     CTE,
@@ -393,6 +394,8 @@ def process_auto(
         return process_persist(environment, statement, hooks)
     elif isinstance(statement, SelectStatement):
         return process_query(environment, statement, hooks)
+    elif isinstance(statement, ConceptDeclarationStatement):
+        return None
     raise ValueError(f"Do not know how to process {type(statement)}")