pytrilogy 0.0.2.19__py3-none-any.whl → 0.0.2.21__py3-none-any.whl

{pytrilogy-0.0.2.19.dist-info → pytrilogy-0.0.2.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.19
+Version: 0.0.2.21
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.19.dist-info → pytrilogy-0.0.2.21.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=3gRtKqbvnX1RJDJWia2dlhjAU87WHpfzoZZM3KSaFaw,291
+trilogy/__init__.py,sha256=OCXbyl49qGgFMeTRpc5o4WNKO8b9oOs7BpkoN7kt4sg,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=pZkOneh_65f9Ua6NICu1bHAFAbmQxmiXRXS7tsmCWbQ,1235
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -7,8 +7,8 @@ trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=A9VC0lbP5eo9sndm2TzA-nNJRRmvbjE918ZiEXtcQ_c,6043
+trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
+trilogy/core/enums.py,sha256=IHi-Z656detnp2MKMeCphsnVL7-uaogehulJT3JA64I,6087
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
@@ -16,9 +16,9 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ShFTStIKbgI-3EZIU0xTumI78AC5QlvARwnBM53P2O0,10677
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=nSCOXedYRFgYdZa45dputERCvgPRQpRvCyiGglyeM0g,150985
+trilogy/core/models.py,sha256=2YoDJz0cK5EljGtWCt1uFtcV56VWS9v8RfHw1nvaFrc,153416
 trilogy/core/optimization.py,sha256=od_60A9F8J8Nj24MHgrxl4vwRwmBFH13TMdoMQvgVKs,7717
-trilogy/core/query_processor.py,sha256=kXuBsIaRHu1s7zB_rAnT_gRe4-VgRSrPE1TnVJXFLtc,16447
+trilogy/core/query_processor.py,sha256=D4-FKc-wP6c98-35DPcBE7-Gp6vbyj9DTZrubgtqACA,16561
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
@@ -38,7 +38,7 @@ trilogy/core/processing/node_generators/multiselect_node.py,sha256=_KO9lqzHQoy4V
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=4aoSkynWYcKAxeN4fU5jnCdxausa5rNgFokoVhPXI80,13511
 trilogy/core/processing/node_generators/rowset_node.py,sha256=gU_ybfYXO9tZqHjUSABIioVpb8AWtITpegj3IGSf2GI,4587
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=MKjlXqFBSin6cTnS6n5lEcNBJsMvSefDIXOwYNVbM0s,10371
-trilogy/core/processing/node_generators/select_node.py,sha256=vUg3gXHGvagdbniIAE7DdqJcQ0V1VAfHtTrw3edYPso,1734
+trilogy/core/processing/node_generators/select_node.py,sha256=nwXHQF6C-aQUIelx9dyxN2pK3muL-4-6RIqnqQqNwtw,1808
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cZ26CN338CBnd6asML1OBUtNcDzmNlFpY0Vnade4yrc,2256
 trilogy/core/processing/node_generators/window_node.py,sha256=jy3FF8uN0VA7yyrBeR40B9CAqR_5qBP4PiS6Gr-f-7w,2590
 trilogy/core/processing/nodes/__init__.py,sha256=qS5EJDRwwIrCEfS7ibCA2ESE0RPzsAIii1UWd_wNsHA,4760
@@ -70,14 +70,14 @@ trilogy/parsing/common.py,sha256=-4LM71ocidA8DI2RngqFEOmhzBrIt8VdBTO4x2BpD8E,950
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=r0B7tXAIfTMJ42ZJwP70ulQBrOoM1AAiXFbCQ2rOpQs,64720
+trilogy/parsing/parse_engine.py,sha256=_orHmZTdExJYkuNRmAaazdXpZcvtPMLKIBI9vBAgq6w,64871
 trilogy/parsing/render.py,sha256=8yxerPAi4AhlhPBlAfbYbOM3F9rz6HzpWVEWPtK2VEg,12321
 trilogy/parsing/trilogy.lark,sha256=0JAvQBACFNL-X61I0tB_0QPZgsguZgerfHBv903oKh0,11623
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.19.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.19.dist-info/METADATA,sha256=BgEvyZ13aahBi8bULS0lT9Rx5ixjpZ99ZbHeC8mg5Mw,8132
-pytrilogy-0.0.2.19.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-pytrilogy-0.0.2.19.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.19.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.19.dist-info/RECORD,,
+pytrilogy-0.0.2.21.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.21.dist-info/METADATA,sha256=zUKc5pQLdUDjCdk3crvgw6wTA_caPxs0qbHGdm_kD8E,8132
+pytrilogy-0.0.2.21.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+pytrilogy-0.0.2.21.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.21.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.21.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.19"
+__version__ = "0.0.2.21"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/constants.py

@@ -1,3 +1,4 @@
 CONSTANT_DATASET: str = "preql_internal_constant_dataset"
 ALL_ROWS_CONCEPT = "all_rows"
 INTERNAL_NAMESPACE = "__preql_internal"
+PERSISTED_CONCEPT_PREFIX = "__pre_persist"

trilogy/core/enums.py

@@ -12,6 +12,7 @@ class UnnestMode(Enum):
 
 class ConceptSource(Enum):
     MANUAL = "manual"
+    PERSIST_STATEMENT = "persist_statement"
     AUTO_DERIVED = "auto_derived"
 
 

trilogy/core/models.py

@@ -44,6 +44,7 @@ from trilogy.core.constants import (
     ALL_ROWS_CONCEPT,
     INTERNAL_NAMESPACE,
     CONSTANT_DATASET,
+    PERSISTED_CONCEPT_PREFIX,
 )
 from trilogy.core.enums import (
     InfiniteFunctionArgs,
@@ -559,19 +560,16 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
-        if not self._address_cache:
-            self._address_cache = f"{self.namespace}.{self.name}"
-        return self._address_cache
-
-    @address.setter
-    def address(self, address: str) -> None:
-        self._address_cache = address
+        return f"{self.namespace}.{self.name}"
 
     def set_name(self, name: str):
         self.name = name
-        self.address = f"{self.namespace}.{self.name}"
+        try:
+            del self.address
+        except AttributeError:
+            pass
 
     @property
     def output(self) -> "Concept":
@@ -915,10 +913,10 @@ class Grain(Mergeable, BaseModel):
         )
 
     @cached_property
-    def set(self):
+    def set(self) -> set[str]:
         base = []
         for x in self.components_copy:
-            if x.derivation == PurposeLineage.ROWSET:
+            if isinstance(x.lineage, RowsetItem):
                 base.append(x.lineage.content.address)
             else:
                 base.append(x.address)
@@ -3134,6 +3132,12 @@ class EnvironmentConceptDict(dict):
     def values(self) -> ValuesView[Concept]:  # type: ignore
         return super().values()
 
+    def get(self, key: str, default: Concept | None = None) -> Concept | None:  # type: ignore
+        try:
+            return self.__getitem__(key)
+        except UndefinedConceptException:
+            return default
+
     def __getitem__(
         self, key, line_no: int | None = None
     ) -> Concept | UndefinedConcept:
@@ -3260,34 +3264,62 @@ class Environment(BaseModel):
         for datasource in self.datasources.values():
             for concept in datasource.output_concepts:
                 concrete_addresses.add(concept.address)
-        current_mat = [x.address for x in self.materialized_concepts]
-        self.materialized_concepts = [
-            c for c in self.concepts.values() if c.address in concrete_addresses
-        ]
-        # include aliased concepts
-        self.materialized_concepts += [
-            c
-            for c in self.alias_origin_lookup.values()
-            if c.address in concrete_addresses
-        ]
-        new = [
-            x.address
-            for x in self.materialized_concepts
-            if x.address not in current_mat
-        ]
-        if new:
-            logger.debug(f"Environment added new materialized concepts {new}")
+        self.materialized_concepts = unique(
+            [c for c in self.concepts.values() if c.address in concrete_addresses]
+            + [
+                c
+                for c in self.alias_origin_lookup.values()
+                if c.address in concrete_addresses
+            ],
+            "address",
+        )
 
-    def validate_concept(self, lookup: str, meta: Meta | None = None):
+    def validate_concept(self, new_concept: Concept, meta: Meta | None = None):
+        lookup = new_concept.address
         existing: Concept = self.concepts.get(lookup)  # type: ignore
         if not existing:
             return
-        elif existing and self.environment_config.allow_duplicate_declaration:
+
+        def handle_persist():
+            deriv_lookup = (
+                f"{existing.namespace}.{PERSISTED_CONCEPT_PREFIX}_{existing.name}"
+            )
+
+            alt_source = self.alias_origin_lookup.get(deriv_lookup)
+            if not alt_source:
+                return None
+            # if the new concept binding has no lineage
+            # nothing to cause us to think a persist binding
+            # needs to be invalidated
+            if not new_concept.lineage:
+                return existing
+            if str(alt_source.lineage) == str(new_concept.lineage):
+                logger.info(
+                    f"Persisted concept {existing.address} matched redeclaration, keeping current persistence binding."
+                )
+                return existing
+            logger.warning(
+                f"Persisted concept {existing.address} lineage {str(alt_source.lineage)} did not match redeclaration {str(new_concept.lineage)}, overwriting and invalidating persist binding."
+            )
+            for k, datasource in self.datasources.items():
+                if existing.address in datasource.output_concepts:
+                    datasource.columns = [
+                        x
+                        for x in datasource.columns
+                        if x.concept.address != existing.address
+                    ]
+            return None
+
+        if existing and self.environment_config.allow_duplicate_declaration:
+            if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
+                return handle_persist()
             return
         elif existing.metadata:
+            if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
+                return handle_persist()
             # if the existing concept is auto derived, we can overwrite it
             if existing.metadata.concept_source == ConceptSource.AUTO_DERIVED:
-                return
+                return None
         elif meta and existing.metadata:
             raise ValueError(
                 f"Assignment to concept '{lookup}' on line {meta.line} is a duplicate"
@@ -3400,39 +3432,58 @@ class Environment(BaseModel):
         meta: Meta | None = None,
         force: bool = False,
         add_derived: bool = True,
+        _ignore_cache: bool = False,
     ):
         if not force:
-            self.validate_concept(concept.address, meta=meta)
+            existing = self.validate_concept(concept, meta=meta)
+            if existing:
+                concept = existing
         if concept.namespace == DEFAULT_NAMESPACE:
             self.concepts[concept.name] = concept
-        else:
-            self.concepts[concept.address] = concept
+        self.concepts[concept.address] = concept
         from trilogy.core.environment_helpers import generate_related_concepts
 
         generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
-        self.gen_concept_list_caches()
+        if not _ignore_cache:
+            self.gen_concept_list_caches()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
         meta: Meta | None = None,
+        _ignore_cache: bool = False,
     ):
-
         self.datasources[datasource.env_label] = datasource
-        for column in datasource.columns:
-            current_concept = column.concept
+        for current_concept in datasource.output_concepts:
             current_derivation = current_concept.derivation
+            # TODO: refine this section;
+            # too hacky for maintainability
             if current_derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT):
-                new_concept = current_concept.model_copy(deep=True)
-                new_concept.set_name("_pre_persist_" + current_concept.name)
-                # remove the associated lineage
-                current_concept.lineage = None
-                self.add_concept(new_concept, meta=meta, force=True)
-                self.add_concept(current_concept, meta=meta, force=True)
-                self.merge_concept(new_concept, current_concept, [])
-
-        self.gen_concept_list_caches()
+                persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + current_concept.name
+                # override the current concept source to reflect that it's now coming from a datasource
+                if (
+                    current_concept.metadata.concept_source
+                    != ConceptSource.PERSIST_STATEMENT
+                ):
+                    new_concept = current_concept.model_copy(deep=True)
+                    new_concept.set_name(persisted)
+                    self.add_concept(
+                        new_concept, meta=meta, force=True, _ignore_cache=True
+                    )
+                    current_concept.metadata.concept_source = (
+                        ConceptSource.PERSIST_STATEMENT
+                    )
+                    # remove the associated lineage
+                    # to make this a root for discovery purposes
+                    # as it now "exists" in a table
+                    current_concept.lineage = None
+                    self.add_concept(
+                        current_concept, meta=meta, force=True, _ignore_cache=True
+                    )
+                    self.merge_concept(new_concept, current_concept, [])
+        if not _ignore_cache:
+            self.gen_concept_list_caches()
         return datasource
 
     def delete_datasource(

trilogy/core/processing/node_generators/select_node.py

@@ -39,9 +39,10 @@ def gen_select_node(
         ]
     )
     if materialized_lcl != all_lcl:
+        missing = all_lcl.difference(materialized_lcl)
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
-            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}) "
+            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}, missing {missing}) "
         )
         if fail_if_not_found:
             raise NoDatasourceException(f"No datasource exists for {concept}")

trilogy/core/query_processor.py

@@ -11,6 +11,7 @@ from trilogy.core.models import (
     Concept,
     Environment,
     PersistStatement,
+    ConceptDeclarationStatement,
     SelectStatement,
     MultiSelectStatement,
     CTE,
@@ -393,6 +394,8 @@ def process_auto(
         return process_persist(environment, statement, hooks)
     elif isinstance(statement, SelectStatement):
         return process_query(environment, statement, hooks)
+    elif isinstance(statement, ConceptDeclarationStatement):
+        return None
     raise ValueError(f"Do not know how to process {type(statement)}")
 
 

trilogy/parsing/parse_engine.py

@@ -814,14 +814,19 @@ class ParseToObjects(Transformer):
                 raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
         for _, concept in nparser.environment.concepts.items():
-            self.environment.add_concept(concept.with_namespace(alias))
+            self.environment.add_concept(
+                concept.with_namespace(alias), _ignore_cache=True
+            )
 
         for _, datasource in nparser.environment.datasources.items():
-            self.environment.add_datasource(datasource.with_namespace(alias))
+            self.environment.add_datasource(
+                datasource.with_namespace(alias), _ignore_cache=True
+            )
         imps = ImportStatement(
             alias=alias, path=Path(args[0]), environment=nparser.environment
         )
         self.environment.imports[alias] = imps
+        self.environment.gen_concept_list_caches()
         return imps
 
     @v_args(meta=True)