pytrilogy 0.0.2.32__tar.gz → 0.0.2.34__tar.gz

{pytrilogy-0.0.2.32/pytrilogy.egg-info → pytrilogy-0.0.2.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.32
+Version: 0.0.2.34
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.32
+Version: 0.0.2.34
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/pytrilogy.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ tests/test_imports.py
 tests/test_metadata.py
 tests/test_models.py
 tests/test_multi_join_assignments.py
+tests/test_parse_engine.py
 tests/test_parsing.py
 tests/test_partial_handling.py
 tests/test_query_processing.py

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/tests/test_environment.py

@@ -1,6 +1,7 @@
 from trilogy.core.models import Environment
 from pathlib import Path
 from trilogy.core.enums import Modifier
+from trilogy.core.exceptions import UndefinedConceptException
 
 
 def test_environment_serialization(test_environment: Environment):
@@ -23,6 +24,21 @@ def test_environment_from_path():
     assert "local.id" in env.concepts
 
 
+def test_environment_invalid():
+
+    env = Environment()
+    env.concepts.fail_on_missing = False
+    x = env.concepts["abc"]
+    assert x.name == "abc"
+
+    env.concepts.fail_on_missing = True
+    try:
+        x = env.concepts["abc"]
+        assert 1 == 0
+    except Exception as e:
+        assert isinstance(e, UndefinedConceptException)
+
+
 def test_environment_merge():
     env1: Environment
     env1, _ = Environment().parse(

pytrilogy-0.0.2.34/tests/test_metadata.py

@@ -0,0 +1,27 @@
+from trilogy import parse, Environment
+from pathlib import Path
+
+
+def test_metadata():
+    env, _ = parse(
+        """key user_id int metadata(description="the description");
+property user_id.display_name string metadata(description="The display name");"""
+    )
+
+    assert env.concepts["user_id"].metadata.description == "the description"
+    assert env.concepts["display_name"].metadata.description == "The display name"
+
+
+def test_import_metadata():
+    env = Environment(working_path=Path(__file__).parent)
+    env, _ = parse(
+        """import test_env as env; # Dragon metrics
+        import test_env as env2;""",
+        environment=env,
+    )
+
+    assert "Dragon metrics" in env.concepts["env.id"].metadata.description
+
+    env2 = env.concepts["env2.id"]
+    assert env2.namespace == "env2"
+    assert env.concepts["env2.id"].metadata.description is None

pytrilogy-0.0.2.34/tests/test_parse_engine.py

@@ -0,0 +1,30 @@
+from trilogy.parsing.parse_engine import ParseToObjects, PARSER, unpack_visit_error
+from trilogy import Environment
+from trilogy.core.exceptions import UndefinedConceptException
+from pytest import raises
+
+TEXT = """
+const a <- 1;
+
+select
+    a,
+    b
+;
+"""
+
+
+def test_parser():
+    env = Environment()
+    x = ParseToObjects(environment=env)
+    x.environment.concepts.fail_on_missing = False
+    x.set_text(TEXT)
+    tokens = PARSER.parse(TEXT)
+    x.transform(tokens)
+    failed = False
+    try:
+        x.hydrate_missing()
+    except Exception as e:
+        failed = True
+        with raises(UndefinedConceptException):
+            unpack_visit_error(e)
+    assert failed

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/tests/test_undefined_concept.py

@@ -32,6 +32,6 @@ def test_undefined_concept_dict():
     try:
         env["orid"]
     except UndefinedConceptException as e:
-        assert e.suggestions == ["local.order_id"]
+        assert e.suggestions == ["order_id"]
         assert "suggestions" in e.message.lower()
         assert "order_id" in e.message.lower()

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.32"
+__version__ = "0.0.2.34"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/constants.py

@@ -36,7 +36,7 @@ class Comments:
     basic: bool = True
     joins: bool = True
     nullable: bool = False
-    partial: bool = False
+    partial: bool = True
 
 
 # TODO: support loading from environments

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/core/enums.py

@@ -13,6 +13,7 @@ class UnnestMode(Enum):
 class ConceptSource(Enum):
     MANUAL = "manual"
     CTE = "cte"
+    SELECT = "select"
     PERSIST_STATEMENT = "persist_statement"
     AUTO_DERIVED = "auto_derived"
 

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/core/models.py

@@ -2077,6 +2077,11 @@ class Datasource(HasUUID, Namespaced, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         original = [c for c in self.columns if c.concept.address == source.address]
+        early_exit_check = [
+            c for c in self.columns if c.concept.address == target.address
+        ]
+        if early_exit_check:
+            return None
         if len(original) != 1:
             raise ValueError(
                 f"Expected exactly one column to merge, got {len(original)} for {source.address}, {[x.alias for x in original]}"
@@ -3213,7 +3218,7 @@ class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(self, *args, **kwargs)
         self.undefined: dict[str, UndefinedConcept] = {}
-        self.fail_on_missing: bool = False
+        self.fail_on_missing: bool = True
         self.populate_default_concepts()
 
     def populate_default_concepts(self):
@@ -3332,7 +3337,6 @@ class Environment(BaseModel):
 
     materialized_concepts: List[Concept] = Field(default_factory=list)
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
-    _parse_count: int = 0
 
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
@@ -3442,19 +3446,22 @@ class Environment(BaseModel):
         exists = False
         existing = self.imports[alias]
         if imp_stm:
-            if any([x.path == imp_stm.path for x in existing]):
+            if any(
+                [x.path == imp_stm.path and x.alias == imp_stm.alias for x in existing]
+            ):
                 exists = True
-
         else:
-            if any([x.path == source.working_path for x in existing]):
+            if any(
+                [x.path == source.working_path and x.alias == alias for x in existing]
+            ):
                 exists = True
             imp_stm = ImportStatement(alias=alias, path=Path(source.working_path))
-
         same_namespace = alias == self.namespace
 
         if not exists:
             self.imports[alias].append(imp_stm)
-
+        # we can't exit early
+        # as there may be new concepts
         for k, concept in source.concepts.items():
             if same_namespace:
                 new = self.add_concept(concept, _ignore_cache=True)
@@ -3485,13 +3492,25 @@ class Environment(BaseModel):
         self.gen_concept_list_caches()
         return self
 
-    def add_file_import(self, path: str, alias: str, env: Environment | None = None):
-        from trilogy.parsing.parse_engine import ParseToObjects, PARSER
-
-        apath = path.split(".")
-        apath[-1] = apath[-1] + ".preql"
+    def add_file_import(
+        self, path: str | Path, alias: str, env: Environment | None = None
+    ):
+        from trilogy.parsing.parse_engine import (
+            ParseToObjects,
+            PARSER,
+            gen_cache_lookup,
+        )
 
-        target: Path = Path(self.working_path, *apath)
+        if isinstance(path, str):
+            if path.endswith(".preql"):
+                path = path.rsplit(".", 1)[0]
+            if "." not in path:
+                target = Path(self.working_path, path)
+            else:
+                target = Path(self.working_path, *path.split("."))
+            target = target.with_suffix(".preql")
+        else:
+            target = path
         if alias in self.imports:
             imports = self.imports[alias]
             for x in imports:
@@ -3502,18 +3521,24 @@ class Environment(BaseModel):
                 ImportStatement(alias=alias, path=target, environment=env)
             )
         else:
+            parse_address = gen_cache_lookup(str(target), alias, str(self.working_path))
             try:
                 with open(target, "r", encoding="utf-8") as f:
                     text = f.read()
+                nenv = Environment(
+                    working_path=target.parent,
+                )
+                nenv.concepts.fail_on_missing = False
                 nparser = ParseToObjects(
-                    visit_tokens=True,
-                    text=text,
                     environment=Environment(
                         working_path=target.parent,
                     ),
-                    parse_address=str(target),
+                    parse_address=parse_address,
+                    token_address=target,
                 )
+                nparser.set_text(text)
                 nparser.transform(PARSER.parse(text))
+
             except Exception as e:
                 raise ImportError(
                     f"Unable to import file {target.parent}, parsing error: {e}"

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/core/processing/utility.py

@@ -425,6 +425,8 @@ def is_scalar_condition(
             return True
         if element.lineage and isinstance(element.lineage, AggregateWrapper):
             return is_scalar_condition(element.lineage, materialized)
+        if element.lineage and isinstance(element.lineage, Function):
+            return is_scalar_condition(element.lineage, materialized)
         return True
     elif isinstance(element, AggregateWrapper):
         return is_scalar_condition(element.function, materialized)

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/parsing/common.py

@@ -133,7 +133,9 @@ def constant_to_concept(
     )
 
 
-def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
+def function_to_concept(
+    parent: Function, name: str, namespace: str, metadata: Metadata | None = None
+) -> Concept:
     pkeys: List[Concept] = []
     for x in parent.arguments:
         pkeys += [
@@ -159,6 +161,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
         purpose = Purpose.CONSTANT
     else:
         purpose = parent.output_purpose
+    fmetadata = metadata or Metadata()
     if grain is not None:
         return Concept(
             name=name,
@@ -169,6 +172,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
             keys=keys,
             modifiers=modifiers,
             grain=grain,
+            metadata=fmetadata,
         )
 
     return Concept(
@@ -179,6 +183,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
         namespace=namespace,
         keys=keys,
         modifiers=modifiers,
+        metadata=fmetadata,
     )
 
 
@@ -305,7 +310,7 @@ def arbitrary_to_concept(
     elif isinstance(parent, Function):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
-        return function_to_concept(parent, name, namespace)
+        return function_to_concept(parent, name, namespace, metadata=metadata)
     elif isinstance(parent, ListWrapper):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"

{pytrilogy-0.0.2.32 → pytrilogy-0.0.2.34}/trilogy/parsing/parse_engine.py

@@ -1,7 +1,7 @@
 from os.path import dirname, join
 from typing import List, Optional, Tuple, Union
 from re import IGNORECASE
-from lark import Lark, Transformer, v_args, Tree
+from lark import Lark, Transformer, v_args, Tree, ParseTree
 from lark.exceptions import (
     UnexpectedCharacters,
     UnexpectedEOF,
@@ -32,6 +32,7 @@ from trilogy.core.enums import (
     ShowCategory,
     FunctionClass,
     IOType,
+    ConceptSource,
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
@@ -126,13 +127,16 @@ from trilogy.parsing.common import (
 from dataclasses import dataclass
 
 
+CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
+
+SELF_LABEL = "root"
+
+
 @dataclass
 class WholeGrainWrapper:
     where: WhereClause
 
 
-CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
-
 with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
     PARSER = Lark(
         f.read(),
@@ -144,6 +148,10 @@ with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
     )
 
 
+def gen_cache_lookup(path: str, alias: str, parent: str) -> str:
+    return path + alias + parent
+
+
 def parse_concept_reference(
     name: str, environment: Environment, purpose: Optional[Purpose] = None
 ) -> Tuple[str, str, str, str | None]:
@@ -219,39 +227,47 @@ def unwrap_transformation(
 class ParseToObjects(Transformer):
     def __init__(
         self,
-        visit_tokens,
-        text,
         environment: Environment,
         parse_address: str | None = None,
+        token_address: Path | None = None,
         parsed: dict[str, "ParseToObjects"] | None = None,
+        tokens: dict[Path | str, ParseTree] | None = None,
+        text_lookup: dict[Path | str, str] | None = None,
     ):
-        Transformer.__init__(self, visit_tokens)
-        self.text = text
+        Transformer.__init__(self, True)
         self.environment: Environment = environment
-        self.parse_address = parse_address or "root"
+        self.parse_address: str = parse_address or SELF_LABEL
+        self.token_address: Path | str = token_address or SELF_LABEL
         self.parsed: dict[str, ParseToObjects] = parsed if parsed else {}
+        self.tokens: dict[Path | str, ParseTree] = tokens or {}
+        self.text_lookup: dict[Path | str, str] = text_lookup or {}
         # we do a second pass to pick up circular dependencies
         # after initial parsing
         self.pass_count = 1
-        self._results_stash = None
 
-    def transform(self, tree):
+    def set_text(self, text: str):
+        self.text_lookup[self.token_address] = text
+
+    def transform(self, tree: Tree):
         results = super().transform(tree)
-        self._results_stash = results
-        self.environment._parse_count += 1
+        self.tokens[self.token_address] = tree
         return results
 
+    def prepare_parse(self):
+        self.pass_count = 1
+        self.environment.concepts.fail_on_missing = False
+        for _, v in self.parsed.items():
+            v.prepare_parse()
+
     def hydrate_missing(self):
         self.pass_count = 2
         for k, v in self.parsed.items():
-
             if v.pass_count == 2:
                 continue
+            print(f"Hydrating {k}")
             v.hydrate_missing()
         self.environment.concepts.fail_on_missing = True
-        # if not self.environment.concepts.undefined:
-        #     return self._results_stash
-        reparsed = self.transform(PARSER.parse(self.text))
+        reparsed = self.transform(self.tokens[self.token_address])
         self.environment.concepts.undefined = {}
         return reparsed
 
@@ -266,6 +282,18 @@ class ParseToObjects(Transformer):
                     output.concept.metadata.description
                     or args[1].text.split("#")[1].strip()
                 )
+        if isinstance(output, ImportStatement):
+            if len(args) > 1 and isinstance(args[1], Comment):
+                comment = args[1].text.split("#")[1].strip()
+                namespace = output.alias
+                for _, v in self.environment.concepts.items():
+                    if v.namespace == namespace:
+                        if v.metadata.description:
+                            v.metadata.description = (
+                                f"{comment}: {v.metadata.description}"
+                            )
+                        else:
+                            v.metadata.description = comment
 
         return args[0]
 
@@ -647,45 +675,41 @@ class ParseToObjects(Transformer):
         return Comment(text=args.value)
 
     @v_args(meta=True)
-    def select_transform(self, meta, args) -> ConceptTransform:
+    def select_transform(self, meta: Meta, args) -> ConceptTransform:
 
         output: str = args[1]
-        function = unwrap_transformation(args[0])
+        transformation = unwrap_transformation(args[0])
         lookup, namespace, output, parent = parse_concept_reference(
             output, self.environment
         )
 
-        if isinstance(function, AggregateWrapper):
-            concept = agg_wrapper_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, WindowItem):
-            concept = window_item_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, FilterItem):
-            concept = filter_item_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, CONSTANT_TYPES):
-            concept = constant_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, Function):
-            concept = function_to_concept(function, namespace=namespace, name=output)
-        else:
-            if function.output_purpose == Purpose.PROPERTY:
-                pkeys = [x for x in function.arguments if isinstance(x, Concept)]
-                grain = Grain(components=pkeys)
-                keys = tuple(grain.components_copy)
-            else:
-                grain = None
-                keys = None
-            concept = Concept(
-                name=output,
-                datatype=function.output_datatype,
-                purpose=function.output_purpose,
-                lineage=function,
-                namespace=namespace,
-                grain=Grain(components=[]) if not grain else grain,
-                keys=keys,
+        metadata = Metadata(line_number=meta.line, concept_source=ConceptSource.SELECT)
+
+        if isinstance(transformation, AggregateWrapper):
+            concept = agg_wrapper_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
             )
-        if concept.metadata:
-            concept.metadata.line_number = meta.line
+        elif isinstance(transformation, WindowItem):
+            concept = window_item_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        elif isinstance(transformation, FilterItem):
+            concept = filter_item_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        elif isinstance(transformation, CONSTANT_TYPES):
+            concept = constant_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        elif isinstance(transformation, Function):
+            concept = function_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        else:
+            raise SyntaxError("Invalid transformation")
+
         self.environment.add_concept(concept, meta=meta)
-        return ConceptTransform(function=function, output=concept)
+        return ConceptTransform(function=transformation, output=concept)
 
     @v_args(meta=True)
     def concept_nullable_modifier(self, meta: Meta, args) -> Modifier:
@@ -709,7 +733,7 @@ class ParseToObjects(Transformer):
         if len(args) != 1:
             raise ParseError(
                 "Malformed select statement"
-                f" {args} {self.text[meta.start_pos:meta.end_pos]}"
+                f" {args} {self.text_lookup[self.parse_address][meta.start_pos:meta.end_pos]}"
             )
         content = args[0]
         if isinstance(content, ConceptTransform):
@@ -807,25 +831,46 @@ class ParseToObjects(Transformer):
         path = args[0].split(".")
 
         target = join(self.environment.working_path, *path) + ".preql"
-        if target in self.parsed:
-            nparser = self.parsed[target]
+
+        # tokens + text are cached by path
+        token_lookup = Path(target)
+
+        # cache lookups by the target, the alias, and the file we're importing it from
+        cache_lookup = gen_cache_lookup(
+            path=target, alias=alias, parent=str(self.token_address)
+        )
+        if token_lookup in self.tokens:
+            raw_tokens = self.tokens[token_lookup]
+            text = self.text_lookup[token_lookup]
+        else:
+            text = self.resolve_import_address(target)
+            self.text_lookup[token_lookup] = text
+
+            raw_tokens = PARSER.parse(text)
+            self.tokens[token_lookup] = raw_tokens
+
+        if cache_lookup in self.parsed:
+            nparser = self.parsed[cache_lookup]
         else:
             try:
-                text = self.resolve_import_address(target)
+                new_env = Environment(
+                    working_path=dirname(target),
+                )
+                new_env.concepts.fail_on_missing = False
                 nparser = ParseToObjects(
-                    visit_tokens=True,
-                    text=text,
-                    environment=Environment(
-                        working_path=dirname(target),
-                        # namespace=alias,
-                    ),
-                    parse_address=target,
+                    environment=new_env,
+                    parse_address=cache_lookup,
+                    token_address=token_lookup,
                     parsed={**self.parsed, **{self.parse_address: self}},
+                    tokens={**self.tokens, **{token_lookup: raw_tokens}},
+                    text_lookup={**self.text_lookup, **{token_lookup: text}},
                 )
-                nparser.transform(PARSER.parse(text))
-                self.parsed[target] = nparser
+                nparser.transform(raw_tokens)
+                self.parsed[cache_lookup] = nparser
                 # add the parsed objects of the import in
                 self.parsed = {**self.parsed, **nparser.parsed}
+                self.tokens = {**self.tokens, **nparser.tokens}
+                self.text_lookup = {**self.text_lookup, **nparser.text_lookup}
             except Exception as e:
                 raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
@@ -1912,11 +1957,14 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
     ],
 ]:
     environment = environment or Environment()
-    parser = ParseToObjects(visit_tokens=True, text=text, environment=environment)
+    parser = ParseToObjects(environment=environment)
 
     try:
+        parser.set_text(text)
+        # disable fail on missing to allow for circular dependencies
+        parser.prepare_parse()
         parser.transform(PARSER.parse(text))
-        # handle circular dependencies
+        # this will reset fail on missing
         pass_two = parser.hydrate_missing()
         output = [v for v in pass_two if v]
     except VisitError as e:

pytrilogy-0.0.2.32/tests/test_metadata.py

@@ -1,11 +0,0 @@
-from trilogy import parse
-
-
-def test_metadata():
-    env, _ = parse(
-        """key user_id int metadata(description="the description");
-property user_id.display_name string metadata(description="The display name");"""
-    )
-
-    assert env.concepts["user_id"].metadata.description == "the description"
-    assert env.concepts["display_name"].metadata.description == "The display name"