pytrilogy 0.0.3.38__tar.gz → 0.0.3.39__tar.gz

{pytrilogy-0.0.3.38/pytrilogy.egg-info → pytrilogy-0.0.3.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.38
+Version: 0.0.3.39
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.38
+Version: 0.0.3.39
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/tests/test_models.py

@@ -328,7 +328,7 @@ select avg_greater_ten;
     lineage = env.concepts["avg_greater_ten"].lineage
     assert isinstance(lineage, AggregateWrapper)
     assert isinstance(
-        env.concepts[lineage.function.concept_arguments[0]].lineage, RowsetItem
+        env.concepts[lineage.function.concept_arguments[0].address].lineage, RowsetItem
     )
 
 

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.38"
+__version__ = "0.0.3.39"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/core/environment_helpers.py

@@ -1,10 +1,10 @@
 from trilogy.constants import DEFAULT_NAMESPACE
 from trilogy.core.enums import ConceptSource, DatePart, FunctionType, Purpose
-from trilogy.core.functions import AttrAccess, FunctionFactory
-from trilogy.core.models.author import Concept, Function, Metadata
+from trilogy.core.functions import AttrAccess
+from trilogy.core.models.author import Concept, Function, Metadata, TraitDataType
 from trilogy.core.models.core import DataType, StructType, arg_to_datatype
 from trilogy.core.models.environment import Environment
-from trilogy.parsing.common import Meta, process_function_args
+from trilogy.parsing.common import Meta
 
 FUNCTION_DESCRIPTION_MAPS = {
     FunctionType.DATE: "The date part of a timestamp/date. Integer, 0-31 depending on month.",
@@ -19,7 +19,6 @@ FUNCTION_DESCRIPTION_MAPS = {
 
 
 def generate_date_concepts(concept: Concept, environment: Environment):
-    factory = FunctionFactory(environment=environment)
     if concept.metadata and concept.metadata.description:
         base_description = concept.metadata.description
     else:
@@ -28,24 +27,36 @@ def generate_date_concepts(concept: Concept, environment: Environment):
         base_line_number = concept.metadata.line_number
     else:
         base_line_number = None
-    for ftype in [
-        FunctionType.MONTH,
-        FunctionType.YEAR,
-        FunctionType.QUARTER,
-        FunctionType.DAY,
-        FunctionType.DAY_OF_WEEK,
-    ]:
+    arg_tuples: list[tuple[FunctionType, TraitDataType]] = [
+        (FunctionType.MONTH, TraitDataType(type=DataType.INTEGER, traits=["month"])),
+        (FunctionType.YEAR, TraitDataType(type=DataType.INTEGER, traits=["year"])),
+        (
+            FunctionType.QUARTER,
+            TraitDataType(type=DataType.INTEGER, traits=["quarter"]),
+        ),
+        (FunctionType.DAY, TraitDataType(type=DataType.INTEGER, traits=["day"])),
+        (
+            FunctionType.DAY_OF_WEEK,
+            TraitDataType(type=DataType.INTEGER, traits=["day_of_week"]),
+        ),
+    ]
+    for ftype, dtype in arg_tuples:
         fname = ftype.name.lower()
+        address = concept.address + f".{fname}"
+        if address in environment.concepts:
+            continue
         default_type = (
             Purpose.CONSTANT
             if concept.purpose == Purpose.CONSTANT
             else Purpose.PROPERTY
         )
-        function = factory.create_function(
+        function = Function.model_construct(
             operator=ftype,
-            args=[concept],
+            arguments=[concept.reference],
+            output_datatype=dtype,
+            output_purpose=default_type,
         )
-        new_concept = Concept(
+        new_concept = Concept.model_construct(
             name=f"{concept.name}.{fname}",
             datatype=function.output_datatype,
             purpose=default_type,
@@ -61,15 +72,19 @@ def generate_date_concepts(concept: Concept, environment: Environment):
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
         )
-        if new_concept.name in environment.concepts:
-            continue
         environment.add_concept(new_concept, add_derived=False)
     for grain in [DatePart.MONTH, DatePart.YEAR]:
-        function = factory.create_function(
+        address = concept.address + f".{grain.value}_start"
+        if address in environment.concepts:
+            continue
+        function = Function.model_construct(
             operator=FunctionType.DATE_TRUNCATE,
-            args=[concept, grain],
+            arguments=[concept.reference, grain],
+            output_datatype=DataType.DATE,
+            output_purpose=default_type,
+            arg_count=2,
         )
-        new_concept = Concept(
+        new_concept = Concept.model_construct(
             name=f"{concept.name}.{grain.value}_start",
             datatype=DataType.DATE,
             purpose=Purpose.PROPERTY,
@@ -85,13 +100,11 @@ def generate_date_concepts(concept: Concept, environment: Environment):
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
         )
-        if new_concept.name in environment.concepts:
-            continue
+
         environment.add_concept(new_concept, add_derived=False)
 
 
 def generate_datetime_concepts(concept: Concept, environment: Environment):
-    factory = FunctionFactory(environment=environment)
     if concept.metadata and concept.metadata.description:
         base_description = concept.metadata.description
     else:
@@ -100,24 +113,29 @@ def generate_datetime_concepts(concept: Concept, environment: Environment):
         base_line_number = concept.metadata.line_number
     else:
         base_line_number = None
-    for ftype in [
-        FunctionType.DATE,
-        FunctionType.HOUR,
-        FunctionType.MINUTE,
-        FunctionType.SECOND,
-    ]:
+    setup_tuples: list[tuple[FunctionType, DataType | TraitDataType]] = [
+        (FunctionType.DATE, DataType.DATE),
+        (FunctionType.HOUR, TraitDataType(type=DataType.INTEGER, traits=["hour"])),
+        (FunctionType.MINUTE, TraitDataType(type=DataType.INTEGER, traits=["minute"])),
+        (FunctionType.SECOND, TraitDataType(type=DataType.INTEGER, traits=["second"])),
+    ]
+    for ftype, datatype in setup_tuples:
         fname = ftype.name.lower()
+        address = concept.address + f".{fname}"
+        if address in environment.concepts:
+            continue
         default_type = (
             Purpose.CONSTANT
             if concept.purpose == Purpose.CONSTANT
             else Purpose.PROPERTY
         )
-
-        const_function = factory.create_function(
+        const_function = Function.model_construct(
             operator=ftype,
-            args=[concept],
+            arguments=[concept.reference],
+            output_datatype=datatype,
+            output_purpose=default_type,
         )
-        new_concept = Concept(
+        new_concept = Concept.model_construct(
             name=f"{concept.name}.{fname}",
             datatype=const_function.output_datatype,
             purpose=default_type,
@@ -148,15 +166,18 @@ def generate_key_concepts(concept: Concept, environment: Environment):
     else:
         base_line_number = None
     for ftype in [FunctionType.COUNT]:
+        address = concept.address + f".{ftype.name.lower()}"
+        if address in environment.concepts:
+            continue
         fname = ftype.name.lower()
         default_type = Purpose.METRIC
-        const_function: Function = Function(
+        const_function: Function = Function.model_construct(
             operator=ftype,
             output_datatype=DataType.INTEGER,
             output_purpose=default_type,
-            arguments=[concept],
+            arguments=[concept.reference],
         )
-        new_concept = Concept(
+        new_concept = Concept.model_construct(
             name=f"{concept.name}.{fname}",
             datatype=DataType.INTEGER,
             purpose=default_type,
@@ -172,8 +193,6 @@ def generate_key_concepts(concept: Concept, environment: Environment):
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
         )
-        if new_concept.name in environment.concepts:
-            continue
         environment.add_concept(new_concept, add_derived=False)
 
 
@@ -199,10 +218,7 @@ def generate_related_concepts(
 
     if isinstance(concept.datatype, StructType):
         for key, value in concept.datatype.fields_map.items():
-            args = process_function_args(
-                [concept, key], meta=meta, environment=environment
-            )
-            auto = Concept(
+            auto = Concept.model_construct(
                 name=key,
                 datatype=arg_to_datatype(value),
                 purpose=Purpose.PROPERTY,
@@ -212,7 +228,8 @@ def generate_related_concepts(
                     and environment.namespace != DEFAULT_NAMESPACE
                     else concept.name
                 ),
-                lineage=AttrAccess(args, environment=environment),
+                lineage=AttrAccess([concept.reference, key], environment=environment),
+                grain=concept.grain,
             )
             environment.add_concept(auto, meta=meta)
             if isinstance(value, Concept):

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/core/functions.py

@@ -17,6 +17,7 @@ from trilogy.core.exceptions import InvalidSyntaxException
 from trilogy.core.models.author import (
     AggregateWrapper,
     Concept,
+    ConceptRef,
     Function,
     Parenthetical,
     UndefinedConcept,
@@ -35,7 +36,7 @@ from trilogy.core.models.core import (
 )
 from trilogy.core.models.environment import Environment
 
-GENERIC_ARGS = Concept | Function | str | int | float | date | datetime
+GENERIC_ARGS = Concept | ConceptRef | Function | str | int | float | date | datetime
 
 
 @dataclass
@@ -773,7 +774,7 @@ class FunctionFactory:
                 output_purpose = Purpose.PROPERTY
         return Function(
             operator=operator,
-            arguments=full_args,
+            arguments=full_args,  # type: ignore
             output_datatype=final_output_type,
             output_purpose=output_purpose,
             valid_inputs=valid_inputs,
@@ -804,7 +805,7 @@ def create_function_derived_concept(
         purpose=purpose,
         lineage=Function(
             operator=operator,
-            arguments=arguments,
+            arguments=[x.reference for x in arguments],
             output_datatype=output_type,
             output_purpose=purpose,
             arg_count=len(arguments),
@@ -896,7 +897,7 @@ def Split(args: list[Concept], environment: Environment) -> Function:
     )
 
 
-def AttrAccess(args: list[GENERIC_ARGS], environment: Environment):
+def AttrAccess(args: list[ConceptRef | str | int], environment: Environment):
     return FunctionFactory(environment).create_function(
         args=args, operator=FunctionType.ATTR_ACCESS
     )

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/core/models/author.py

@@ -1604,9 +1604,6 @@ class Function(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
     ] = None
     arguments: Sequence[FuncArgs]
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
     def __repr__(self):
         return f'{self.operator.value}({",".join([str(a) for a in self.arguments])})'
 

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/core/models/environment.py

@@ -129,6 +129,9 @@ class EnvironmentConceptDict(dict):
     def __getitem__(
         self, key: str, line_no: int | None = None, file: Path | None = None
     ) -> Concept | UndefinedConceptFull:
+        # fast access path
+        if key in self.keys():
+            return super(EnvironmentConceptDict, self).__getitem__(key)
         if isinstance(key, ConceptRef):
             return self.__getitem__(key.address, line_no=line_no, file=file)
         try:
@@ -311,11 +314,15 @@ class Environment(BaseModel):
             f.write(self.model_dump_json())
         return ppath
 
-    def validate_concept(self, new_concept: Concept, meta: Meta | None = None):
+    def validate_concept(
+        self, new_concept: Concept, meta: Meta | None = None
+    ) -> Concept | None:
         lookup = new_concept.address
+        if lookup not in self.concepts:
+            return None
         existing: Concept = self.concepts.get(lookup)  # type: ignore
-        if not existing or isinstance(existing, UndefinedConcept):
-            return
+        if isinstance(existing, UndefinedConcept):
+            return None
 
         def handle_persist():
             deriv_lookup = (
@@ -361,7 +368,7 @@ class Environment(BaseModel):
         if existing and self.config.allow_duplicate_declaration:
             if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
                 return handle_persist()
-            return
+            return None
         elif existing.metadata:
             if existing.metadata.concept_source == ConceptSource.PERSIST_STATEMENT:
                 return handle_persist()

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/parsing/common.py

@@ -102,8 +102,10 @@ def process_function_arg(
             concept.metadata.line_number = meta.line
         environment.add_concept(concept, meta=meta)
         return concept
+    elif isinstance(arg, Concept):
+        return arg.reference
     elif isinstance(arg, ConceptRef):
-        return environment.concepts[arg.address]
+        return environment.concepts[arg.address].reference
     return arg
 
 
@@ -111,8 +113,8 @@ def process_function_args(
     args,
     meta: Meta | None,
     environment: Environment,
-) -> List[Concept | Function | str | int | float | date | datetime]:
-    final: List[Concept | Function | str | int | float | date | datetime] = []
+) -> List[ConceptRef | Function | str | int | float | date | datetime]:
+    final: List[ConceptRef | Function | str | int | float | date | datetime] = []
     for arg in args:
         final.append(process_function_arg(arg, meta, environment))
     return final

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/parsing/parse_engine.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from datetime import date, datetime
 from enum import Enum
+from logging import getLogger
 from os.path import dirname, join
 from pathlib import Path
 from re import IGNORECASE
@@ -137,6 +138,8 @@ from trilogy.parsing.common import (
 )
 from trilogy.parsing.exceptions import ParseError
 
+perf_logger = getLogger("trilogy.parse.performance")
+
 
 class ParsePass(Enum):
     INITIAL = 1
@@ -407,6 +410,7 @@ class ParseToObjects(Transformer):
         mapping = self.environment.concepts[address]
         datatype = mapping.output_datatype
         return ConceptRef(
+            # this is load-bearing to handle pseudonyms
             address=mapping.address,
             metadata=Metadata(line_number=meta.line),
             datatype=datatype,
@@ -1010,6 +1014,7 @@ class ParseToObjects(Transformer):
         return text
 
     def import_statement(self, args: list[str]) -> ImportStatement:
+        start = datetime.now()
         if len(args) == 2:
             alias = args[-1]
             cache_key = args[-1]
@@ -1046,9 +1051,11 @@ class ParseToObjects(Transformer):
             )
 
         if token_lookup in self.tokens:
+            perf_logger.debug(f"\tTokens cached for {token_lookup}")
             raw_tokens = self.tokens[token_lookup]
             text = self.text_lookup[token_lookup]
         else:
+            perf_logger.debug(f"\tTokens not cached for {token_lookup}, resolving")
             text = self.resolve_import_address(target, is_stdlib)
             self.text_lookup[token_lookup] = text
 
@@ -1061,12 +1068,19 @@ class ParseToObjects(Transformer):
             self.tokens[token_lookup] = raw_tokens
 
         if cache_lookup in self.parsed:
+            perf_logger.debug(f"\tEnvironment cached for {token_lookup}")
             nparser = self.parsed[cache_lookup]
             new_env = nparser.environment
             if nparser.parse_pass != ParsePass.VALIDATION:
                 # nparser.transform(raw_tokens)
+                second_pass_start = datetime.now()
                 nparser.run_second_parse_pass()
+                second_pass_end = datetime.now()
+                perf_logger.debug(
+                    f"{second_pass_end - second_pass_start} seconds | Import {alias} key ({cache_key}) second pass took {second_pass_end - second_pass_start} to parse, {len(new_env.concepts)} concepts"
+                )
         else:
+            perf_logger.debug(f"\tParsing new for {token_lookup}")
             try:
                 new_env = Environment(
                     working_path=dirname(target),
@@ -1098,6 +1112,10 @@ class ParseToObjects(Transformer):
         self.environment.add_import(
             alias, new_env, Import(alias=alias, path=parsed_path)
         )
+        end = datetime.now()
+        perf_logger.debug(
+            f"{end - start} seconds | Import {alias} key ({cache_key}) took  to parse, {len(new_env.concepts)} concepts"
+        )
         return imps
 
     @v_args(meta=True)
@@ -1273,8 +1291,8 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def function_binding_item(self, meta: Meta, args) -> ArgBinding:
         if len(args) == 2:
-            return ArgBinding(name=args[0], default=args[1])
-        return ArgBinding(name=args[0], default=None)
+            return ArgBinding.model_construct(name=args[0], default=args[1])
+        return ArgBinding.model_construct(name=args[0], default=None)
 
     @v_args(meta=True)
     def raw_function(self, meta: Meta, args) -> FunctionDeclaration:
@@ -1857,7 +1875,7 @@ def parse_text(
     parser = ParseToObjects(
         environment=environment, import_keys=["root"], parse_config=parse_config
     )
-
+    start = datetime.now()
     try:
         parser.set_text(text)
         # disable fail on missing to allow for circular dependencies
@@ -1867,6 +1885,10 @@ def parse_text(
         pass_two = parser.run_second_parse_pass()
         output = [v for v in pass_two if v]
         environment.concepts.fail_on_missing = True
+        end = datetime.now()
+        perf_logger.debug(
+            f"Parse time: {end - start} for {len(text)} characters, {len(output)} objects"
+        )
     except VisitError as e:
         unpack_visit_error(e)
         # this will never be reached

{pytrilogy-0.0.3.38 → pytrilogy-0.0.3.39}/trilogy/parsing/trilogy.lark

@@ -129,7 +129,7 @@
     
     limit: "LIMIT"i /[0-9]+/
     
-    !window_order: ("TOP"i | "BOTTOM"i)
+    !window_order: /TOP|BOTTOM/i
     
     window: window_order /[0-9]+/
     
@@ -139,19 +139,26 @@
     
     over_list: concept_lit (","  concept_lit )* ","?
     
-    !ordering: ("ASC"i | "DESC"i ) ("NULLS"i ("FIRST"i | "LAST"i | "AUTO"i) )?
+    !ordering: /ASC|DESC/i  ("NULLS"i /FIRST|LAST|AUTO/i )?
     
     order_by: "ORDER"i "BY"i order_list
     
     //WHERE STATEMENT
+    LOGICAL_OR: "or"i
+    LOGICAL_AND: "and"i
     
-    !logical_operator: "and"i | "or"i
-    
-    condition_parenthetical: "(" conditional ")"
+    conditional: _or_condition
+
+    _or_condition: _and_condition
+               | (_or_condition LOGICAL_OR _and_condition)
 
-    _condition_unit: (expr | condition_parenthetical)
+    _and_condition: _condition_unit
+               | (_and_condition LOGICAL_AND _condition_unit)
 
-    conditional: _condition_unit (logical_operator _condition_unit)*
+    condition_parenthetical: "(" conditional ")"
+
+    _condition_unit: expr
+                  | condition_parenthetical
     
     where: "WHERE"i conditional
     
@@ -159,7 +166,7 @@
 
     !array_comparison: ( ("NOT"i "IN"i) | "IN"i)
 
-    COMPARISON_OPERATOR: (/is[\s]+not/ | "is" |"=" | ">" | "<" | ">=" | "<=" | "!=")
+    COMPARISON_OPERATOR: /(is\s+not|is|=|>=|<=|!=|>|<)/i
     
     comparison: expr COMPARISON_OPERATOR expr
 
@@ -185,16 +192,28 @@
     attr_access: expr  "." string_lit 
 
 
-    expr:   _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | union | _static_functions |  literal |  concept_lit  | index_access |  map_key_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like |  aggregate_by | custom_function 
-    
+    expr: _basic_expr | _functional_expr | _operation_expr |  _access_expr
+
+   # Most common/basic expressions
+   _basic_expr: literal | concept_lit | parenthetical | expr_tuple
+
+   # Operations and comparisons 
+   _operation_expr: comparison | alt_like | between_comparison | subselect_comparison
+
+   # Function-like expressions
+   _functional_expr:  _constant_functions | _static_functions  | filter_item | aggregate_functions | window_item | custom_function | fgroup | unnest | union |  aggregate_by
+
+   # Access patterns
+   _access_expr: index_access | map_key_access | attr_access
     // functions
     
-    fadd: ("add"i "(" expr "," expr ")" ) | ( expr "+" expr )
+    fadd: (/add\(/ expr "," expr ")" ) | ( expr "+" expr )
     fsub: ("subtract"i "(" expr "," expr ")" ) | ( expr "-" expr )
     fmul: ("multiply"i "(" expr "," expr ")" ) | ( expr "*" expr )
     fdiv: ( "divide"i "(" expr "," expr ")") | ( expr "/" expr )
     fmod: ( "mod"i "(" expr "," (int_lit | concept_lit ) ")") | ( expr "%" (int_lit | concept_lit ) )
-    fround: "round"i "(" expr "," expr ")"
+    _ROUND.1: "round"i "("
+    fround: _ROUND expr "," expr ")"
     fabs: "abs"i "(" expr ")"
     _SQRT.1: "sqrt("
     fsqrt:  _SQRT expr ")"
@@ -355,9 +374,9 @@
     
     literal: null_lit | string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit | struct_lit | tuple_lit
 
-    MODIFIER: "Optional"i | "Partial"i | "Nullable"i
+    MODIFIER: /OPTIONAL|PARTIAL|NULLABLE/i
 
-    SHORTHAND_MODIFIER: "~" | "?"
+    SHORTHAND_MODIFIER: /~|\?/
 
     struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
 
@@ -373,7 +392,6 @@
     PROPERTY: "property"i
     CONST: "const"i | "constant"i
     AUTO: "AUTO"i 
-
     // meta functions
     CONCEPTS: "CONCEPTS"i
     DATASOURCES: "DATASOURCES"i