pytrilogy 0.0.2.49__py3-none-any.whl → 0.0.2.51__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pytrilogy might be problematic. Click here for more details.

Files changed (43) hide show
  1. {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/METADATA +1 -1
  2. {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/RECORD +43 -41
  3. trilogy/__init__.py +1 -1
  4. trilogy/core/enums.py +11 -0
  5. trilogy/core/functions.py +4 -1
  6. trilogy/core/internal.py +5 -1
  7. trilogy/core/models.py +135 -263
  8. trilogy/core/processing/concept_strategies_v3.py +14 -7
  9. trilogy/core/processing/node_generators/basic_node.py +7 -3
  10. trilogy/core/processing/node_generators/common.py +8 -5
  11. trilogy/core/processing/node_generators/filter_node.py +5 -8
  12. trilogy/core/processing/node_generators/group_node.py +24 -9
  13. trilogy/core/processing/node_generators/group_to_node.py +0 -2
  14. trilogy/core/processing/node_generators/multiselect_node.py +4 -5
  15. trilogy/core/processing/node_generators/node_merge_node.py +14 -3
  16. trilogy/core/processing/node_generators/rowset_node.py +3 -5
  17. trilogy/core/processing/node_generators/select_helpers/__init__.py +0 -0
  18. trilogy/core/processing/node_generators/select_helpers/datasource_injection.py +203 -0
  19. trilogy/core/processing/node_generators/select_merge_node.py +153 -66
  20. trilogy/core/processing/node_generators/union_node.py +0 -1
  21. trilogy/core/processing/node_generators/unnest_node.py +0 -2
  22. trilogy/core/processing/node_generators/window_node.py +0 -2
  23. trilogy/core/processing/nodes/base_node.py +2 -36
  24. trilogy/core/processing/nodes/filter_node.py +0 -3
  25. trilogy/core/processing/nodes/group_node.py +19 -13
  26. trilogy/core/processing/nodes/merge_node.py +2 -5
  27. trilogy/core/processing/nodes/select_node_v2.py +0 -4
  28. trilogy/core/processing/nodes/union_node.py +0 -3
  29. trilogy/core/processing/nodes/unnest_node.py +0 -3
  30. trilogy/core/processing/nodes/window_node.py +0 -3
  31. trilogy/core/processing/utility.py +3 -0
  32. trilogy/core/query_processor.py +0 -1
  33. trilogy/dialect/base.py +14 -2
  34. trilogy/dialect/duckdb.py +7 -0
  35. trilogy/hooks/graph_hook.py +17 -1
  36. trilogy/parsing/common.py +68 -17
  37. trilogy/parsing/parse_engine.py +70 -20
  38. trilogy/parsing/render.py +8 -1
  39. trilogy/parsing/trilogy.lark +3 -1
  40. {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/LICENSE.md +0 -0
  41. {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/WHEEL +0 -0
  42. {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/entry_points.txt +0 -0
  43. {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/top_level.txt +0 -0
@@ -34,7 +34,6 @@ class SelectNode(StrategyNode):
34
34
  input_concepts: List[Concept],
35
35
  output_concepts: List[Concept],
36
36
  environment: Environment,
37
- g,
38
37
  datasource: Datasource | None = None,
39
38
  whole_grain: bool = False,
40
39
  parents: List["StrategyNode"] | None = None,
@@ -52,7 +51,6 @@ class SelectNode(StrategyNode):
52
51
  input_concepts=input_concepts,
53
52
  output_concepts=output_concepts,
54
53
  environment=environment,
55
- g=g,
56
54
  whole_grain=whole_grain,
57
55
  parents=parents,
58
56
  depth=depth,
@@ -197,7 +195,6 @@ class SelectNode(StrategyNode):
197
195
  input_concepts=list(self.input_concepts),
198
196
  output_concepts=list(self.output_concepts),
199
197
  environment=self.environment,
200
- g=self.g,
201
198
  datasource=self.datasource,
202
199
  depth=self.depth,
203
200
  parents=self.parents,
@@ -221,7 +218,6 @@ class ConstantNode(SelectNode):
221
218
  input_concepts=list(self.input_concepts),
222
219
  output_concepts=list(self.output_concepts),
223
220
  environment=self.environment,
224
- g=self.g,
225
221
  datasource=self.datasource,
226
222
  depth=self.depth,
227
223
  partial_concepts=list(self.partial_concepts),
@@ -18,7 +18,6 @@ class UnionNode(StrategyNode):
18
18
  input_concepts: List[Concept],
19
19
  output_concepts: List[Concept],
20
20
  environment,
21
- g,
22
21
  whole_grain: bool = False,
23
22
  parents: List["StrategyNode"] | None = None,
24
23
  depth: int = 0,
@@ -27,7 +26,6 @@ class UnionNode(StrategyNode):
27
26
  input_concepts=input_concepts,
28
27
  output_concepts=output_concepts,
29
28
  environment=environment,
30
- g=g,
31
29
  whole_grain=whole_grain,
32
30
  parents=parents,
33
31
  depth=depth,
@@ -43,7 +41,6 @@ class UnionNode(StrategyNode):
43
41
  input_concepts=list(self.input_concepts),
44
42
  output_concepts=list(self.output_concepts),
45
43
  environment=self.environment,
46
- g=self.g,
47
44
  whole_grain=self.whole_grain,
48
45
  parents=self.parents,
49
46
  depth=self.depth,
@@ -23,7 +23,6 @@ class UnnestNode(StrategyNode):
23
23
  input_concepts: List[Concept],
24
24
  output_concepts: List[Concept],
25
25
  environment,
26
- g,
27
26
  whole_grain: bool = False,
28
27
  parents: List["StrategyNode"] | None = None,
29
28
  depth: int = 0,
@@ -32,7 +31,6 @@ class UnnestNode(StrategyNode):
32
31
  input_concepts=input_concepts,
33
32
  output_concepts=output_concepts,
34
33
  environment=environment,
35
- g=g,
36
34
  whole_grain=whole_grain,
37
35
  parents=parents,
38
36
  depth=depth,
@@ -62,7 +60,6 @@ class UnnestNode(StrategyNode):
62
60
  input_concepts=list(self.input_concepts),
63
61
  output_concepts=list(self.output_concepts),
64
62
  environment=self.environment,
65
- g=self.g,
66
63
  whole_grain=self.whole_grain,
67
64
  parents=self.parents,
68
65
  depth=self.depth,
@@ -12,7 +12,6 @@ class WindowNode(StrategyNode):
12
12
  input_concepts: List[Concept],
13
13
  output_concepts: List[Concept],
14
14
  environment,
15
- g,
16
15
  whole_grain: bool = False,
17
16
  parents: List["StrategyNode"] | None = None,
18
17
  depth: int = 0,
@@ -21,7 +20,6 @@ class WindowNode(StrategyNode):
21
20
  input_concepts=input_concepts,
22
21
  output_concepts=output_concepts,
23
22
  environment=environment,
24
- g=g,
25
23
  whole_grain=whole_grain,
26
24
  parents=parents,
27
25
  depth=depth,
@@ -36,7 +34,6 @@ class WindowNode(StrategyNode):
36
34
  input_concepts=list(self.input_concepts),
37
35
  output_concepts=list(self.output_concepts),
38
36
  environment=self.environment,
39
- g=self.g,
40
37
  whole_grain=self.whole_grain,
41
38
  parents=self.parents,
42
39
  depth=self.depth,
@@ -1,4 +1,5 @@
1
1
  from dataclasses import dataclass
2
+ from datetime import date, datetime
2
3
  from enum import Enum
3
4
  from logging import Logger
4
5
  from typing import Any, Dict, List, Set, Tuple
@@ -380,6 +381,8 @@ def is_scalar_condition(
380
381
  int
381
382
  | str
382
383
  | float
384
+ | date
385
+ | datetime
383
386
  | list[Any]
384
387
  | WindowItem
385
388
  | FilterItem
@@ -391,7 +391,6 @@ def get_query_node(
391
391
  input_concepts=ds.output_concepts,
392
392
  parents=[ds],
393
393
  environment=ds.environment,
394
- g=ds.g,
395
394
  partial_concepts=ds.partial_concepts,
396
395
  conditions=final,
397
396
  )
trilogy/dialect/base.py CHANGED
@@ -1,3 +1,4 @@
1
+ from datetime import date, datetime
1
2
  from typing import Any, Callable, Dict, List, Optional, Sequence, Union
2
3
 
3
4
  from jinja2 import Template
@@ -102,13 +103,15 @@ WINDOW_FUNCTION_MAP = {
102
103
  WindowType.AVG: window_factory("avg", include_concept=True),
103
104
  }
104
105
 
105
- DATATYPE_MAP = {
106
+ DATATYPE_MAP: dict[DataType, str] = {
106
107
  DataType.STRING: "string",
107
108
  DataType.INTEGER: "int",
108
109
  DataType.FLOAT: "float",
109
110
  DataType.BOOL: "bool",
110
111
  DataType.NUMERIC: "numeric",
111
112
  DataType.MAP: "map",
113
+ DataType.DATE: "date",
114
+ DataType.DATETIME: "datetime",
112
115
  }
113
116
 
114
117
 
@@ -131,6 +134,7 @@ FUNCTION_MAP = {
131
134
  FunctionType.SPLIT: lambda x: f"split({x[0]}, {x[1]})",
132
135
  FunctionType.IS_NULL: lambda x: f"isnull({x[0]})",
133
136
  FunctionType.BOOL: lambda x: f"CASE WHEN {x[0]} THEN TRUE ELSE FALSE END",
137
+ FunctionType.PARENTHETICAL: lambda x: f"({x[0]})",
134
138
  # Complex
135
139
  FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
136
140
  FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
@@ -138,6 +142,8 @@ FUNCTION_MAP = {
138
142
  FunctionType.ATTR_ACCESS: lambda x: f"""{x[0]}.{x[1].replace("'", "")}""",
139
143
  FunctionType.STRUCT: lambda x: f"{{{', '.join(struct_arg(x))}}}",
140
144
  FunctionType.ARRAY: lambda x: f"[{', '.join(x)}]",
145
+ FunctionType.DATE_LITERAL: lambda x: f"date '{x}'",
146
+ FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
141
147
  # math
142
148
  FunctionType.ADD: lambda x: " + ".join(x),
143
149
  FunctionType.SUBTRACT: lambda x: " - ".join(x),
@@ -454,6 +460,8 @@ class BaseDialect:
454
460
  list,
455
461
  bool,
456
462
  float,
463
+ date,
464
+ datetime,
457
465
  DataType,
458
466
  Function,
459
467
  Parenthetical,
@@ -612,7 +620,7 @@ class BaseDialect:
612
620
  elif isinstance(e, list):
613
621
  return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e])}"
614
622
  elif isinstance(e, DataType):
615
- return str(e.value)
623
+ return self.DATATYPE_MAP.get(e, e.value)
616
624
  elif isinstance(e, DatePart):
617
625
  return str(e.value)
618
626
  elif isinstance(e, NumericType):
@@ -620,6 +628,10 @@ class BaseDialect:
620
628
  elif isinstance(e, MagicConstants):
621
629
  if e == MagicConstants.NULL:
622
630
  return "null"
631
+ elif isinstance(e, date):
632
+ return self.FUNCTION_MAP[FunctionType.DATE_LITERAL](e)
633
+ elif isinstance(e, datetime):
634
+ return self.FUNCTION_MAP[FunctionType.DATETIME_LITERAL](e)
623
635
  else:
624
636
  raise ValueError(f"Unable to render type {type(e)} {e}")
625
637
 
trilogy/dialect/duckdb.py CHANGED
@@ -3,6 +3,7 @@ from typing import Any, Callable, Mapping
3
3
  from jinja2 import Template
4
4
 
5
5
  from trilogy.core.enums import FunctionType, UnnestMode, WindowType
6
+ from trilogy.core.models import DataType
6
7
  from trilogy.dialect.base import BaseDialect
7
8
 
8
9
  WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
@@ -30,6 +31,8 @@ FUNCTION_MAP = {
30
31
  FunctionType.DATE_PART: lambda x: f"date_part('{x[1]}', {x[0]})",
31
32
  FunctionType.DATE_DIFF: lambda x: f"date_diff('{x[2]}', {x[0]}, {x[1]})",
32
33
  FunctionType.CONCAT: lambda x: f"({' || '.join(x)})",
34
+ FunctionType.DATE_LITERAL: lambda x: f"date '{x}'",
35
+ FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
33
36
  }
34
37
 
35
38
  # if an aggregate function is called on a source that is at the same grain as the aggregate
@@ -44,6 +47,9 @@ FUNCTION_GRAIN_MATCH_MAP = {
44
47
  FunctionType.MIN: lambda args: f"{args[0]}",
45
48
  }
46
49
 
50
+ DATATYPE_MAP: dict[DataType, str] = {}
51
+
52
+
47
53
  DUCKDB_TEMPLATE = Template(
48
54
  """{%- if output %}
49
55
  CREATE OR REPLACE TABLE {{ output.address.location }} AS
@@ -84,6 +90,7 @@ class DuckDBDialect(BaseDialect):
84
90
  **BaseDialect.FUNCTION_GRAIN_MATCH_MAP,
85
91
  **FUNCTION_GRAIN_MATCH_MAP,
86
92
  }
93
+ DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, **DATATYPE_MAP}
87
94
  QUOTE_CHARACTER = '"'
88
95
  SQL_TEMPLATE = DUCKDB_TEMPLATE
89
96
  UNNEST_MODE = UnnestMode.DIRECT
@@ -1,7 +1,19 @@
1
+ import sys
2
+ from os import environ
3
+
1
4
  import networkx as nx
2
5
 
3
6
  from trilogy.hooks.base_hook import BaseHook
4
7
 
8
+ if not environ.get("TCL_LIBRARY"):
9
+ minor = sys.version_info.minor
10
+ if minor == 13:
11
+ environ["TCL_LIBRARY"] = r"C:\Program Files\Python313\tcl\tcl8.6"
12
+ elif minor == 12:
13
+ environ["TCL_LIBRARY"] = r"C:\Program Files\Python312\tcl\tcl8.6"
14
+ else:
15
+ pass
16
+
5
17
 
6
18
  class GraphHook(BaseHook):
7
19
  def __init__(self):
@@ -11,11 +23,14 @@ class GraphHook(BaseHook):
11
23
  except ImportError:
12
24
  raise ImportError("GraphHook requires matplotlib and scipy to be installed")
13
25
 
26
+ # https://github.com/python/cpython/issues/125235#issuecomment-2412948604
27
+
14
28
  def query_graph_built(
15
29
  self,
16
30
  graph: nx.DiGraph,
17
31
  target: str | None = None,
18
32
  highlight_nodes: list[str] | None = None,
33
+ remove_isolates: bool = True,
19
34
  ):
20
35
  from matplotlib import pyplot as plt
21
36
 
@@ -24,7 +39,8 @@ class GraphHook(BaseHook):
24
39
  for node in nodes:
25
40
  if "__preql_internal" in node:
26
41
  graph.remove_node(node)
27
- graph.remove_nodes_from(list(nx.isolates(graph)))
42
+ if remove_isolates:
43
+ graph.remove_nodes_from(list(nx.isolates(graph)))
28
44
  color_map = []
29
45
  highlight_nodes = highlight_nodes or []
30
46
  for node in graph:
trilogy/parsing/common.py CHANGED
@@ -1,9 +1,16 @@
1
+ from datetime import date, datetime
1
2
  from typing import List, Tuple
2
3
 
3
4
  from trilogy.constants import (
4
5
  VIRTUAL_CONCEPT_PREFIX,
5
6
  )
6
- from trilogy.core.enums import FunctionType, Modifier, PurposeLineage, WindowType
7
+ from trilogy.core.enums import (
8
+ FunctionType,
9
+ Granularity,
10
+ Modifier,
11
+ PurposeLineage,
12
+ WindowType,
13
+ )
7
14
  from trilogy.core.functions import arg_to_datatype, function_args_to_output_purpose
8
15
  from trilogy.core.models import (
9
16
  AggregateWrapper,
@@ -37,14 +44,22 @@ def process_function_args(
37
44
  args,
38
45
  meta: Meta | None,
39
46
  environment: Environment,
40
- ):
41
- final: List[Concept | Function] = []
47
+ ) -> List[Concept | Function | str | int | float | date | datetime]:
48
+ final: List[Concept | Function | str | int | float | date | datetime] = []
42
49
  for arg in args:
43
50
  # if a function has an anonymous function argument
44
51
  # create an implicit concept
45
- while isinstance(arg, Parenthetical):
46
- arg = arg.content
47
- if isinstance(arg, Function):
52
+ if isinstance(arg, Parenthetical):
53
+ processed = process_function_args([arg.content], meta, environment)
54
+ final.append(
55
+ Function(
56
+ operator=FunctionType.PARENTHETICAL,
57
+ arguments=processed,
58
+ output_datatype=arg_to_datatype(processed[0]),
59
+ output_purpose=function_args_to_output_purpose(processed),
60
+ )
61
+ )
62
+ elif isinstance(arg, Function):
48
63
  # if it's not an aggregate function, we can skip the virtual concepts
49
64
  # to simplify anonymous function handling
50
65
  if (
@@ -57,7 +72,7 @@ def process_function_args(
57
72
  concept = function_to_concept(
58
73
  arg,
59
74
  name=f"{VIRTUAL_CONCEPT_PREFIX}_{arg.operator.value}_{id_hash}",
60
- namespace=environment.namespace,
75
+ environment=environment,
61
76
  )
62
77
  # to satisfy mypy, concept will always have metadata
63
78
  if concept.metadata and meta:
@@ -71,7 +86,7 @@ def process_function_args(
71
86
  concept = arbitrary_to_concept(
72
87
  arg,
73
88
  name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
74
- namespace=environment.namespace,
89
+ environment=environment,
75
90
  )
76
91
  if concept.metadata and meta:
77
92
  concept.metadata.line_number = meta.line
@@ -130,10 +145,39 @@ def constant_to_concept(
130
145
  )
131
146
 
132
147
 
148
+ def concepts_to_grain_concepts(
149
+ concepts: List[Concept] | List[str] | set[str], environment: Environment | None
150
+ ) -> list[Concept]:
151
+ environment = Environment() if environment is None else environment
152
+ pconcepts: list[Concept] = [
153
+ c if isinstance(c, Concept) else environment.concepts[c] for c in concepts
154
+ ]
155
+
156
+ final: List[Concept] = []
157
+ for sub in pconcepts:
158
+ if sub.purpose in (Purpose.PROPERTY, Purpose.METRIC) and sub.keys:
159
+ if any([c.address in pconcepts for c in sub.keys]):
160
+ continue
161
+ if sub.purpose in (Purpose.METRIC,):
162
+ if all([c in pconcepts for c in sub.grain.components]):
163
+ continue
164
+ if sub.granularity == Granularity.SINGLE_ROW:
165
+ continue
166
+ final.append(sub)
167
+ final = unique(final, "address")
168
+ v2 = sorted(final, key=lambda x: x.name)
169
+ return v2
170
+
171
+
133
172
  def function_to_concept(
134
- parent: Function, name: str, namespace: str, metadata: Metadata | None = None
173
+ parent: Function,
174
+ name: str,
175
+ environment: Environment,
176
+ namespace: str | None = None,
177
+ metadata: Metadata | None = None,
135
178
  ) -> Concept:
136
179
  pkeys: List[Concept] = []
180
+ namespace = namespace or environment.namespace
137
181
  for x in parent.arguments:
138
182
  pkeys += [
139
183
  x
@@ -153,7 +197,7 @@ def function_to_concept(
153
197
  key_grain += [*x.keys]
154
198
  else:
155
199
  key_grain.append(x)
156
- keys = tuple(Grain(components=key_grain).components_copy)
200
+ keys = tuple(concepts_to_grain_concepts(key_grain, environment))
157
201
  if not pkeys:
158
202
  purpose = Purpose.CONSTANT
159
203
  else:
@@ -247,7 +291,7 @@ def window_item_to_concept(
247
291
  lineage=parent,
248
292
  metadata=fmetadata,
249
293
  # filters are implicitly at the grain of the base item
250
- grain=Grain(components=grain),
294
+ grain=Grain.from_concepts(grain),
251
295
  namespace=namespace,
252
296
  keys=keys,
253
297
  modifiers=modifiers,
@@ -259,9 +303,8 @@ def agg_wrapper_to_concept(
259
303
  namespace: str,
260
304
  name: str,
261
305
  metadata: Metadata | None = None,
262
- purpose: Purpose | None = None,
263
306
  ) -> Concept:
264
- local_purpose, keys = get_purpose_and_keys(
307
+ _, keys = get_purpose_and_keys(
265
308
  Purpose.METRIC, tuple(parent.by) if parent.by else None
266
309
  )
267
310
  # anything grouped to a grain should be a property
@@ -275,7 +318,7 @@ def agg_wrapper_to_concept(
275
318
  purpose=Purpose.METRIC,
276
319
  metadata=fmetadata,
277
320
  lineage=parent,
278
- grain=Grain(components=parent.by) if parent.by else Grain(),
321
+ grain=Grain.from_concepts(parent.by) if parent.by else Grain(),
279
322
  namespace=namespace,
280
323
  keys=tuple(parent.by) if parent.by else keys,
281
324
  modifiers=modifiers,
@@ -295,15 +338,17 @@ def arbitrary_to_concept(
295
338
  | float
296
339
  | str
297
340
  ),
298
- namespace: str,
341
+ environment: Environment,
342
+ namespace: str | None = None,
299
343
  name: str | None = None,
300
344
  metadata: Metadata | None = None,
301
345
  purpose: Purpose | None = None,
302
346
  ) -> Concept:
347
+ namespace = namespace or environment.namespace
303
348
  if isinstance(parent, AggregateWrapper):
304
349
  if not name:
305
350
  name = f"{VIRTUAL_CONCEPT_PREFIX}_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
306
- return agg_wrapper_to_concept(parent, namespace, name, metadata, purpose)
351
+ return agg_wrapper_to_concept(parent, namespace, name, metadata)
307
352
  elif isinstance(parent, WindowItem):
308
353
  if not name:
309
354
  name = f"{VIRTUAL_CONCEPT_PREFIX}_window_{parent.type.value}_{string_to_hash(str(parent))}"
@@ -315,7 +360,13 @@ def arbitrary_to_concept(
315
360
  elif isinstance(parent, Function):
316
361
  if not name:
317
362
  name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
318
- return function_to_concept(parent, name, namespace, metadata=metadata)
363
+ return function_to_concept(
364
+ parent,
365
+ name,
366
+ metadata=metadata,
367
+ environment=environment,
368
+ namespace=namespace,
369
+ )
319
370
  elif isinstance(parent, ListWrapper):
320
371
  if not name:
321
372
  name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
@@ -1,4 +1,5 @@
1
1
  from dataclasses import dataclass
2
+ from datetime import date, datetime
2
3
  from os.path import dirname, join
3
4
  from pathlib import Path
4
5
  from re import IGNORECASE
@@ -462,7 +463,7 @@ class ParseToObjects(Transformer):
462
463
  datatype=args[2],
463
464
  purpose=args[0],
464
465
  metadata=metadata,
465
- grain=Grain(components=parents),
466
+ grain=Grain(components={x.address for x in parents}),
466
467
  namespace=namespace,
467
468
  keys=parents,
468
469
  modifiers=modifiers,
@@ -529,6 +530,7 @@ class ParseToObjects(Transformer):
529
530
  source_value,
530
531
  name=name,
531
532
  namespace=namespace,
533
+ environment=self.environment,
532
534
  purpose=purpose,
533
535
  metadata=metadata,
534
536
  )
@@ -597,7 +599,7 @@ class ParseToObjects(Transformer):
597
599
  output_purpose=Purpose.CONSTANT,
598
600
  arguments=[constant],
599
601
  ),
600
- grain=Grain(components=[]),
602
+ grain=Grain(components=set()),
601
603
  namespace=namespace,
602
604
  )
603
605
  if concept.metadata:
@@ -622,7 +624,9 @@ class ParseToObjects(Transformer):
622
624
  return args
623
625
 
624
626
  def grain_clause(self, args) -> Grain:
625
- return Grain(components=[self.environment.concepts[a] for a in args[0]])
627
+ return Grain(
628
+ components=set([self.environment.concepts[a].address for a in args[0]])
629
+ )
626
630
 
627
631
  def whole_grain_clause(self, args) -> WholeGrainWrapper:
628
632
  return WholeGrainWrapper(where=args[0])
@@ -669,6 +673,12 @@ class ParseToObjects(Transformer):
669
673
  )
670
674
  for column in columns:
671
675
  column.concept = column.concept.with_grain(datasource.grain)
676
+ if datasource.where:
677
+ for x in datasource.where.concept_arguments:
678
+ if x.address not in datasource.output_concepts:
679
+ raise ValueError(
680
+ f"Datasource {name} where condition depends on concept {x.address} that does not exist on the datasource, line {meta.line}."
681
+ )
672
682
  self.environment.add_datasource(datasource, meta=meta)
673
683
  return datasource
674
684
 
@@ -708,7 +718,11 @@ class ParseToObjects(Transformer):
708
718
  )
709
719
  elif isinstance(transformation, Function):
710
720
  concept = function_to_concept(
711
- transformation, namespace=namespace, name=output, metadata=metadata
721
+ transformation,
722
+ namespace=namespace,
723
+ name=output,
724
+ metadata=metadata,
725
+ environment=self.environment,
712
726
  )
713
727
  else:
714
728
  raise SyntaxError("Invalid transformation")
@@ -764,10 +778,8 @@ class ParseToObjects(Transformer):
764
778
  def handle_order_item(x, namespace: str):
765
779
  if not isinstance(x, Concept):
766
780
  x = arbitrary_to_concept(
767
- x,
768
- namespace=namespace,
781
+ x, namespace=namespace, environment=self.environment
769
782
  )
770
- self.environment.add_concept(x)
771
783
  return x
772
784
 
773
785
  return [
@@ -1022,12 +1034,32 @@ class ParseToObjects(Transformer):
1022
1034
  order_by=order_by,
1023
1035
  meta=Metadata(line_number=meta.line),
1024
1036
  )
1025
- for parse_pass in [1, 2]:
1037
+ for parse_pass in [
1038
+ 1,
1039
+ 2,
1040
+ ]:
1026
1041
  # the first pass will result in all concepts being defined
1027
1042
  # the second will get grains appropriately
1028
1043
  # eg if someone does sum(x)->a, b+c -> z - we don't know if Z is a key to group by or an aggregate
1029
1044
  # until after the first pass, and so don't know the grain of a
1030
- nselect = []
1045
+
1046
+ if parse_pass == 1:
1047
+ grain = Grain.from_concepts(
1048
+ [
1049
+ x.content
1050
+ for x in output.selection
1051
+ if isinstance(x.content, Concept)
1052
+ ],
1053
+ where_clause=output.where_clause,
1054
+ )
1055
+ if parse_pass == 2:
1056
+ grain = Grain.from_concepts(
1057
+ output.output_components, where_clause=output.where_clause
1058
+ )
1059
+ print(
1060
+ f"end pass {parse_pass} grain {grain} from {output.output_components}"
1061
+ )
1062
+ output.grain = grain
1031
1063
  for item in select_items:
1032
1064
  # we don't know the grain of an aggregate at assignment time
1033
1065
  # so rebuild at this point in the tree
@@ -1057,7 +1089,7 @@ class ParseToObjects(Transformer):
1057
1089
  environment=self.environment,
1058
1090
  )
1059
1091
  output.local_concepts[item.content.address] = item.content
1060
- nselect.append(item)
1092
+
1061
1093
  if order_by:
1062
1094
  output.order_by = order_by.with_select_context(
1063
1095
  local_concepts=output.local_concepts,
@@ -1174,7 +1206,7 @@ class ParseToObjects(Transformer):
1174
1206
  if isinstance(args[0], AggregateWrapper):
1175
1207
  left = arbitrary_to_concept(
1176
1208
  args[0],
1177
- namespace=self.environment.namespace,
1209
+ environment=self.environment,
1178
1210
  )
1179
1211
  self.environment.add_concept(left)
1180
1212
  else:
@@ -1182,7 +1214,7 @@ class ParseToObjects(Transformer):
1182
1214
  if isinstance(args[2], AggregateWrapper):
1183
1215
  right = arbitrary_to_concept(
1184
1216
  args[2],
1185
- namespace=self.environment.namespace,
1217
+ environment=self.environment,
1186
1218
  )
1187
1219
  self.environment.add_concept(right)
1188
1220
  else:
@@ -1220,10 +1252,7 @@ class ParseToObjects(Transformer):
1220
1252
  ):
1221
1253
  right = right.content
1222
1254
  if isinstance(right, (Function, FilterItem, WindowItem, AggregateWrapper)):
1223
- right = arbitrary_to_concept(
1224
- right,
1225
- namespace=self.environment.namespace,
1226
- )
1255
+ right = arbitrary_to_concept(right, environment=self.environment)
1227
1256
  self.environment.add_concept(right, meta=meta)
1228
1257
  return SubselectComparison(
1229
1258
  left=args[0],
@@ -1292,10 +1321,7 @@ class ParseToObjects(Transformer):
1292
1321
  elif isinstance(item, WindowType):
1293
1322
  type = item
1294
1323
  else:
1295
- concept = arbitrary_to_concept(
1296
- item,
1297
- namespace=self.environment.namespace,
1298
- )
1324
+ concept = arbitrary_to_concept(item, environment=self.environment)
1299
1325
  self.environment.add_concept(concept, meta=meta)
1300
1326
  assert concept
1301
1327
  return WindowItem(
@@ -1783,6 +1809,30 @@ class ParseToObjects(Transformer):
1783
1809
  @v_args(meta=True)
1784
1810
  def fcast(self, meta, args) -> Function:
1785
1811
  args = process_function_args(args, meta=meta, environment=self.environment)
1812
+ if isinstance(args[0], str):
1813
+ processed: date | datetime | int | float | bool | str
1814
+ if args[1] == DataType.DATE:
1815
+ processed = date.fromisoformat(args[0])
1816
+ elif args[1] == DataType.DATETIME:
1817
+ processed = datetime.fromisoformat(args[0])
1818
+ elif args[1] == DataType.TIMESTAMP:
1819
+ processed = datetime.fromisoformat(args[0])
1820
+ elif args[1] == DataType.INTEGER:
1821
+ processed = int(args[0])
1822
+ elif args[1] == DataType.FLOAT:
1823
+ processed = float(args[0])
1824
+ elif args[1] == DataType.BOOL:
1825
+ processed = args[0].capitalize() == "True"
1826
+ elif args[1] == DataType.STRING:
1827
+ processed = args[0]
1828
+ else:
1829
+ raise SyntaxError(f"Invalid cast type {args[1]}")
1830
+ return Function(
1831
+ operator=FunctionType.CONSTANT,
1832
+ output_datatype=args[1],
1833
+ output_purpose=Purpose.CONSTANT,
1834
+ arguments=[processed],
1835
+ )
1786
1836
  output_datatype = args[1]
1787
1837
  return Function(
1788
1838
  operator=FunctionType.CAST,
trilogy/parsing/render.py CHANGED
@@ -158,7 +158,14 @@ class Renderer:
158
158
 
159
159
  @to_string.register
160
160
  def _(self, arg: "Grain"):
161
- components = ",".join(self.to_string(x) for x in arg.components)
161
+ final = []
162
+ for comp in arg.components:
163
+ if comp.startswith(DEFAULT_NAMESPACE):
164
+ final.append(comp.split(".", 1)[1])
165
+ else:
166
+ final.append(comp)
167
+ final = sorted(final)
168
+ components = ",".join(x for x in final)
162
169
  return f"grain ({components})"
163
170
 
164
171
  @to_string.register
@@ -193,7 +193,9 @@
193
193
  _math_functions: fmul | fdiv | fadd | fsub | fround | fmod | fabs
194
194
 
195
195
  //generic
196
- fcast: "cast"i "(" expr "as"i data_type ")"
196
+ _fcast_primary: "cast"i "(" expr "as"i data_type ")"
197
+ _fcast_alt: expr "::" data_type
198
+ fcast: _fcast_primary | _fcast_alt
197
199
  concat: ("concat"i "(" (expr ",")* expr ")") | (expr "||" expr)
198
200
  fcoalesce: "coalesce"i "(" (expr ",")* expr ")"
199
201
  fcase_when: "WHEN"i conditional "THEN"i expr