pytrilogy 0.0.2.26__tar.gz → 0.0.2.28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pytrilogy might be problematic. Click here for more details.
- {pytrilogy-0.0.2.26/pytrilogy.egg-info → pytrilogy-0.0.2.28}/PKG-INFO +1 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28/pytrilogy.egg-info}/PKG-INFO +1 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/pytrilogy.egg-info/SOURCES.txt +1 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_environment.py +1 -1
- pytrilogy-0.0.2.28/tests/test_executor.py +8 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_parsing.py +23 -3
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_partial_handling.py +1 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_query_processing.py +1 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_undefined_concept.py +1 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/__init__.py +1 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/graph_models.py +2 -2
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/models.py +146 -92
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/optimizations/inline_datasource.py +4 -4
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/select_merge_node.py +7 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/base_node.py +3 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/merge_node.py +10 -10
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/select_node_v2.py +6 -2
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/utility.py +6 -9
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/query_processor.py +21 -17
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/executor.py +17 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/hooks/query_debugger.py +5 -1
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/parse_engine.py +17 -14
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/render.py +25 -7
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/trilogy.lark +4 -2
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/LICENSE.md +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/README.md +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/pyproject.toml +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/pytrilogy.egg-info/dependency_links.txt +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/pytrilogy.egg-info/entry_points.txt +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/pytrilogy.egg-info/requires.txt +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/pytrilogy.egg-info/top_level.txt +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/setup.cfg +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/setup.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_datatypes.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_declarations.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_derived_concepts.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_discovery_nodes.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_functions.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_imports.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_metadata.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_models.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_multi_join_assignments.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_select.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_show.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_statements.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/tests/test_where_clause.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/compiler.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/constants.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/constants.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/enums.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/env_processor.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/environment_helpers.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/ergonomics.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/exceptions.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/functions.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/internal.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/optimization.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/optimizations/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/optimizations/base_optimization.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/optimizations/inline_constant.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/optimizations/predicate_pushdown.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/concept_strategies_v3.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/graph_utils.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/basic_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/common.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/filter_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/group_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/group_to_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/multiselect_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/node_merge_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/rowset_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/select_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/unnest_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/node_generators/window_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/filter_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/group_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/unnest_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/core/processing/nodes/window_node.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/base.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/bigquery.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/common.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/config.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/duckdb.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/enums.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/postgres.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/presto.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/snowflake.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/dialect/sql_server.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/engine.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/hooks/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/hooks/base_hook.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/hooks/graph_hook.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/metadata/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parser.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/common.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/config.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/exceptions.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/parsing/helpers.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/py.typed +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/scripts/__init__.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/scripts/trilogy.py +0 -0
- {pytrilogy-0.0.2.26 → pytrilogy-0.0.2.28}/trilogy/utility.py +0 -0
|
@@ -178,7 +178,7 @@ select
|
|
|
178
178
|
)
|
|
179
179
|
|
|
180
180
|
for name in ["name_alphabetical", "name_alphabetical_2"]:
|
|
181
|
-
assert name in env.concepts
|
|
181
|
+
assert f"local.{name}" in env.concepts
|
|
182
182
|
assert env.concepts[name].purpose == Purpose.PROPERTY
|
|
183
183
|
assert env.concepts[name].keys == (env.concepts["id"],)
|
|
184
184
|
|
|
@@ -197,7 +197,6 @@ select
|
|
|
197
197
|
)
|
|
198
198
|
|
|
199
199
|
for name in ["join_id"]:
|
|
200
|
-
assert name in env.concepts
|
|
201
200
|
assert env.concepts[name].purpose == Purpose.PROPERTY
|
|
202
201
|
assert env.concepts[name].keys == (
|
|
203
202
|
env.concepts["id"],
|
|
@@ -225,7 +224,6 @@ select
|
|
|
225
224
|
)
|
|
226
225
|
# assert output_purpose == Purpose.METRIC
|
|
227
226
|
for name in ["test_name_count"]:
|
|
228
|
-
assert name in env.concepts
|
|
229
227
|
assert env.concepts[name].purpose == Purpose.METRIC
|
|
230
228
|
|
|
231
229
|
|
|
@@ -491,6 +489,28 @@ select x;
|
|
|
491
489
|
assert "abcdef as test" in results, results
|
|
492
490
|
|
|
493
491
|
|
|
492
|
+
def test_datasource_where_equivalent():
|
|
493
|
+
|
|
494
|
+
text = """
|
|
495
|
+
key x int;
|
|
496
|
+
key y int;
|
|
497
|
+
|
|
498
|
+
datasource test (
|
|
499
|
+
x:x,
|
|
500
|
+
y:~y)
|
|
501
|
+
grain(x)
|
|
502
|
+
complete where y > 10
|
|
503
|
+
address `abc:def`
|
|
504
|
+
;
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
"""
|
|
508
|
+
env, parsed = parse_text(text)
|
|
509
|
+
|
|
510
|
+
ds = parsed[-1]
|
|
511
|
+
assert ds.non_partial_for.conditional.right == 10
|
|
512
|
+
|
|
513
|
+
|
|
494
514
|
def test_filter_concise():
|
|
495
515
|
|
|
496
516
|
text = """
|
|
@@ -134,7 +134,7 @@ def test_query_aggregation(test_environment, test_environment_graph):
|
|
|
134
134
|
environment=test_environment, graph=test_environment_graph, statement=select
|
|
135
135
|
)
|
|
136
136
|
|
|
137
|
-
assert {datasource.identifier} == {"
|
|
137
|
+
assert {datasource.identifier} == {"revenue_at_local_order_id_at_abstract"}
|
|
138
138
|
check = datasource
|
|
139
139
|
assert len(check.input_concepts) == 2
|
|
140
140
|
assert check.input_concepts[0].name == "revenue"
|
|
@@ -32,6 +32,6 @@ def test_undefined_concept_dict():
|
|
|
32
32
|
try:
|
|
33
33
|
env["orid"]
|
|
34
34
|
except UndefinedConceptException as e:
|
|
35
|
-
assert e.suggestions == ["order_id"]
|
|
35
|
+
assert e.suggestions == ["local.order_id"]
|
|
36
36
|
assert "suggestions" in e.message.lower()
|
|
37
37
|
assert "order_id" in e.message.lower()
|
|
@@ -6,7 +6,7 @@ from trilogy.core.models import Concept, Datasource
|
|
|
6
6
|
def concept_to_node(input: Concept) -> str:
|
|
7
7
|
# if input.purpose == Purpose.METRIC:
|
|
8
8
|
# return f"c~{input.namespace}.{input.name}@{input.grain}"
|
|
9
|
-
return f"c~{input.
|
|
9
|
+
return f"c~{input.address}@{input.grain}"
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
def datasource_to_node(input: Datasource) -> str:
|
|
@@ -14,7 +14,7 @@ def datasource_to_node(input: Datasource) -> str:
|
|
|
14
14
|
# return "ds~join~" + ",".join(
|
|
15
15
|
# [datasource_to_node(sub) for sub in input.datasources]
|
|
16
16
|
# )
|
|
17
|
-
return f"ds~{input.
|
|
17
|
+
return f"ds~{input.identifier}"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class ReferenceGraph(nx.DiGraph):
|
|
@@ -606,6 +606,8 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
|
|
|
606
606
|
return self.grain.components_copy if self.grain else []
|
|
607
607
|
|
|
608
608
|
def with_namespace(self, namespace: str) -> "Concept":
|
|
609
|
+
if namespace == self.namespace:
|
|
610
|
+
return self
|
|
609
611
|
return self.__class__(
|
|
610
612
|
name=self.name,
|
|
611
613
|
datatype=self.datatype,
|
|
@@ -1719,7 +1721,7 @@ class SelectStatement(HasUUID, Mergeable, Namespaced, SelectTypeMixin, BaseModel
|
|
|
1719
1721
|
def to_datasource(
|
|
1720
1722
|
self,
|
|
1721
1723
|
namespace: str,
|
|
1722
|
-
|
|
1724
|
+
name: str,
|
|
1723
1725
|
address: Address,
|
|
1724
1726
|
grain: Grain | None = None,
|
|
1725
1727
|
) -> Datasource:
|
|
@@ -1753,7 +1755,7 @@ class SelectStatement(HasUUID, Mergeable, Namespaced, SelectTypeMixin, BaseModel
|
|
|
1753
1755
|
condition = self.having_clause.conditional
|
|
1754
1756
|
|
|
1755
1757
|
new_datasource = Datasource(
|
|
1756
|
-
|
|
1758
|
+
name=name,
|
|
1757
1759
|
address=address,
|
|
1758
1760
|
grain=grain or self.grain,
|
|
1759
1761
|
columns=columns,
|
|
@@ -2059,7 +2061,7 @@ class MergeStatementV2(HasUUID, Namespaced, BaseModel):
|
|
|
2059
2061
|
|
|
2060
2062
|
|
|
2061
2063
|
class Datasource(HasUUID, Namespaced, BaseModel):
|
|
2062
|
-
|
|
2064
|
+
name: str
|
|
2063
2065
|
columns: List[ColumnAssignment]
|
|
2064
2066
|
address: Union[Address, str]
|
|
2065
2067
|
grain: Grain = Field(
|
|
@@ -2094,10 +2096,14 @@ class Datasource(HasUUID, Namespaced, BaseModel):
|
|
|
2094
2096
|
self.add_column(target, original[0].alias, modifiers)
|
|
2095
2097
|
|
|
2096
2098
|
@property
|
|
2097
|
-
def
|
|
2099
|
+
def identifier(self) -> str:
|
|
2098
2100
|
if not self.namespace or self.namespace == DEFAULT_NAMESPACE:
|
|
2099
|
-
return self.
|
|
2100
|
-
return f"{self.namespace}.{self.
|
|
2101
|
+
return self.name
|
|
2102
|
+
return f"{self.namespace}.{self.name}"
|
|
2103
|
+
|
|
2104
|
+
@property
|
|
2105
|
+
def safe_identifier(self) -> str:
|
|
2106
|
+
return self.identifier.replace(".", "_")
|
|
2101
2107
|
|
|
2102
2108
|
@property
|
|
2103
2109
|
def condition(self):
|
|
@@ -2166,13 +2172,13 @@ class Datasource(HasUUID, Namespaced, BaseModel):
|
|
|
2166
2172
|
return self
|
|
2167
2173
|
|
|
2168
2174
|
def __repr__(self):
|
|
2169
|
-
return f"Datasource<{self.
|
|
2175
|
+
return f"Datasource<{self.identifier}@<{self.grain}>"
|
|
2170
2176
|
|
|
2171
2177
|
def __str__(self):
|
|
2172
2178
|
return self.__repr__()
|
|
2173
2179
|
|
|
2174
2180
|
def __hash__(self):
|
|
2175
|
-
return self.
|
|
2181
|
+
return self.identifier.__hash__()
|
|
2176
2182
|
|
|
2177
2183
|
def with_namespace(self, namespace: str):
|
|
2178
2184
|
new_namespace = (
|
|
@@ -2181,7 +2187,7 @@ class Datasource(HasUUID, Namespaced, BaseModel):
|
|
|
2181
2187
|
else namespace
|
|
2182
2188
|
)
|
|
2183
2189
|
return Datasource(
|
|
2184
|
-
|
|
2190
|
+
name=self.name,
|
|
2185
2191
|
namespace=new_namespace,
|
|
2186
2192
|
grain=self.grain.with_namespace(namespace),
|
|
2187
2193
|
address=self.address,
|
|
@@ -2231,19 +2237,6 @@ class Datasource(HasUUID, Namespaced, BaseModel):
|
|
|
2231
2237
|
f" {existing}."
|
|
2232
2238
|
)
|
|
2233
2239
|
|
|
2234
|
-
@property
|
|
2235
|
-
def name(self) -> str:
|
|
2236
|
-
return self.identifier
|
|
2237
|
-
# TODO: namespace all references
|
|
2238
|
-
# return f'{self.namespace}_{self.identifier}'
|
|
2239
|
-
|
|
2240
|
-
@property
|
|
2241
|
-
def full_name(self) -> str:
|
|
2242
|
-
if not self.namespace:
|
|
2243
|
-
return self.identifier
|
|
2244
|
-
namespace = self.namespace.replace(".", "_") if self.namespace else ""
|
|
2245
|
-
return f"{namespace}_{self.identifier}"
|
|
2246
|
-
|
|
2247
2240
|
@property
|
|
2248
2241
|
def safe_location(self) -> str:
|
|
2249
2242
|
if isinstance(self.address, Address):
|
|
@@ -2298,7 +2291,7 @@ class BaseJoin(BaseModel):
|
|
|
2298
2291
|
super().__init__(**data)
|
|
2299
2292
|
if (
|
|
2300
2293
|
self.left_datasource
|
|
2301
|
-
and self.left_datasource.
|
|
2294
|
+
and self.left_datasource.identifier == self.right_datasource.identifier
|
|
2302
2295
|
):
|
|
2303
2296
|
raise SyntaxError(
|
|
2304
2297
|
f"Cannot join a dataself to itself, joining {self.left_datasource} and"
|
|
@@ -2410,6 +2403,10 @@ class QueryDatasource(BaseModel):
|
|
|
2410
2403
|
def __repr__(self):
|
|
2411
2404
|
return f"{self.identifier}@<{self.grain}>"
|
|
2412
2405
|
|
|
2406
|
+
@property
|
|
2407
|
+
def safe_identifier(self):
|
|
2408
|
+
return self.identifier.replace(".", "_")
|
|
2409
|
+
|
|
2413
2410
|
@property
|
|
2414
2411
|
def non_partial_concept_addresses(self) -> List[str]:
|
|
2415
2412
|
return [
|
|
@@ -2474,10 +2471,6 @@ class QueryDatasource(BaseModel):
|
|
|
2474
2471
|
def name(self):
|
|
2475
2472
|
return self.identifier
|
|
2476
2473
|
|
|
2477
|
-
@property
|
|
2478
|
-
def full_name(self):
|
|
2479
|
-
return self.identifier
|
|
2480
|
-
|
|
2481
2474
|
@property
|
|
2482
2475
|
def group_required(self) -> bool:
|
|
2483
2476
|
if self.force_group is True:
|
|
@@ -2524,10 +2517,12 @@ class QueryDatasource(BaseModel):
|
|
|
2524
2517
|
merged_datasources = {}
|
|
2525
2518
|
|
|
2526
2519
|
for ds in [*self.datasources, *other.datasources]:
|
|
2527
|
-
if ds.
|
|
2528
|
-
merged_datasources[ds.
|
|
2520
|
+
if ds.safe_identifier in merged_datasources:
|
|
2521
|
+
merged_datasources[ds.safe_identifier] = (
|
|
2522
|
+
merged_datasources[ds.safe_identifier] + ds
|
|
2523
|
+
)
|
|
2529
2524
|
else:
|
|
2530
|
-
merged_datasources[ds.
|
|
2525
|
+
merged_datasources[ds.safe_identifier] = ds
|
|
2531
2526
|
|
|
2532
2527
|
final_source_map = defaultdict(set)
|
|
2533
2528
|
for key in self.source_map:
|
|
@@ -2538,7 +2533,9 @@ class QueryDatasource(BaseModel):
|
|
|
2538
2533
|
if key not in final_source_map:
|
|
2539
2534
|
final_source_map[key] = other.source_map[key]
|
|
2540
2535
|
for k, v in final_source_map.items():
|
|
2541
|
-
final_source_map[k] = set(
|
|
2536
|
+
final_source_map[k] = set(
|
|
2537
|
+
merged_datasources[x.safe_identifier] for x in list(v)
|
|
2538
|
+
)
|
|
2542
2539
|
self_hidden = self.hidden_concepts or []
|
|
2543
2540
|
other_hidden = other.hidden_concepts or []
|
|
2544
2541
|
hidden = [x for x in self_hidden if x.address in other_hidden]
|
|
@@ -2578,7 +2575,7 @@ class QueryDatasource(BaseModel):
|
|
|
2578
2575
|
)
|
|
2579
2576
|
# partial = "_".join([str(c.address).replace(".", "_") for c in self.partial_concepts])
|
|
2580
2577
|
return (
|
|
2581
|
-
"_join_".join([d.
|
|
2578
|
+
"_join_".join([d.identifier for d in self.datasources])
|
|
2582
2579
|
+ (f"_at_{grain}" if grain else "_at_abstract")
|
|
2583
2580
|
+ (f"_filtered_by_{filters}" if filters else "")
|
|
2584
2581
|
# + (f"_partial_{partial}" if partial else "")
|
|
@@ -2594,8 +2591,9 @@ class QueryDatasource(BaseModel):
|
|
|
2594
2591
|
for x in self.datasources:
|
|
2595
2592
|
# query datasources should be referenced by their alias, always
|
|
2596
2593
|
force_alias = isinstance(x, QueryDatasource)
|
|
2594
|
+
#
|
|
2597
2595
|
use_raw_name = isinstance(x, Datasource) and not force_alias
|
|
2598
|
-
if source and x.
|
|
2596
|
+
if source and x.safe_identifier != source:
|
|
2599
2597
|
continue
|
|
2600
2598
|
try:
|
|
2601
2599
|
return x.get_alias(
|
|
@@ -2649,6 +2647,14 @@ class CTE(BaseModel):
|
|
|
2649
2647
|
base_name_override: Optional[str] = None
|
|
2650
2648
|
base_alias_override: Optional[str] = None
|
|
2651
2649
|
|
|
2650
|
+
@property
|
|
2651
|
+
def identifier(self):
|
|
2652
|
+
return self.name
|
|
2653
|
+
|
|
2654
|
+
@property
|
|
2655
|
+
def safe_identifier(self):
|
|
2656
|
+
return self.name
|
|
2657
|
+
|
|
2652
2658
|
@computed_field # type: ignore
|
|
2653
2659
|
@property
|
|
2654
2660
|
def output_lcl(self) -> LooseConceptList:
|
|
@@ -2746,7 +2752,7 @@ class CTE(BaseModel):
|
|
|
2746
2752
|
return False
|
|
2747
2753
|
if any(
|
|
2748
2754
|
[
|
|
2749
|
-
x.
|
|
2755
|
+
x.safe_identifier == ds_being_inlined.safe_identifier
|
|
2750
2756
|
for x in self.source.datasources
|
|
2751
2757
|
]
|
|
2752
2758
|
):
|
|
@@ -2757,39 +2763,49 @@ class CTE(BaseModel):
|
|
|
2757
2763
|
*[
|
|
2758
2764
|
x
|
|
2759
2765
|
for x in self.source.datasources
|
|
2760
|
-
if x.
|
|
2766
|
+
if x.safe_identifier != qds_being_inlined.safe_identifier
|
|
2761
2767
|
],
|
|
2762
2768
|
]
|
|
2763
2769
|
# need to identify this before updating joins
|
|
2764
2770
|
if self.base_name == parent.name:
|
|
2765
2771
|
self.base_name_override = ds_being_inlined.safe_location
|
|
2766
|
-
self.base_alias_override = ds_being_inlined.
|
|
2772
|
+
self.base_alias_override = ds_being_inlined.safe_identifier
|
|
2767
2773
|
|
|
2768
2774
|
for join in self.joins:
|
|
2769
2775
|
if isinstance(join, InstantiatedUnnestJoin):
|
|
2770
2776
|
continue
|
|
2771
|
-
if
|
|
2777
|
+
if (
|
|
2778
|
+
join.left_cte
|
|
2779
|
+
and join.left_cte.safe_identifier == parent.safe_identifier
|
|
2780
|
+
):
|
|
2772
2781
|
join.inline_cte(parent)
|
|
2773
2782
|
if join.joinkey_pairs:
|
|
2774
2783
|
for pair in join.joinkey_pairs:
|
|
2775
|
-
if pair.cte and pair.cte.
|
|
2784
|
+
if pair.cte and pair.cte.safe_identifier == parent.safe_identifier:
|
|
2776
2785
|
join.inline_cte(parent)
|
|
2777
|
-
if join.right_cte.
|
|
2786
|
+
if join.right_cte.safe_identifier == parent.safe_identifier:
|
|
2778
2787
|
join.inline_cte(parent)
|
|
2779
2788
|
for k, v in self.source_map.items():
|
|
2780
2789
|
if isinstance(v, list):
|
|
2781
2790
|
self.source_map[k] = [
|
|
2782
|
-
|
|
2791
|
+
(
|
|
2792
|
+
ds_being_inlined.safe_identifier
|
|
2793
|
+
if x == parent.safe_identifier
|
|
2794
|
+
else x
|
|
2795
|
+
)
|
|
2796
|
+
for x in v
|
|
2783
2797
|
]
|
|
2784
|
-
elif v == parent.
|
|
2785
|
-
self.source_map[k] = [ds_being_inlined.
|
|
2798
|
+
elif v == parent.safe_identifier:
|
|
2799
|
+
self.source_map[k] = [ds_being_inlined.safe_identifier]
|
|
2786
2800
|
|
|
2787
2801
|
# zip in any required values for lookups
|
|
2788
2802
|
for k in ds_being_inlined.output_lcl.addresses:
|
|
2789
2803
|
if k in self.source_map and self.source_map[k]:
|
|
2790
2804
|
continue
|
|
2791
|
-
self.source_map[k] = [ds_being_inlined.
|
|
2792
|
-
self.parent_ctes = [
|
|
2805
|
+
self.source_map[k] = [ds_being_inlined.safe_identifier]
|
|
2806
|
+
self.parent_ctes = [
|
|
2807
|
+
x for x in self.parent_ctes if x.safe_identifier != parent.safe_identifier
|
|
2808
|
+
]
|
|
2793
2809
|
if force_group:
|
|
2794
2810
|
self.group_to_grain = True
|
|
2795
2811
|
return True
|
|
@@ -3006,28 +3022,22 @@ class Join(BaseModel):
|
|
|
3006
3022
|
def inline_cte(self, cte: CTE):
|
|
3007
3023
|
self.inlined_ctes.add(cte.name)
|
|
3008
3024
|
|
|
3009
|
-
# @property
|
|
3010
|
-
# def left_name(self) -> str:
|
|
3011
|
-
# if self.left_cte.name in self.inlined_ctes:
|
|
3012
|
-
# return self.left_cte.source.datasources[0].identifier
|
|
3013
|
-
# return self.left_cte.name
|
|
3014
|
-
|
|
3015
3025
|
def get_name(self, cte: CTE):
|
|
3016
|
-
if cte.
|
|
3017
|
-
return cte.source.datasources[0].
|
|
3018
|
-
return cte.
|
|
3026
|
+
if cte.identifier in self.inlined_ctes:
|
|
3027
|
+
return cte.source.datasources[0].safe_identifier
|
|
3028
|
+
return cte.safe_identifier
|
|
3019
3029
|
|
|
3020
3030
|
@property
|
|
3021
3031
|
def right_name(self) -> str:
|
|
3022
|
-
if self.right_cte.
|
|
3023
|
-
return self.right_cte.source.datasources[0].
|
|
3024
|
-
return self.right_cte.
|
|
3032
|
+
if self.right_cte.identifier in self.inlined_ctes:
|
|
3033
|
+
return self.right_cte.source.datasources[0].safe_identifier
|
|
3034
|
+
return self.right_cte.safe_identifier
|
|
3025
3035
|
|
|
3026
3036
|
@property
|
|
3027
3037
|
def right_ref(self) -> str:
|
|
3028
|
-
if self.right_cte.
|
|
3029
|
-
return f"{self.right_cte.source.datasources[0].safe_location} as {self.right_cte.source.datasources[0].
|
|
3030
|
-
return self.right_cte.
|
|
3038
|
+
if self.right_cte.identifier in self.inlined_ctes:
|
|
3039
|
+
return f"{self.right_cte.source.datasources[0].safe_location} as {self.right_cte.source.datasources[0].safe_identifier}"
|
|
3040
|
+
return self.right_cte.safe_identifier
|
|
3031
3041
|
|
|
3032
3042
|
@property
|
|
3033
3043
|
def unique_id(self) -> str:
|
|
@@ -3245,7 +3255,6 @@ class EnvironmentConceptDict(dict):
|
|
|
3245
3255
|
)
|
|
3246
3256
|
self.undefined[key] = undefined
|
|
3247
3257
|
return undefined
|
|
3248
|
-
|
|
3249
3258
|
matches = self._find_similar_concepts(key)
|
|
3250
3259
|
message = f"Undefined concept: {key}."
|
|
3251
3260
|
if matches:
|
|
@@ -3255,8 +3264,15 @@ class EnvironmentConceptDict(dict):
|
|
|
3255
3264
|
raise UndefinedConceptException(f"line: {line_no}: " + message, matches)
|
|
3256
3265
|
raise UndefinedConceptException(message, matches)
|
|
3257
3266
|
|
|
3258
|
-
def _find_similar_concepts(self, concept_name):
|
|
3259
|
-
|
|
3267
|
+
def _find_similar_concepts(self, concept_name: str):
|
|
3268
|
+
def strip_local(input: str):
|
|
3269
|
+
if input.startswith(f"{DEFAULT_NAMESPACE}."):
|
|
3270
|
+
return input[len(DEFAULT_NAMESPACE) + 1 :]
|
|
3271
|
+
return input
|
|
3272
|
+
|
|
3273
|
+
matches = difflib.get_close_matches(
|
|
3274
|
+
strip_local(concept_name), [strip_local(x) for x in self.keys()]
|
|
3275
|
+
)
|
|
3260
3276
|
return matches
|
|
3261
3277
|
|
|
3262
3278
|
def items(self) -> ItemsView[str, Concept]: # type: ignore
|
|
@@ -3306,7 +3322,9 @@ class Environment(BaseModel):
|
|
|
3306
3322
|
] = Field(default_factory=EnvironmentDatasourceDict)
|
|
3307
3323
|
functions: Dict[str, Function] = Field(default_factory=dict)
|
|
3308
3324
|
data_types: Dict[str, DataType] = Field(default_factory=dict)
|
|
3309
|
-
imports: Dict[str, ImportStatement] = Field(
|
|
3325
|
+
imports: Dict[str, list[ImportStatement]] = Field(
|
|
3326
|
+
default_factory=lambda: defaultdict(list)
|
|
3327
|
+
)
|
|
3310
3328
|
namespace: str = DEFAULT_NAMESPACE
|
|
3311
3329
|
working_path: str | Path = Field(default_factory=lambda: os.getcwd())
|
|
3312
3330
|
environment_config: EnvironmentOptions = Field(default_factory=EnvironmentOptions)
|
|
@@ -3315,7 +3333,6 @@ class Environment(BaseModel):
|
|
|
3315
3333
|
|
|
3316
3334
|
materialized_concepts: List[Concept] = Field(default_factory=list)
|
|
3317
3335
|
alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
|
|
3318
|
-
canonical_map: Dict[str, str] = Field(default_factory=dict)
|
|
3319
3336
|
_parse_count: int = 0
|
|
3320
3337
|
|
|
3321
3338
|
@classmethod
|
|
@@ -3420,14 +3437,52 @@ class Environment(BaseModel):
|
|
|
3420
3437
|
f"Assignment to concept '{lookup}' is a duplicate declaration;"
|
|
3421
3438
|
)
|
|
3422
3439
|
|
|
3423
|
-
def add_import(
|
|
3424
|
-
self
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3440
|
+
def add_import(
|
|
3441
|
+
self, alias: str, source: Environment, imp_stm: ImportStatement | None = None
|
|
3442
|
+
):
|
|
3443
|
+
exists = False
|
|
3444
|
+
existing = self.imports[alias]
|
|
3445
|
+
if imp_stm:
|
|
3446
|
+
if any([x.path == imp_stm.path for x in existing]):
|
|
3447
|
+
exists = True
|
|
3448
|
+
|
|
3449
|
+
else:
|
|
3450
|
+
if any([x.path == source.working_path for x in existing]):
|
|
3451
|
+
exists = True
|
|
3452
|
+
imp_stm = ImportStatement(alias=alias, path=Path(source.working_path))
|
|
3453
|
+
|
|
3454
|
+
same_namespace = alias == self.namespace
|
|
3455
|
+
|
|
3456
|
+
if not exists:
|
|
3457
|
+
self.imports[alias].append(imp_stm)
|
|
3458
|
+
|
|
3459
|
+
for k, concept in source.concepts.items():
|
|
3460
|
+
if same_namespace:
|
|
3461
|
+
new = self.add_concept(concept, _ignore_cache=True)
|
|
3462
|
+
else:
|
|
3463
|
+
new = self.add_concept(
|
|
3464
|
+
concept.with_namespace(alias), _ignore_cache=True
|
|
3465
|
+
)
|
|
3466
|
+
|
|
3467
|
+
k = address_with_namespace(k, alias)
|
|
3468
|
+
# set this explicitly, to handle aliasing
|
|
3469
|
+
self.concepts[k] = new
|
|
3470
|
+
|
|
3471
|
+
for _, datasource in source.datasources.items():
|
|
3472
|
+
if same_namespace:
|
|
3473
|
+
self.add_datasource(datasource, _ignore_cache=True)
|
|
3474
|
+
else:
|
|
3475
|
+
self.add_datasource(
|
|
3476
|
+
datasource.with_namespace(alias), _ignore_cache=True
|
|
3477
|
+
)
|
|
3478
|
+
for key, val in source.alias_origin_lookup.items():
|
|
3479
|
+
if same_namespace:
|
|
3480
|
+
self.alias_origin_lookup[key] = val
|
|
3481
|
+
else:
|
|
3482
|
+
self.alias_origin_lookup[address_with_namespace(key, alias)] = (
|
|
3483
|
+
val.with_namespace(alias)
|
|
3484
|
+
)
|
|
3485
|
+
|
|
3431
3486
|
self.gen_concept_list_caches()
|
|
3432
3487
|
return self
|
|
3433
3488
|
|
|
@@ -3438,18 +3493,15 @@ class Environment(BaseModel):
|
|
|
3438
3493
|
apath[-1] = apath[-1] + ".preql"
|
|
3439
3494
|
|
|
3440
3495
|
target: Path = Path(self.working_path, *apath)
|
|
3496
|
+
if alias in self.imports:
|
|
3497
|
+
imports = self.imports[alias]
|
|
3498
|
+
for x in imports:
|
|
3499
|
+
if x.path == target:
|
|
3500
|
+
return imports
|
|
3441
3501
|
if env:
|
|
3442
|
-
self.imports[alias]
|
|
3443
|
-
alias=alias, path=target, environment=env
|
|
3502
|
+
self.imports[alias].append(
|
|
3503
|
+
ImportStatement(alias=alias, path=target, environment=env)
|
|
3444
3504
|
)
|
|
3445
|
-
|
|
3446
|
-
elif alias in self.imports:
|
|
3447
|
-
current = self.imports[alias]
|
|
3448
|
-
env = self.imports[alias].environment
|
|
3449
|
-
if current.path != target:
|
|
3450
|
-
raise ImportError(
|
|
3451
|
-
f"Attempted to import {target} with alias {alias} but {alias} is already imported from {current.path}"
|
|
3452
|
-
)
|
|
3453
3505
|
else:
|
|
3454
3506
|
try:
|
|
3455
3507
|
with open(target, "r", encoding="utf-8") as f:
|
|
@@ -3468,14 +3520,13 @@ class Environment(BaseModel):
|
|
|
3468
3520
|
f"Unable to import file {target.parent}, parsing error: {e}"
|
|
3469
3521
|
)
|
|
3470
3522
|
env = nparser.environment
|
|
3471
|
-
|
|
3472
|
-
|
|
3473
|
-
self.add_concept(concept.with_namespace(alias))
|
|
3523
|
+
for _, concept in env.concepts.items():
|
|
3524
|
+
self.add_concept(concept.with_namespace(alias))
|
|
3474
3525
|
|
|
3475
|
-
|
|
3476
|
-
|
|
3526
|
+
for _, datasource in env.datasources.items():
|
|
3527
|
+
self.add_datasource(datasource.with_namespace(alias))
|
|
3477
3528
|
imps = ImportStatement(alias=alias, path=target, environment=env)
|
|
3478
|
-
self.imports[alias]
|
|
3529
|
+
self.imports[alias].append(imps)
|
|
3479
3530
|
return imps
|
|
3480
3531
|
|
|
3481
3532
|
def parse(
|
|
@@ -3522,8 +3573,6 @@ class Environment(BaseModel):
|
|
|
3522
3573
|
existing = self.validate_concept(concept, meta=meta)
|
|
3523
3574
|
if existing:
|
|
3524
3575
|
concept = existing
|
|
3525
|
-
if concept.namespace == DEFAULT_NAMESPACE:
|
|
3526
|
-
self.concepts[concept.name] = concept
|
|
3527
3576
|
self.concepts[concept.address] = concept
|
|
3528
3577
|
from trilogy.core.environment_helpers import generate_related_concepts
|
|
3529
3578
|
|
|
@@ -3538,8 +3587,14 @@ class Environment(BaseModel):
|
|
|
3538
3587
|
meta: Meta | None = None,
|
|
3539
3588
|
_ignore_cache: bool = False,
|
|
3540
3589
|
):
|
|
3541
|
-
self.datasources[datasource.
|
|
3590
|
+
self.datasources[datasource.identifier] = datasource
|
|
3591
|
+
|
|
3592
|
+
eligible_to_promote_roots = datasource.non_partial_for is None
|
|
3593
|
+
# mark this as canonical source
|
|
3542
3594
|
for current_concept in datasource.output_concepts:
|
|
3595
|
+
if not eligible_to_promote_roots:
|
|
3596
|
+
continue
|
|
3597
|
+
|
|
3543
3598
|
current_derivation = current_concept.derivation
|
|
3544
3599
|
# TODO: refine this section;
|
|
3545
3600
|
# too hacky for maintainability
|
|
@@ -3605,7 +3660,6 @@ class Environment(BaseModel):
|
|
|
3605
3660
|
v.pseudonyms.add(source.address)
|
|
3606
3661
|
if v.address == source.address:
|
|
3607
3662
|
replacements[k] = target
|
|
3608
|
-
self.canonical_map[k] = target.address
|
|
3609
3663
|
v.pseudonyms.add(target.address)
|
|
3610
3664
|
# we need to update keys and grains of all concepts
|
|
3611
3665
|
else:
|
|
@@ -63,14 +63,14 @@ class InlineDatasource(OptimizationRule):
|
|
|
63
63
|
for replaceable in to_inline:
|
|
64
64
|
if replaceable.name not in self.candidates[cte.name]:
|
|
65
65
|
self.candidates[cte.name].add(replaceable.name)
|
|
66
|
-
self.count[replaceable.source.
|
|
66
|
+
self.count[replaceable.source.identifier] += 1
|
|
67
67
|
return True
|
|
68
68
|
if (
|
|
69
|
-
self.count[replaceable.source.
|
|
69
|
+
self.count[replaceable.source.identifier]
|
|
70
70
|
> CONFIG.optimizations.constant_inline_cutoff
|
|
71
71
|
):
|
|
72
72
|
self.log(
|
|
73
|
-
f"Skipping inlining raw datasource {replaceable.source.
|
|
73
|
+
f"Skipping inlining raw datasource {replaceable.source.identifier} ({replaceable.name}) due to multiple references"
|
|
74
74
|
)
|
|
75
75
|
continue
|
|
76
76
|
if not replaceable.source.datasources[0].grain.issubset(replaceable.grain):
|
|
@@ -81,7 +81,7 @@ class InlineDatasource(OptimizationRule):
|
|
|
81
81
|
result = cte.inline_parent_datasource(replaceable, force_group=force_group)
|
|
82
82
|
if result:
|
|
83
83
|
self.log(
|
|
84
|
-
f"Inlined parent {replaceable.name} with {replaceable.source.
|
|
84
|
+
f"Inlined parent {replaceable.name} with {replaceable.source.identifier}"
|
|
85
85
|
)
|
|
86
86
|
optimized = True
|
|
87
87
|
else:
|