pytrilogy 0.0.2.25__py3-none-any.whl → 0.0.2.27__py3-none-any.whl

trilogy/core/query_processor.py

@@ -5,7 +5,7 @@ from trilogy.core.graph_models import ReferenceGraph
 from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.processing.concept_strategies_v3 import source_query_concepts
 from trilogy.core.enums import BooleanOperator
-from trilogy.constants import CONFIG, DEFAULT_NAMESPACE
+from trilogy.constants import CONFIG
 from trilogy.core.processing.nodes import SelectNode, StrategyNode, History
 from trilogy.core.models import (
     Concept,
@@ -17,7 +17,6 @@ from trilogy.core.models import (
     CTE,
     Join,
     UnnestJoin,
-    JoinKey,
     MaterializedDataset,
     ProcessedQuery,
     ProcessedQueryPersist,
@@ -28,6 +27,7 @@ from trilogy.core.models import (
     Conditional,
     ProcessedCopyStatement,
     CopyStatement,
+    CTEConceptPair,
 )
 
 from trilogy.utility import unique
@@ -52,44 +52,53 @@ def base_join_to_join(
             concept_to_unnest=base_join.parent.concept_arguments[0],
             alias=base_join.alias,
         )
-    if base_join.left_datasource.identifier == base_join.right_datasource.identifier:
-        raise ValueError(f"Joining on same datasource {base_join}")
-    left_ctes = [
-        cte
-        for cte in ctes
-        if (cte.source.full_name == base_join.left_datasource.full_name)
-    ]
-    if not left_ctes:
-        left_ctes = [
-            cte
-            for cte in ctes
-            if (
-                cte.source.datasources[0].full_name
-                == base_join.left_datasource.full_name
+
+    def get_datasource_cte(datasource: Datasource | QueryDatasource) -> CTE:
+        for cte in ctes:
+            if cte.source.identifier == datasource.identifier:
+                return cte
+        for cte in ctes:
+            if cte.source.datasources[0].identifier == datasource.identifier:
+                return cte
+        raise ValueError(f"Could not find CTE for datasource {datasource.identifier}")
+
+    if base_join.left_datasource is not None:
+        left_cte = get_datasource_cte(base_join.left_datasource)
+    else:
+        # multiple left ctes
+        left_cte = None
+    right_cte = get_datasource_cte(base_join.right_datasource)
+    if base_join.concept_pairs:
+        final_pairs = [
+            CTEConceptPair(
+                left=pair.left,
+                right=pair.right,
+                existing_datasource=pair.existing_datasource,
+                modifiers=pair.modifiers,
+                cte=get_datasource_cte(pair.existing_datasource),
             )
+            for pair in base_join.concept_pairs
         ]
-    left_cte = left_ctes[0]
-    right_ctes = [
-        cte
-        for cte in ctes
-        if (cte.source.full_name == base_join.right_datasource.full_name)
-    ]
-    if not right_ctes:
-        right_ctes = [
-            cte
-            for cte in ctes
-            if (
-                cte.source.datasources[0].full_name
-                == base_join.right_datasource.full_name
+    elif base_join.concepts and base_join.left_datasource:
+        final_pairs = [
+            CTEConceptPair(
+                left=concept,
+                right=concept,
+                existing_datasource=base_join.left_datasource,
+                modifiers=[],
+                cte=get_datasource_cte(
+                    base_join.left_datasource,
+                ),
             )
+            for concept in base_join.concepts
         ]
-    right_cte = right_ctes[0]
+    else:
+        final_pairs = []
     return Join(
         left_cte=left_cte,
         right_cte=right_cte,
-        joinkeys=[JoinKey(concept=concept) for concept in base_join.concepts],
         jointype=base_join.join_type,
-        joinkey_pairs=base_join.concept_pairs if base_join.concept_pairs else None,
+        joinkey_pairs=final_pairs,
     )
 
 
@@ -100,7 +109,7 @@ def generate_source_map(
     # now populate anything derived in this level
     for qdk, qdv in query_datasource.source_map.items():
         unnest = [x for x in qdv if isinstance(x, UnnestJoin)]
-        for x in unnest:
+        for _ in unnest:
             source_map[qdk] = []
         if (
             qdk not in source_map
@@ -110,16 +119,18 @@ def generate_source_map(
             source_map[qdk] = []
         basic = [x for x in qdv if isinstance(x, Datasource)]
         for base in basic:
-            source_map[qdk].append(base.name)
+            source_map[qdk].append(base.safe_identifier)
 
         ctes = [x for x in qdv if isinstance(x, QueryDatasource)]
         if ctes:
-            names = set([x.name for x in ctes])
-            matches = [cte for cte in all_new_ctes if cte.source.name in names]
+            names = set([x.safe_identifier for x in ctes])
+            matches = [
+                cte for cte in all_new_ctes if cte.source.safe_identifier in names
+            ]
 
             if not matches and names:
                 raise SyntaxError(
-                    f"Missing parent CTEs for source map; expecting {names}, have {[cte.source.name for cte in all_new_ctes]}"
+                    f"Missing parent CTEs for source map; expecting {names}, have {[cte.source.safe_identifier for cte in all_new_ctes]}"
                 )
             for cte in matches:
                 output_address = [
@@ -128,11 +139,11 @@ def generate_source_map(
                     if x.address not in [z.address for z in cte.partial_concepts]
                 ]
                 if qdk in output_address:
-                    source_map[qdk].append(cte.name)
+                    source_map[qdk].append(cte.safe_identifier)
             # now do a pass that accepts partials
             for cte in matches:
                 if qdk not in source_map:
-                    source_map[qdk] = [cte.name]
+                    source_map[qdk] = [cte.safe_identifier]
         if qdk not in source_map:
             if not qdv:
                 source_map[qdk] = []
@@ -145,8 +156,10 @@ def generate_source_map(
     # as they cannot be referenced in row resolution
     existence_source_map: Dict[str, list[str]] = defaultdict(list)
     for ek, ev in query_datasource.existence_source_map.items():
-        names = set([x.name for x in ev])
-        ematches = [cte.name for cte in all_new_ctes if cte.source.name in names]
+        ids = set([x.safe_identifier for x in ev])
+        ematches = [
+            cte.name for cte in all_new_ctes if cte.source.safe_identifier in ids
+        ]
         existence_source_map[ek] = ematches
     return {
         k: [] if not v else list(set(v)) for k, v in source_map.items()
@@ -195,16 +208,19 @@ def resolve_cte_base_name_and_alias_v2(
     source_map: Dict[str, list[str]],
     raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
-    joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
     if (
         isinstance(source.datasources[0], Datasource)
         and not source.datasources[0].name == CONSTANT_DATASET
     ):
         ds = source.datasources[0]
-        return ds.safe_location, ds.identifier
+        return ds.safe_location, ds.safe_identifier
 
+    joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
     if joins and len(joins) > 0:
-        candidates = [x.left_cte.name for x in joins]
+        candidates = [x.left_cte.name for x in joins if x.left_cte]
+        for join in joins:
+            if join.joinkey_pairs:
+                candidates += [x.cte.name for x in join.joinkey_pairs if x.cte]
         disallowed = [x.right_cte.name for x in joins]
         try:
             cte = [y for y in candidates if y not in disallowed][0]
@@ -213,7 +229,6 @@ def resolve_cte_base_name_and_alias_v2(
             raise SyntaxError(
                 f"Invalid join configuration {candidates} {disallowed} for {name}",
             )
-
     counts: dict[str, int] = defaultdict(lambda: 0)
     output_addresses = [x.address for x in source.output_concepts]
     input_address = [x.address for x in source.input_concepts]
@@ -257,23 +272,20 @@ def datasource_to_ctes(
         # this is required to ensure that constant datasets
         # render properly on initial access; since they have
         # no actual source
-        if source.full_name == DEFAULT_NAMESPACE + "_" + CONSTANT_DATASET:
+        if source.name == CONSTANT_DATASET:
             source_map = {k: [] for k in query_datasource.source_map}
             existence_map = source_map
         else:
             source_map = {
-                k: [] if not v else [source.identifier]
+                k: [] if not v else [source.safe_identifier]
                 for k, v in query_datasource.source_map.items()
             }
             existence_map = source_map
 
-    human_id = generate_cte_name(query_datasource.full_name, name_map)
+    human_id = generate_cte_name(query_datasource.identifier, name_map)
+
+    final_joins = [base_join_to_join(join, parents) for join in query_datasource.joins]
 
-    final_joins = [
-        x
-        for x in [base_join_to_join(join, parents) for join in query_datasource.joins]
-        if x
-    ]
     base_name, base_alias = resolve_cte_base_name_and_alias_v2(
         human_id, query_datasource, source_map, final_joins
     )

trilogy/dialect/base.py

@@ -607,6 +607,7 @@ class BaseDialect:
                     else:
                         having = having + x if having else x
 
+        logger.info(f"{len(final_joins)} joins for cte {cte.name}")
         return CompiledCTE(
             name=cte.name,
             statement=self.SQL_TEMPLATE.render(

trilogy/dialect/common.py

@@ -68,39 +68,18 @@ def render_join(
         if unnest_mode == UnnestMode.CROSS_JOIN_ALIAS:
             return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
         return f"FULL JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
-    left_name = join.left_name
+    # left_name = join.left_name
     right_name = join.right_name
     right_base = join.right_ref
-    base_joinkeys = [
-        null_wrapper(
-            render_join_concept(
-                left_name,
-                quote_character,
-                join.left_cte,
-                key.concept,
-                render_expr_func,
-                join.inlined_ctes,
-            ),
-            render_join_concept(
-                right_name,
-                quote_character,
-                join.right_cte,
-                key.concept,
-                render_expr_func,
-                join.inlined_ctes,
-            ),
-            modifiers=key.concept.modifiers or [],
-        )
-        for key in join.joinkeys
-    ]
+    base_joinkeys = []
     if join.joinkey_pairs:
         base_joinkeys.extend(
             [
                 null_wrapper(
                     render_join_concept(
-                        left_name,
+                        join.get_name(pair.cte),
                         quote_character,
-                        join.left_cte,
+                        pair.cte,
                         pair.left,
                         render_expr_func,
                         join.inlined_ctes,

trilogy/executor.py

@@ -334,7 +334,9 @@ class Executor(object):
             text(command),
         )
 
-    def execute_text(self, command: str) -> List[CursorResult]:
+    def execute_text(
+        self, command: str, non_interactive: bool = False
+    ) -> List[CursorResult]:
         """Run a preql text command"""
         output = []
         # connection = self.engine.connect()
@@ -351,11 +353,18 @@ class Executor(object):
                     )
                 )
                 continue
+            if non_interactive:
+                if not isinstance(
+                    statement, (ProcessedCopyStatement, ProcessedQueryPersist)
+                ):
+                    continue
             output.append(self.execute_query(statement))
         return output
 
-    def execute_file(self, file: str | Path) -> List[CursorResult]:
+    def execute_file(
+        self, file: str | Path, non_interactive: bool = False
+    ) -> List[CursorResult]:
         file = Path(file)
         with open(file, "r") as f:
             command = f.read()
-        return self.execute_text(command)
+        return self.execute_text(command, non_interactive=non_interactive)

trilogy/hooks/query_debugger.py

@@ -77,7 +77,11 @@ def print_recursive_nodes(
             ]
         ]
     for child in input.parents:
-        display += print_recursive_nodes(child, mode=mode, depth=depth + 1)
+        display += print_recursive_nodes(
+            child,
+            mode=mode,
+            depth=depth + 1,
+        )
     return display
 
 

trilogy/parsing/common.py

@@ -292,21 +292,19 @@ def arbitrary_to_concept(
 
     if isinstance(parent, AggregateWrapper):
         if not name:
-            name = (
-                f"_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
-            )
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
         return agg_wrapper_to_concept(parent, namespace, name, metadata, purpose)
     elif isinstance(parent, WindowItem):
         if not name:
-            name = f"_window_{parent.type.value}_{string_to_hash(str(parent))}"
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_window_{parent.type.value}_{string_to_hash(str(parent))}"
         return window_item_to_concept(parent, name, namespace, purpose, metadata)
     elif isinstance(parent, FilterItem):
         if not name:
-            name = f"_filter_{parent.content.name}_{string_to_hash(str(parent))}"
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{parent.content.name}_{string_to_hash(str(parent))}"
         return filter_item_to_concept(parent, name, namespace, purpose, metadata)
     elif isinstance(parent, Function):
         if not name:
-            name = f"_func_{parent.operator.value}_{string_to_hash(str(parent))}"
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
         return function_to_concept(parent, name, namespace)
     elif isinstance(parent, ListWrapper):
         if not name:

trilogy/parsing/parse_engine.py

@@ -123,6 +123,13 @@ from trilogy.parsing.common import (
     arbitrary_to_concept,
     process_function_args,
 )
+from dataclasses import dataclass
+
+
+@dataclass
+class WholeGrainWrapper:
+    where: WhereClause
+
 
 CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
 
@@ -254,8 +261,9 @@ class ParseToObjects(Transformer):
     def IDENTIFIER(self, args) -> str:
         return args.value
 
-    def concept_lit(self, args) -> Concept:
-        return self.environment.concepts.__getitem__(args[0])
+    @v_args(meta=True)
+    def concept_lit(self, meta: Meta, args) -> Concept:
+        return self.environment.concepts.__getitem__(args[0], meta.line)
 
     def ADDRESS(self, args) -> Address:
         return Address(location=args.value, quoted=False)
@@ -565,9 +573,11 @@ class ParseToObjects(Transformer):
         return args
 
     def grain_clause(self, args) -> Grain:
-        #            namespace=self.environment.namespace,
         return Grain(components=[self.environment.concepts[a] for a in args[0]])
 
+    def whole_grain_clause(self, args) -> WholeGrainWrapper:
+        return WholeGrainWrapper(where=args[0])
+
     def MULTILINE_STRING(self, args) -> str:
         return args[3:-3]
 
@@ -581,11 +591,14 @@ class ParseToObjects(Transformer):
         grain: Optional[Grain] = None
         address: Optional[Address] = None
         where: Optional[WhereClause] = None
+        non_partial_for: Optional[WhereClause] = None
         for val in args[1:]:
             if isinstance(val, Address):
                 address = val
             elif isinstance(val, Grain):
                 grain = val
+            elif isinstance(val, WholeGrainWrapper):
+                non_partial_for = val.where
             elif isinstance(val, Query):
                 address = Address(location=f"({val.text})", is_query=True)
             elif isinstance(val, WhereClause):
@@ -595,7 +608,7 @@ class ParseToObjects(Transformer):
                 "Malformed datasource, missing address or query declaration"
             )
         datasource = Datasource(
-            identifier=name,
+            name=name,
             columns=columns,
             # grain will be set by default from args
             # TODO: move to factory
@@ -603,6 +616,7 @@ class ParseToObjects(Transformer):
             address=address,
             namespace=self.environment.namespace,
             where=where,
+            non_partial_for=non_partial_for,
         )
         for column in columns:
             column.concept = column.concept.with_grain(datasource.grain)
@@ -800,20 +814,10 @@ class ParseToObjects(Transformer):
             except Exception as e:
                 raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
-        for _, concept in nparser.environment.concepts.items():
-            self.environment.add_concept(
-                concept.with_namespace(alias), _ignore_cache=True
-            )
-
-        for _, datasource in nparser.environment.datasources.items():
-            self.environment.add_datasource(
-                datasource.with_namespace(alias), _ignore_cache=True
-            )
         imps = ImportStatement(
             alias=alias, path=Path(args[0]), environment=nparser.environment
         )
-        self.environment.imports[alias] = imps
-        self.environment.gen_concept_list_caches()
+        self.environment.add_import(alias, nparser.environment, imps)
         return imps
 
     @v_args(meta=True)
@@ -840,7 +844,7 @@ class ParseToObjects(Transformer):
                 if self.environment.namespace
                 else DEFAULT_NAMESPACE
             ),
-            identifier=identifier,
+            name=identifier,
             address=Address(location=address),
             grain=grain,
         )

trilogy/parsing/render.py

@@ -17,6 +17,7 @@ from trilogy.core.models import (
     SelectItem,
     WhereClause,
     Conditional,
+    SubselectComparison,
     Comparison,
     Environment,
     ConceptDeclarationStatement,
@@ -25,6 +26,7 @@ from trilogy.core.models import (
     WindowItem,
     FilterItem,
     ColumnAssignment,
+    RawColumnExpr,
     CaseElse,
     CaseWhen,
     ImportStatement,
@@ -50,13 +52,16 @@ from collections import defaultdict
 
 
 QUERY_TEMPLATE = Template(
-    """SELECT{%- for select in select_columns %}
-    {{ select }},{% endfor %}{% if where %}
-WHERE
-    {{ where }}{% endif %}{%- if order_by %}
+    """{% if where %}WHERE
+    {{ where }}
+{% endif %}SELECT{%- for select in select_columns %}
+    {{ select }},{% endfor %}{% if having %}
+HAVING
+    {{ having }}
+{% endif %}{%- if order_by %}
 ORDER BY{% for order in order_by %}
-    {{ order }}{% if not loop.last %},{% endif %}
-{% endfor %}{% endif %}{%- if limit is not none %}
+    {{ order }}{% if not loop.last %},{% endif %}{% endfor %}
+{% endif %}{%- if limit is not none %}
 LIMIT {{ limit }}{% endif %};"""
 )
 
@@ -75,6 +80,9 @@ class Renderer:
         metrics = []
         # first, keys
         for concept in arg.concepts.values():
+            if "__preql_internal" in concept.address:
+                continue
+
             # don't render anything that came from an import
             if concept.namespace in arg.imports:
                 continue
@@ -117,10 +125,10 @@ class Renderer:
             for datasource in arg.datasources.values()
             if datasource.namespace == DEFAULT_NAMESPACE
         ]
-        rendered_imports = [
-            self.to_string(import_statement)
-            for import_statement in arg.imports.values()
-        ]
+        rendered_imports = []
+        for _, imports in arg.imports.items():
+            for import_statement in imports:
+                rendered_imports.append(self.to_string(import_statement))
         components = []
         if rendered_imports:
             components.append(rendered_imports)
@@ -128,19 +136,26 @@ class Renderer:
             components.append(rendered_concepts)
         if rendered_datasources:
             components.append(rendered_datasources)
+
         final = "\n\n".join("\n".join(x) for x in components)
         return final
 
     @to_string.register
     def _(self, arg: Datasource):
-        assignments = ",\n\t".join([self.to_string(x) for x in arg.columns])
+        assignments = ",\n    ".join([self.to_string(x) for x in arg.columns])
+        if arg.non_partial_for:
+            non_partial = f"\ncomplete where {self.to_string(arg.non_partial_for)}"
+        else:
+            non_partial = ""
         base = f"""datasource {arg.name} (
     {assignments}
-    ) 
-{self.to_string(arg.grain)} 
+    )
+{self.to_string(arg.grain)}{non_partial}
 {self.to_string(arg.address)}"""
+
         if arg.where:
-            base += f"\n{self.to_string(arg.where)}"
+            base += f"\nwhere {self.to_string(arg.where)}"
+
         base += ";"
         return base
 
@@ -209,7 +224,19 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "ColumnAssignment"):
-        return f"{arg.alias}: {self.to_string(arg.concept)}"
+        if arg.modifiers:
+            modifiers = "".join(
+                [self.to_string(modifier) for modifier in arg.modifiers]
+            )
+        else:
+            modifiers = ""
+        if isinstance(arg.alias, str):
+            return f"{arg.alias}: {modifiers}{self.to_string(arg.concept)}"
+        return f"{self.to_string(arg.alias)}: {modifiers}{self.to_string(arg.concept)}"
+
+    @to_string.register
+    def _(self, arg: "RawColumnExpr"):
+        return f"raw('''{arg.text}''')"
 
     @to_string.register
     def _(self, arg: "ConceptDeclarationStatement"):
@@ -266,6 +293,7 @@ class Renderer:
         return QUERY_TEMPLATE.render(
             select_columns=[self.to_string(c) for c in arg.selection],
             where=self.to_string(arg.where_clause) if arg.where_clause else None,
+            having=self.to_string(arg.having_clause) if arg.having_clause else None,
             order_by=(
                 [self.to_string(c) for c in arg.order_by.items]
                 if arg.order_by
@@ -301,16 +329,23 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: OrderBy):
-        return ",\t".join([self.to_string(c) for c in arg.items])
+        return ",\n".join([self.to_string(c) for c in arg.items])
 
     @to_string.register
     def _(self, arg: "WhereClause"):
-        return f"{self.to_string(arg.conditional)}"
+        base = f"{self.to_string(arg.conditional)}"
+        if base[0] == "(" and base[-1] == ")":
+            return base[1:-1]
+        return base
 
     @to_string.register
     def _(self, arg: "Conditional"):
         return f"({self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)})"
 
+    @to_string.register
+    def _(self, arg: "SubselectComparison"):
+        return f"{self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)}"
+
     @to_string.register
     def _(self, arg: "Comparison"):
         return f"{self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)}"
@@ -319,10 +354,12 @@ class Renderer:
     def _(self, arg: "WindowItem"):
         over = ",".join(self.to_string(c) for c in arg.over)
         order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
+        if over and order:
             return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
+                f"{arg.type.value} {self.to_string(arg.content)} by {order} over {over}"
             )
+        elif over:
+            return f"{arg.type.value} {self.to_string(arg.content)} over {over}"
         return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
 
     @to_string.register
@@ -331,6 +368,8 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "ImportStatement"):
+        if arg.alias == DEFAULT_NAMESPACE:
+            return f"import {arg.path};"
         return f"import {arg.path} as {arg.alias};"
 
     @to_string.register
@@ -343,13 +382,15 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "ConceptTransform"):
-        return f"{self.to_string(arg.function)}->{arg.output.name}"
+        return f"{self.to_string(arg.function)} -> {arg.output.name}"
 
     @to_string.register
     def _(self, arg: "Function"):
         inputs = ",".join(self.to_string(c) for c in arg.arguments)
         if arg.operator == FunctionType.CONSTANT:
             return f"{inputs}"
+        if arg.operator == FunctionType.CAST:
+            return f"CAST({self.to_string(arg.arguments[0])} AS {self.to_string(arg.arguments[1])})"
         if arg.operator == FunctionType.INDEX_ACCESS:
             return f"{self.to_string(arg.arguments[0])}[{self.to_string(arg.arguments[1])}]"
         return f"{arg.operator.value}({inputs})"
@@ -361,7 +402,8 @@ class Renderer:
     @to_string.register
     def _(self, arg: AggregateWrapper):
         if arg.by:
-            return f"{self.to_string(arg.function)} by {self.to_string(arg.by)}"
+            by = ", ".join([self.to_string(x) for x in arg.by])
+            return f"{self.to_string(arg.function)} by {by}"
         return f"{self.to_string(arg.function)}"
 
     @to_string.register

trilogy/parsing/trilogy.lark

@@ -35,8 +35,10 @@
     prop_ident: "<" IDENTIFIER ("," IDENTIFIER )* ","? ">" "." IDENTIFIER
 
     // datasource concepts
-    datasource: "datasource" IDENTIFIER  "("  column_assignment_list ")"  grain_clause? (address | query) where?
-    
+    datasource: "datasource" IDENTIFIER  "("  column_assignment_list ")"  grain_clause? whole_grain_clause? (address | query) where?
+   
+    whole_grain_clause: "complete" where
+
     grain_clause: "grain" "(" column_list ")"
     
     address: "address" (QUOTED_ADDRESS | ADDRESS)
@@ -289,8 +291,8 @@
 
     // base language constructs
     concept_lit: IDENTIFIER
-    IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.\-]*/
-    QUOTED_ADDRESS:  /`[a-zA-Z\_][a-zA-Z0-9\_\-\.\-\*\:]*`/
+    IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.]*/
+    QUOTED_ADDRESS:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:]*`/
     ADDRESS: IDENTIFIER
 
     MULTILINE_STRING: /\'{3}(.*?)\'{3}/s