jupyter-duckdb 1.2.0.3__py3-none-any.whl → 1.2.0.5__py3-none-any.whl

duckdb_kernel/kernel.py

@@ -12,8 +12,9 @@ from ipykernel.kernelbase import Kernel
 from .db import Connection, DatabaseError, Table
 from .db.error import *
 from .magics import *
-from .parser import RAParser, DCParser
+from .parser import RAParser, DCParser, ParserError
 from .util.ResultSetComparator import ResultSetComparator
+from .util.TestError import TestError
 from .util.formatting import row_count, rows_table, wrap_image
 from .visualization import *
 
@@ -45,8 +46,12 @@ class DuckDBKernel(Kernel):
             MagicCommand('query_max_rows').arg('count').on(self._query_max_rows_magic),
             MagicCommand('schema').flag('td').opt('only').on(self._schema_magic),
             MagicCommand('store').arg('file').flag('noheader').result(True).on(self._store_magic),
-            MagicCommand('ra').flag('analyze').code(True).on(self._ra_magic),
-            MagicCommand('dc').code(True).on(self._dc_magic)
+            MagicCommand('ra').disable('dc', 'auto_parser').flag('analyze').code(True).on(self._ra_magic),
+            MagicCommand('all_ra').arg('value', '1').on(self._all_ra_magic),
+            MagicCommand('dc').disable('ra', 'auto_parser').code(True).on(self._dc_magic),
+            MagicCommand('all_dc').arg('value', '1').on(self._all_dc_magic),
+            MagicCommand('auto_parser').disable('ra', 'dc').code(True).on(self._auto_parser_magic),
+            MagicCommand('guess_parser').arg('value', '1').on(self._guess_parser_magic),
         )
 
         # create placeholders for database and tests
@@ -139,6 +144,7 @@ class DuckDBKernel(Kernel):
             return False
 
     def _execute_stmt(self, query: str, silent: bool,
+                      column_name_mapping: Dict[str, str],
                       max_rows: Optional[int]) -> Tuple[Optional[List[str]], Optional[List[List]]]:
         if self._db is None:
             raise AssertionError('load a database first')
@@ -168,7 +174,8 @@ class DuckDBKernel(Kernel):
         else:
             if columns is not None:
                 # table header
-                table_header = ''.join(f'<th>{c}</th>' for c in columns)
+                mapped_columns = (column_name_mapping.get(c, c) for c in columns)
+                table_header = ''.join(f'<th>{c}</th>' for c in mapped_columns)
 
                 # table data
                 if max_rows is not None and len(rows) > max_rows:
@@ -302,12 +309,23 @@ class DuckDBKernel(Kernel):
         result_columns = [col.rsplit('.', 1)[-1] for col in result_columns]
 
         # extract data for test
-        data = self._tests[name]
+        test_data = self._tests[name]
 
+        # execute test
+        try:
+            self._execute_test(test_data, result_columns, result)
+            self.print_data(wrap_image(True))
+        except TestError as e:
+            self.print_data(wrap_image(False, e.message))
+            if os.environ.get('DUCKDB_TESTS_RAISE_EXCEPTION', 'false').lower() in ('true', '1'):
+                raise e
+
+    @staticmethod
+    def _execute_test(test_data: Dict, result_columns: List[str], result: List[List]):
         # check columns if required
-        if isinstance(data['equals'], dict):
+        if isinstance(test_data['equals'], dict):
             # get column order
-            data_columns = list(data['equals'].keys())
+            data_columns = list(test_data['equals'].keys())
             column_order = []
 
             for dc in data_columns:
@@ -318,39 +336,37 @@ class DuckDBKernel(Kernel):
                         found += 1
 
                 if found == 0:
-                    return self.print_data(wrap_image(False, f'attribute {dc} missing'))
+                    raise TestError(f'attribute {dc} missing')
                 if found >= 2:
-                    return self.print_data(wrap_image(False, f'ambiguous attribute {dc}'))
+                    raise TestError(f'ambiguous attribute {dc}')
 
             # abort if columns from result are unnecessary
             for i, rc in enumerate(result_columns):
                 if i not in column_order:
-                    return self.print_data(wrap_image(False, f'unnecessary attribute {rc}'))
+                    raise TestError(f'unnecessary attribute {rc}')
 
             # reorder columns and transform to list of lists
             sorted_columns = [x for _, x in sorted(zip(column_order, data_columns))]
             rows = []
 
-            for row in zip(*(data['equals'][col] for col in sorted_columns)):
+            for row in zip(*(test_data['equals'][col] for col in sorted_columns)):
                 rows.append(row)
 
         else:
-            rows = data['equals']
+            rows = test_data['equals']
 
         # ordered test
-        if data['ordered']:
+        if test_data['ordered']:
             # calculate diff
             rsc = ResultSetComparator(result, rows)
 
             missing = len(rsc.ordered_right_only)
             if missing > 0:
-                return self.print_data(wrap_image(False, f'{row_count(missing)} missing'))
+                raise TestError(f'{row_count(missing)} missing')
 
             missing = len(rsc.ordered_left_only)
             if missing > 0:
-                return self.print_data(wrap_image(False, f'{row_count(missing)} more than required'))
-
-            return self.print_data(wrap_image(True))
+                raise TestError(f'{row_count(missing)} more than required')
 
         # unordered test
         else:
@@ -362,13 +378,11 @@ class DuckDBKernel(Kernel):
 
             # print result
             if below > 0 and above > 0:
-                self.print_data(wrap_image(False, f'{row_count(below)} missing, {row_count(above)} unnecessary'))
+                raise TestError(f'{row_count(below)} missing, {row_count(above)} unnecessary')
             elif below > 0:
-                self.print_data(wrap_image(False, f'{row_count(below)} missing'))
+                raise TestError(f'{row_count(below)} missing')
             elif above > 0:
-                self.print_data(wrap_image(False, f'{row_count(above)} unnecessary'))
-            else:
-                self.print_data(wrap_image(True))
+                raise TestError(f'{row_count(above)} unnecessary')
 
     def _all_magic(self, silent: bool):
         return {
@@ -486,6 +500,15 @@ class DuckDBKernel(Kernel):
             'generated_code': sql
         }
 
+    def _all_ra_magic(self, silent: bool, value: str):
+        if value.lower() in ('1', 'on', 'true'):
+            self._magics['ra'].default(True)
+            self._magics['dc'].default(False)
+
+            self.print('All further cells are interpreted as %RA.\n')
+        else:
+            self._magics['ra'].default(False)
+
     def _dc_magic(self, silent: bool, code: str):
         if self._db is None:
             raise AssertionError('load a database first')
@@ -503,12 +526,42 @@ class DuckDBKernel(Kernel):
         root_node = DCParser.parse_query(code)
 
         # generate sql
-        sql = root_node.to_sql(tables)
+        sql, cnm = root_node.to_sql_with_renamed_columns(tables)
 
         return {
-            'generated_code': sql
+            'generated_code': sql,
+            'column_name_mapping': cnm
         }
 
+    def _all_dc_magic(self, silent: bool, value: str):
+        if value.lower() in ('1', 'on', 'true'):
+            self._magics['dc'].default(True)
+            self._magics['ra'].default(False)
+
+            self.print('All further cells are interpreted as %DC.\n')
+        else:
+            self._magics['dc'].default(False)
+
+    def _guess_parser_magic(self, silent: bool, value: str):
+        if value.lower() in ('1', 'on', 'true'):
+            self._magics['auto_parser'].default(True)
+            self.print('The correct parser is guessed for each subsequently executed cell.\n')
+        else:
+            self._magics['auto_parser'].default(False)
+
+    def _auto_parser_magic(self, silent: bool, code: str):
+        try:
+            return self._ra_magic(silent, code, analyze=False)
+        except ParserError as e:
+            if e.depth > 0:
+                raise e
+
+        try:
+            return self._dc_magic(silent, code)
+        except ParserError as e:
+            if e.depth > 0:
+                raise e
+
     # jupyter related functions
     def do_execute(self, code: str, silent: bool,
                    store_history: bool = True, user_expressions: dict = None, allow_stdin: bool = False,
@@ -530,6 +583,10 @@ class DuckDBKernel(Kernel):
                 clean_code = execution_args['generated_code']
                 del execution_args['generated_code']
 
+            # set default column name mapping if none provided
+            if 'column_name_mapping' not in execution_args:
+                execution_args['column_name_mapping'] = {}
+
             # execute statement if needed
             if clean_code.strip():
                 cols, rows = self._execute_stmt(clean_code, silent, **execution_args)

duckdb_kernel/magics/MagicCommand.py

@@ -1,27 +1,28 @@
-from typing import Any, List, Tuple, Callable, Dict
+from typing import Any, List, Tuple, Callable, Dict, Set
 
 
 class MagicCommand:
-    _ARG = '''([^ ]+?|'.+?'|".+?")'''
+    _ARG = '''([^ ]+?|'.+?'|".+?")?'''
 
     def __init__(self, *names: str):
-        self._names: Tuple[str] = names
+        self._names: Tuple[str, ...] = names
 
-        self._arguments: List[Tuple[str, str]] = []
+        self._arguments: List[Tuple[str, Any, str]] = []
         self._flags: List[Tuple[str, str]] = []
         self._optionals: List[Tuple[str, Any, str]] = []
-
+        self._disables: Set[str] = set()
         self._code: bool = False
         self._result: bool = False
+        self._default: bool = False
 
         self._on: List[Callable] = []
 
     @property
-    def names(self) -> Tuple[str]:
+    def names(self) -> Tuple[str, ...]:
         return self._names
 
     @property
-    def args(self) -> List[Tuple[str, str]]:
+    def args(self) -> List[Tuple[str, Any, str]]:
         return self._arguments
 
     @property
@@ -32,6 +33,10 @@ class MagicCommand:
     def optionals(self) -> List[Tuple[str, Any, str]]:
         return self._optionals
 
+    @property
+    def disables(self) -> Set[str]:
+        return self._disables
+
     @property
     def requires_code(self) -> bool:
         return self._code
@@ -40,8 +45,17 @@ class MagicCommand:
     def requires_query_result(self) -> bool:
         return self._result
 
-    def arg(self, name: str, description: str = None) -> 'MagicCommand':
-        self._arguments.append((name, description))
+    @property
+    def is_default(self) -> bool:
+        return self._default
+
+    def arg(self, name: str, default_value: Any = None, description: str = None) -> 'MagicCommand':
+        if len(self._arguments) > 0:
+            ln, ldv, _ = self._arguments[-1]
+            if ldv is not None and default_value is None:
+                raise ValueError(f'argument {name} without default value registered after argument {ln} with default value {ldv}')
+
+        self._arguments.append((name, default_value, description))
         return self
 
     def opt(self, name: str, default_value: Any = None, description: str = None) -> 'MagicCommand':
@@ -52,6 +66,12 @@ class MagicCommand:
         self._flags.append((name, description))
         return self
 
+    def disable(self, *name: str) -> 'MagicCommand':
+        for n in name:
+            self._disables.add(n)
+
+        return self
+
     def code(self, code: bool) -> 'MagicCommand':
         self._code = code
         return self
@@ -60,10 +80,14 @@ class MagicCommand:
         self._result = result
         return self
 
-    def on(self, fun: Callable):
+    def on(self, fun: Callable) -> 'MagicCommand':
         self._on.append(fun)
         return self
 
+    def default(self, default: bool) -> 'MagicCommand':
+        self._default = default
+        return self
+
     @property
     def parameters(self) -> str:
         args = ' +'.join([self._ARG] * len(self._arguments))

duckdb_kernel/magics/MagicCommandCallback.py

@@ -1,20 +1,31 @@
 from typing import Optional, List
 
 from . import MagicCommand
+from .StringWrapper import StringWrapper
 
 
 class MagicCommandCallback:
-    def __init__(self, mc: MagicCommand, silent: bool, code: str, *args, **kwargs):
+    def __init__(self, mc: MagicCommand, silent: bool, code: StringWrapper, *args, **kwargs):
         self._mc: MagicCommand = mc
         self._silent: bool = silent
-        self._code: str = code
+        self._code: StringWrapper = code
         self._args = args
         self._kwargs = kwargs
 
+    @property
+    def magic(self) -> MagicCommand:
+        return self._mc
+
     def __call__(self, columns: Optional[List[str]] = None, rows: Optional[List[List]] = None):
         if self._mc.requires_code:
-            return self._mc(self._silent, self._code, *self._args, **self._kwargs)
+            result = self._mc(self._silent, self._code.value, *self._args, **self._kwargs)
+            if 'generated_code' in result:
+                self._code.value = result['generated_code']
+
+            return result
+
         if self._mc.requires_query_result:
             return self._mc(self._silent, columns, rows, *self._args, **self._kwargs)
+
         else:
             return self._mc(self._silent, *self._args, **self._kwargs)

duckdb_kernel/magics/MagicCommandHandler.py

@@ -1,7 +1,8 @@
 import re
-from typing import Dict, Tuple, List
+from typing import Dict, Tuple, List, Optional
 
 from . import MagicCommand, MagicCommandException, MagicCommandCallback
+from .StringWrapper import StringWrapper
 
 
 class MagicCommandHandler:
@@ -14,10 +15,23 @@ class MagicCommandHandler:
                 key = key.lower()
                 self._magics[key] = cmd
 
+    def __getitem__(self, key: str) -> MagicCommand:
+        return self._magics[key.lower()]
+
     def __call__(self, silent: bool, code: str) -> Tuple[str, List[MagicCommandCallback], List[MagicCommandCallback]]:
-        pre_query_callbacks = []
-        post_query_callbacks = []
+        code_wrapper = StringWrapper()
+        enabled_callbacks: List[MagicCommandCallback] = []
+
+        # enable commands with default==True
+        for magic in self._magics.values():
+            if magic.is_default:
+                flags = {name: False for name, _ in magic.flags}
+                optionals = {name: default for name, default, _ in magic.optionals}
+                callback = MagicCommandCallback(magic, silent, code_wrapper, **flags, **optionals)
 
+                enabled_callbacks.append(callback)
+
+        # search for magic commands in code
         while True:
             # ensure code starts with '%' or '%%' but not with '%%%'
             match = re.match(r'^%{1,2}([^% ]+?)([ \t]*$| .+?$)', code, re.MULTILINE | re.IGNORECASE)
@@ -45,7 +59,11 @@ class MagicCommandHandler:
                 raise MagicCommandException(f'could not parse parameters for command "{command}"')
 
             # extract args
-            args = [g for g, _ in zip(match.groups(), magic.args)]
+            args = [group if group is not None else default
+                    for group, (_, default, _) in zip(match.groups(), magic.args)]
+
+            if any(arg is None for arg in args):
+                raise MagicCommandException(f'could not parse parameters for command "{command}"')
 
             i = len(args) + 1
 
@@ -73,12 +91,35 @@ class MagicCommandHandler:
                     optionals[name.lower()] = value
 
             # add to callbacks
-            callback = MagicCommandCallback(magic, silent, code, *args, **flags, **optionals)
+            callback = MagicCommandCallback(magic, silent, code_wrapper, *args, **flags, **optionals)
+            enabled_callbacks.append(callback)
+
+        # disable overwritten callbacks
+        callbacks = []
+        blacklist = set()
+
+        for callback in reversed(enabled_callbacks):
+            for name in callback.magic.names:
+                if name in blacklist:
+                    break
+            else:
+                callbacks.append(callback)
+
+                for name in callback.magic.names:
+                    blacklist.add(name)
+                for disable in callback.magic.disables:
+                    blacklist.add(disable)
+
+        # prepare callback lists
+        pre_query_callbacks = []
+        post_query_callbacks = []
 
-            if not magic.requires_query_result:
+        for callback in reversed(callbacks):
+            if not callback.magic.requires_query_result:
                 pre_query_callbacks.append(callback)
             else:
                 post_query_callbacks.append(callback)
 
         # return callbacks
+        code_wrapper.value = code
         return code, pre_query_callbacks, post_query_callbacks

duckdb_kernel/magics/StringWrapper.py

@@ -0,0 +1,3 @@
+class StringWrapper:
+    def __init__(self, value: str = None):
+        self.value: str = value

duckdb_kernel/parser/DCParser.py

@@ -1,3 +1,4 @@
+from .ParserError import DCParserError
 from .elements import *
 from .tokenizer import *
 
@@ -18,17 +19,17 @@ class DCParser:
                 )
 
         # raise exception if query is not in the correct format
-        raise AssertionError('The expression shall be of the format "{ x1, ..., xn | f(x1, ..., xn) }".')
+        raise DCParserError('The expression shall be of the format "{ x1, ..., xn | f(x1, ..., xn) }".', 0)
 
     @staticmethod
-    def parse_projection(*tokens: Token) -> LogicOperand:
+    def parse_projection(*tokens: Token, depth: int = 0) -> LogicOperand:
         if len(tokens) == 1:
             tokens = tuple(Tokenizer.tokenize(tokens[0]))
 
         return LogicOperand(*tokens)
 
     @staticmethod
-    def parse_condition(*tokens: Token) -> LogicElement:
+    def parse_condition(*tokens: Token, depth: int = 0) -> LogicElement:
         if len(tokens) == 1:
             tokens = tuple(Tokenizer.tokenize(tokens[0]))
 
@@ -40,8 +41,8 @@ class DCParser:
                     # return the operator
                     # with left part of tokens and right part of tokens
                     return operator(
-                        DCParser.parse_condition(*tokens[:-i]),
-                        DCParser.parse_condition(*tokens[-i + 1:])
+                        DCParser.parse_condition(*tokens[:-i], depth=depth + 1),
+                        DCParser.parse_condition(*tokens[-i + 1:], depth=depth + 1)
                     )
 
         # not
@@ -56,10 +57,12 @@ class DCParser:
         elif len(tokens) == 2:
             return DCOperand(
                 tokens[0],
-                tuple(Tokenizer.tokenize(tokens[1]))
+                tuple(Tokenizer.tokenize(tokens[1])),
+                depth=depth + 1
             )
         else:
             return DCOperand(
                 tokens[0],
-                tokens[1:]
+                tokens[1:],
+                depth=depth + 1
             )

duckdb_kernel/parser/LogicParser.py

@@ -4,12 +4,12 @@ from .tokenizer import *
 
 class LogicParser:
     @staticmethod
-    def parse_query(query: str) -> LogicElement:
+    def parse_query(query: str, depth: int = 0) -> LogicElement:
         initial_token = Token(query)
-        return LogicParser.parse_tokens(initial_token)
+        return LogicParser.parse_tokens(initial_token, depth=depth)
 
     @staticmethod
-    def parse_tokens(*tokens: Token) -> LogicElement:
+    def parse_tokens(*tokens: Token, depth: int = 0) -> LogicElement:
         if len(tokens) == 1:
             tokens = tuple(Tokenizer.tokenize(tokens[0]))
 
@@ -21,14 +21,14 @@ class LogicParser:
                     # return the operator
                     # with left part of tokens and right part of tokens
                     return operator(
-                        LogicParser.parse_tokens(*tokens[:-i]),
-                        LogicParser.parse_tokens(*tokens[-i + 1:])
+                        LogicParser.parse_tokens(*tokens[:-i], depth=depth + 1),
+                        LogicParser.parse_tokens(*tokens[-i + 1:], depth=depth + 1)
                     )
 
         # not
         if tokens[0] in LOGIC_NOT.symbols():
             return LOGIC_NOT(
-                LogicParser.parse_tokens(*tokens[1:])
+                LogicParser.parse_tokens(*tokens[1:], depth=depth + 1)
             )
 
         # ArgList

duckdb_kernel/parser/ParserError.py

@@ -0,0 +1,18 @@
+class ParserError(Exception):
+    def __init__(self, message: str, depth: int):
+        super().__init__(message)
+
+        self.message: str = message
+        self.depth: int = depth
+
+
+class RAParserError(ParserError):
+    pass
+
+
+class DCParserError(ParserError):
+    pass
+
+
+class LogicParserError(ParserError):
+    pass

duckdb_kernel/parser/RAParser.py

@@ -1,4 +1,5 @@
 from .LogicParser import LogicParser
+from .ParserError import RAParserError
 from .elements import *
 from .tokenizer import *
 
@@ -10,10 +11,10 @@ class RAParser:
     @staticmethod
     def parse_query(query: str) -> RAElement:
         initial_token = Token(query)
-        return RAParser.parse_tokens(initial_token)
+        return RAParser.parse_tokens(initial_token, depth=0)
 
     @staticmethod
-    def parse_tokens(*tokens: Token, target: RAOperator | RAOperand = None) -> RAElement:
+    def parse_tokens(*tokens: Token, target: RAOperator | RAOperand = None, depth: int = 0) -> RAElement:
         if len(tokens) == 1:
             tokens = tuple(Tokenizer.tokenize(tokens[0]))
 
@@ -24,15 +25,15 @@ class RAParser:
                 if tokens[-i].lower() in operator.symbols():
                     # raise error if left or right operand missing
                     if i == 1:
-                        raise AssertionError(f'right operand missing after {tokens[-i]}')
+                        raise RAParserError(f'right operand missing after {tokens[-i]}', depth)
                     if i == len(tokens):
-                        raise AssertionError(f'left operand missing before {tokens[-i]}')
+                        raise RAParserError(f'left operand missing before {tokens[-i]}', depth)
 
                     # return the operator
                     # with left part of tokens and right part of tokens
                     return operator(
-                        RAParser.parse_tokens(*tokens[:-i]),
-                        RAParser.parse_tokens(*tokens[-i + 1:])
+                        RAParser.parse_tokens(*tokens[:-i], depth=depth + 1),
+                        RAParser.parse_tokens(*tokens[-i + 1:], depth=depth + 1)
                     )
 
         # unary operators
@@ -44,8 +45,8 @@ class RAParser:
                     # the last token is the operators target.
                     if target is None:
                         op = operator(
-                            RAParser.parse_tokens(tokens[-1]),
-                            LogicParser.parse_tokens(*tokens[-i + 1:-1])
+                            RAParser.parse_tokens(tokens[-1], depth=depth + 1),
+                            LogicParser.parse_tokens(*tokens[-i + 1:-1], depth=depth + 1)
                         )
 
                     # Otherwise the handed target is this operator's
@@ -53,16 +54,13 @@ class RAParser:
                     else:
                         op = operator(
                             target,
-                            LogicParser.parse_tokens(*tokens[-i + 1:])
+                            LogicParser.parse_tokens(*tokens[-i + 1:], depth=depth + 1)
                         )
 
                     # If there are any more tokens the operator is
                     # the target for the next step.
                     if i < len(tokens):
-                        return RAParser.parse_tokens(
-                            *tokens[:-i],
-                            target=op
-                        )
+                        return RAParser.parse_tokens(*tokens[:-i], target=op, depth=depth + 1)
 
                     # Otherwise the operator is the return value.
                     else:
@@ -70,6 +68,6 @@ class RAParser:
 
         # return as name
         if len(tokens) > 1:
-            raise AssertionError(f'{tokens=}')
+            raise RAParserError(f'{tokens=}', depth)
 
         return RAOperand(tokens[0])

duckdb_kernel/parser/__init__.py

@@ -1,3 +1,4 @@
 from .DCParser import DCParser
 from .LogicParser import LogicParser
 from .RAParser import RAParser
+from .ParserError import *

duckdb_kernel/parser/elements/DCOperand.py

@@ -1,13 +1,14 @@
 from typing import Tuple
 
 from .LogicOperand import LogicOperand
+from ..ParserError import DCParserError
 from ..tokenizer import Token
 
 
 class DCOperand(LogicOperand):
-    def __new__(cls, relation: Token, columns: Tuple[Token], skip_comma: bool = False):
+    def __new__(cls, relation: Token, columns: Tuple[Token, ...], skip_comma: bool = False, depth: int = 0):
         if not skip_comma and not all(t == ',' for i, t in enumerate(columns) if i % 2 == 1):
-            raise AssertionError('arguments must be separated by commas')
+            raise DCParserError('arguments must be separated by commas', 0)
 
         return tuple.__new__(
             cls,
@@ -18,9 +19,11 @@ class DCOperand(LogicOperand):
             ))
         )
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, relation: Token, columns: Tuple[Token, ...], skip_comma: bool = False, depth: int = 0):
         super().__init__()
-        self.invert = False
+
+        self.depth: int = depth
+        self.invert: bool = False
 
     @property
     def relation(self) -> Token:

duckdb_kernel/parser/elements/binary/ConditionalSet.py

@@ -8,6 +8,7 @@ from ..LogicElement import LogicElement
 from ..LogicOperand import LogicOperand
 from ..LogicOperator import LogicOperator
 from ..unary import Not
+from ...ParserError import DCParserError
 from ...tokenizer import Token
 from ...util.RenamableColumnList import RenamableColumnList
 from ....db import Table
@@ -42,8 +43,11 @@ class ConditionalSet:
 
                 # If a constant was found, we store the value and replace it with a random attribute name.
                 constant = le.names[i]
-                new_token = Token.random()
-                new_operand = DCOperand(le.relation, le.names[:i] + (new_token,) + le.names[i + 1:], skip_comma=True)
+                new_token = Token.random(constant)
+                new_operand = DCOperand(le.relation,
+                                        le.names[:i] + (new_token,) + le.names[i + 1:],
+                                        skip_comma=True,
+                                        depth=le.depth)
 
                 # We now need an equality comparison to ensure the introduced attribute is equal to the constant.
                 equality = Equal(
@@ -103,7 +107,7 @@ class ConditionalSet:
         # The default case is to return the LogicElement with not DCOperands.
         return le, []
 
-    def to_sql(self, tables: Dict[str, Table]) -> str:
+    def to_sql_with_renamed_columns(self, tables: Dict[str, Table]) -> Tuple[str, Dict[str, str]]:
         # First we have to find and remove all DCOperands from the operator tree.
         condition, dc_operands = self.split_tree(self.condition)
 
@@ -121,7 +125,8 @@ class ConditionalSet:
             # Raise an exception if the given number of operands does not match
             # the number of attributes in the relation.
             if len(source_columns) != len(operand.names):
-                raise AssertionError(f'invalid number of attributes for relation {operand.relation}')
+                raise DCParserError(f'invalid number of attributes for relation {operand.relation}',
+                                    depth=operand.depth)
 
             # Create a column list for this operand.
             rcl: RenamableColumnList = RenamableColumnList.from_iter(source_columns)
@@ -215,7 +220,8 @@ class ConditionalSet:
                         if left_name != right_name:
                             break
                     else:
-                        raise AssertionError(f'could not build join for relation {left_name}')
+                        raise DCParserError(f'could not build join for relation {left_name}',
+                                            depth=left_op.depth)
 
                     join_tuple = min(left_name, right_name), max(left_name, right_name)
 
@@ -253,7 +259,7 @@ class ConditionalSet:
 
             # If no joins were discovered using this table, an exception is raised.
             if discovered_joins == 0:
-                raise AssertionError('no common attributes found for join')
+                raise DCParserError('no common attributes found for join', depth=right_op.depth)
 
         # The joins have to be sorted in a topologic order starting from t0.
         used_relations: Set[str] = {'t0'}
@@ -287,7 +293,8 @@ class ConditionalSet:
                     break
 
             else:
-                raise AssertionError('no valid topologic order found for positive joins')
+                raise DCParserError('no valid topologic order found for positive joins',
+                                    depth=min(op.depth for _, _, op in relevant_positive))
 
         all_negative_conditions: Dict[str, List[str]] = {}
         all_negative_filters: Dict[str, List[str]] = {}
@@ -317,7 +324,8 @@ class ConditionalSet:
                     used_relations.add(target_name)
                     break
             else:
-                raise AssertionError('no valid topologic order found for negative joins')
+                raise DCParserError('no valid topologic order found for negative joins',
+                                    depth=min(op.depth for _, _, op in relevant_negative))
 
         # Build the SQL statement.
         sql_select = ', '.join(select_columns[col] if col in select_columns else col
@@ -339,5 +347,18 @@ class ConditionalSet:
             sql_join_filters += f' AND {join_filter}'
 
         sql_condition = condition.to_sql(joined_columns) if condition is not None else '1=1'
+        sql_query = f'SELECT DISTINCT {sql_select} FROM {sql_tables} WHERE ({sql_join_filters}) AND ({sql_condition})'
+
+        # Create a mapping from intermediate column names to constant values.
+        column_name_mapping = {
+            p: p.constant
+            for o in dc_operands
+            for p in o.names
+            if p.constant is not None
+        }
 
-        return f'SELECT DISTINCT {sql_select} FROM {sql_tables} WHERE ({sql_join_filters}) AND ({sql_condition})'
+        return sql_query, column_name_mapping
+
+    def to_sql(self, tables: Dict[str, Table]) -> str:
+        sql, _ = self.to_sql_with_renamed_columns(tables)
+        return sql

duckdb_kernel/parser/tokenizer/Token.py

@@ -1,8 +1,9 @@
+from typing import Optional
 from uuid import uuid4
 
 
 class Token(str):
-    def __new__(cls, value: str):
+    def __new__(cls, value: str, constant: 'Token' = None):
         while True:
             # strip whitespaces
             value = value.strip()
@@ -38,20 +39,40 @@ class Token(str):
 
         return super().__new__(cls, value)
 
+    def __init__(self, value: str, constant: 'Token' = None):
+        self.constant: Optional[Token] = constant
+
     @staticmethod
-    def random() -> 'Token':
-        return Token('__' + str(uuid4()).replace('-', '_'))
+    def random(constant: 'Token' = None) -> 'Token':
+        return Token('__' + str(uuid4()).replace('-', '_'), constant)
 
     @property
     def empty(self) -> bool:
         return len(self) == 0
 
+    @property
+    def is_temporary(self) -> bool:
+        return self.startswith('__')
+
     @property
     def is_constant(self) -> bool:
         return ((self[0] == '"' and self[-1] == '"') or
                 (self[0] == "'" and self[-1] == "'") or
                 self.replace('.', '', 1).isnumeric())
 
+    @property
+    def no_quotes(self) -> str:
+        quotes = ('"', "'")
+
+        if self[0] in quotes and self[-1] in quotes:
+            return self[1:-1]
+        if self[0] in quotes:
+            return self[1:]
+        if self[-1] in quotes:
+            return self[:-1]
+        else:
+            return self
+
     @property
     def single_quotes(self) -> str:
         # TODO Is this comparison useless because tokens are cleaned automatically?

duckdb_kernel/util/TestError.py

@@ -0,0 +1,4 @@
+class TestError(Exception):
+    @property
+    def message(self) -> str:
+        return str(self)

{jupyter_duckdb-1.2.0.3.dist-info → jupyter_duckdb-1.2.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: jupyter-duckdb
-Version: 1.2.0.3
+Version: 1.2.0.5
 Summary: a basic wrapper kernel for DuckDB
 Home-page: https://github.com/erictroebs/jupyter-duckdb
 Author: Eric Tröbs
@@ -32,10 +32,6 @@ This is a simple DuckDB wrapper kernel which accepts SQL as input, executes it
 using a previously loaded DuckDB instance and formats the output as a table.
 There are some magic commands that make teaching easier with this kernel.
 
-## Quick Start
-
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/git/https%3A%2F%2Fdbgit.prakinf.tu-ilmenau.de%2Fertr8623%2Fjupyter-duckdb.git/master)
-
 ## Table of Contents
 
 - [Setup](#setup)
@@ -85,6 +81,12 @@ Execute the following command to pull and run a prepared image.
 docker run -p 8888:8888 troebs/jupyter-duckdb
 ```
 
+There is also a second image. It contains an additional instance of PostgreSQL:
+
+```bash
+docker run -p 8888:8888 troebs/jupyter-duckdb:postgresql
+```
+
 This image can also be used with JupyterHub and the
 [DockerSpawner / SwarmSpawner](https://github.com/jupyterhub/dockerspawner)
 and probably with the
@@ -138,6 +140,13 @@ Please note that `:memory:` is also a valid file path for DuckDB. The data is
 then stored exclusively in the main memory. In combination with `CREATE`
 and `OF` this makes it possible to work on a temporary copy in memory.
 
+Although the name suggests otherwise, the kernel can also be used with other
+databases:
+- **SQLite** is automatically used as a fallback if the DuckDB dependency is
+  missing.
+- To connect to a **PostgreSQL** instance, you need to specify a database URI
+  starting with `(postgresql|postgres|pgsql|psql|pg)://`.
+
 ### Schema Diagrams
 
 The magic command `SCHEMA` can be used to create a simple schema diagram of the
@@ -153,6 +162,10 @@ representation requires more space, but can improve readability.
 %SCHEMA TD
 ```
 
+The optional argument `ONLY`, followed by one or more table names separated by a
+comma, can be used to display only the named tables and all those connected with
+a foreign key.
+
 Graphviz (`dot` in PATH) is required to render schema diagrams.
 
 ### Number of Rows
@@ -234,6 +247,11 @@ UNION
 SELECT 1, 'Name 1'
 ```
 
+By default, failed tests will display an explanation, but the notebook will
+continue to run. Set the `DUCKDB_TESTS_RAISE_EXCEPTION` environment variable to
+`true` to raise an exception when a test fails. This can be useful for automated
+testing in CI environments.
+
 Disclaimer: The integrated testing is work-in-progress and thus subject to
 potentially incompatible changes and enhancements.
 
@@ -244,7 +262,7 @@ magic command `RA` activates the relational algebra mode for a single cell:
 
 ```
 %RA
-π a, b (σ c = 1 (R))
+π [a, b] (σ [c = 1] (R))
 ```
 
 The supported operations are:
@@ -259,6 +277,9 @@ The supported operations are:
 - Cross Product `×`
 - Division `÷`
 
+The optional flag `ANALYZE` can be used to add an execution diagram to the
+output.
+
 The Dockerfile also installs the Jupyter Lab plugin
 [jupyter-ra-extension](https://pypi.org/project/jupyter-ra-extension/). It adds
 the symbols mentioned above and some other supported symbols to the toolbar for
@@ -273,3 +294,13 @@ magic command `DC` activates the domain calculus mode for a single cell:
 %DC
 { a, b | R(a, b, c) ∧ c = 1 }
 ```
+
+### Automated Parser Selection
+
+`%ALL_RA` or `%ALL_DC` enables the corresponding parser for all subsequently
+executed cells.
+
+If the magic command `%AUTO_PARSER` is added to a cell, a parser is
+automatically selected. If `%GUESS_PARSER` is executed, the parser is
+automatically selected for all subsequent cells.
+

{jupyter_duckdb-1.2.0.3.dist-info → jupyter_duckdb-1.2.0.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 duckdb_kernel/__init__.py,sha256=6auU6zeJrsA4fxPSr2PYamS8fG-SMXTn5YQFXF2cseo,33
 duckdb_kernel/__main__.py,sha256=Z3GwHEBWoQjNm2Y84ijnbA0Lk66L7nsFREuqhZ_ptk0,165
 duckdb_kernel/kernel.json,sha256=_7E8Ci2FSdCvnzCjsOaue8QE8AvpS5JLQuxORO5IGtA,127
-duckdb_kernel/kernel.py,sha256=Td6S4qWDC9uuqgAhSllXwR-JHHdnGfT14RHkfYLzXxI,19441
+duckdb_kernel/kernel.py,sha256=4OFNtWvrfSs27udrk9e1ClBMdltvLA9gJemSBZskj0k,21827
 duckdb_kernel/db/Column.py,sha256=GM5P6sFdlYK92hiKln5-6038gIDOTxh1AYbR4kiga_w,559
 duckdb_kernel/db/Connection.py,sha256=5pH-CwGh-r9Q2QwJKGSxvoINBU-sqmvZyG4Q1digfeE,599
 duckdb_kernel/db/Constraint.py,sha256=1YgUHk7s8mHCVedbcuJKyXDykj7_ybbwT3Dk9p2VMis,287
@@ -18,16 +18,18 @@ duckdb_kernel/db/implementation/postgres/__init__.py,sha256=HKogB1es4wOiQUoh7_eT
 duckdb_kernel/db/implementation/postgres/util.py,sha256=4nr1mqXhlwkMVXbJSfJ7dRlUm6UskpvgKApe7GRwmBI,281
 duckdb_kernel/db/implementation/sqlite/Connection.py,sha256=L-aSUuN_7Z-hc5wczLjap2Xu-Vv6-dXmwXcN_WE8o98,6994
 duckdb_kernel/db/implementation/sqlite/__init__.py,sha256=HKogB1es4wOiQUoh7_eT32xnUFLmzoCyR_0LuY9r8YQ,35
-duckdb_kernel/magics/MagicCommand.py,sha256=OoQ6j4cNtIYjaK4MPVzJyv1eYTNu4_a7qoRx-5G3Hg0,2346
-duckdb_kernel/magics/MagicCommandCallback.py,sha256=r1kkJyRR7sZnrnlMH3w4bGqDAJL-BVTIB4-Kn66ynlM,764
+duckdb_kernel/magics/MagicCommand.py,sha256=l0EmnqgGZ0HyqQhdTljAaftflVo_RYp-U5UiDftYxAA,3180
+duckdb_kernel/magics/MagicCommandCallback.py,sha256=9XVidNtzs8Lwq_T59VrH89t5LUgJcc5C7grusElXVW4,1041
 duckdb_kernel/magics/MagicCommandException.py,sha256=MwuWkpA6NoCqz437urdI0RVXhbSbVdziuRoi7slYFPc,49
-duckdb_kernel/magics/MagicCommandHandler.py,sha256=4njm49cNnLBH9j8GazrmA-wF8XkXfSioJDuCoDyumqc,2749
+duckdb_kernel/magics/MagicCommandHandler.py,sha256=TCCtAEaop5yTUn2af4wzcReEto3nql-40-DpIdIjKEw,4367
+duckdb_kernel/magics/StringWrapper.py,sha256=W6qIfeHU51do1edd6yXgw6K7orzjwSHU4oWAI5DuKEE,96
 duckdb_kernel/magics/__init__.py,sha256=DA8gnQeRCUt1Scy3_NQ9w5CPmMEY9i8YwB-g392pN1U,204
-duckdb_kernel/parser/DCParser.py,sha256=ciNdM_ZFfa10bBlAc_bB2tKmPOxnXBcAGpYROjbAkVY,2120
-duckdb_kernel/parser/LogicParser.py,sha256=PI4NTe4UZIsnEvoAe_LgpEtmGraTTmYOsUk5_Qr3QOk,1137
-duckdb_kernel/parser/RAParser.py,sha256=PcyqVxsfHlHJexHy77tL6XIubA2eArBUqbrw13BZz6Q,2891
-duckdb_kernel/parser/__init__.py,sha256=Tcq5YTgF-FQSgleSx-nGzUXfR_GR02DKUNDKgBskKVQ,99
-duckdb_kernel/parser/elements/DCOperand.py,sha256=RGj_n-v4BimiK3bMGSwOtV03g2FTg-97d4lJ8tPUZmM,977
+duckdb_kernel/parser/DCParser.py,sha256=16c1mxa494KP9OreUKQHsSQKoDGZ7NNp2u_gi_D-dkw,2293
+duckdb_kernel/parser/LogicParser.py,sha256=_vZwE5OPRUEN8aEC_fSZAYKR_dpexqNthXog9OFHYRY,1233
+duckdb_kernel/parser/ParserError.py,sha256=qJQVloFtID1HgVDQ1Io247bODT1ic3oO9Z1ZrWR-2Mk,321
+duckdb_kernel/parser/RAParser.py,sha256=dL5Q5J8tmlGuKQMfadvJK6cb9KmzfZLkfcUAthqDmXI,2993
+duckdb_kernel/parser/__init__.py,sha256=nTmDm1ADvNPDHhVJQLxKYmArNJk6967EUXqn5AkT8FM,126
+duckdb_kernel/parser/elements/DCOperand.py,sha256=qEg_6Us4WV1eK4Bq6oUsmFt_L_x5pJPGce_wSapzIYA,1149
 duckdb_kernel/parser/elements/LogicElement.py,sha256=YasKHxWLDDP8UdyLIKbXzqIRA8-XaakjmvTj-1Iuzyc,280
 duckdb_kernel/parser/elements/LogicOperand.py,sha256=B9NvriloQE5eP734dNMZBZwrdaaIfsuAmZlG1t2eMhs,1021
 duckdb_kernel/parser/elements/LogicOperator.py,sha256=lkM4TAGkXUhsO4w4PLKVA0bgCRGPQQFpNA1FcWWOW9Q,1028
@@ -40,7 +42,7 @@ duckdb_kernel/parser/elements/__init__.py,sha256=4DA2M43hh9d1fZb5Z6YnTTI-IBkDyhC
 duckdb_kernel/parser/elements/binary/Add.py,sha256=XGkZMfab01huk9EaI6JUfzkd2STbV1C_-TyC2guKE8I,190
 duckdb_kernel/parser/elements/binary/And.py,sha256=0jgetTG8yo5TJSeK70Kj-PI9ERyek1eyMQXX5HBxa4Y,274
 duckdb_kernel/parser/elements/binary/ArrowLeft.py,sha256=u4fZSoyT9lfvWXBwuhUl4DdjVZAOqyVIKmMVbpElLD4,203
-duckdb_kernel/parser/elements/binary/ConditionalSet.py,sha256=4KzvUTls2bodJw9ejCKx8se32PR5VFJbVupDZVx2NHE,16671
+duckdb_kernel/parser/elements/binary/ConditionalSet.py,sha256=sZ3qrxPux7pb3fMrlyBg4Hw7n4-Ln-AeN70_Jp5dAPo,17652
 duckdb_kernel/parser/elements/binary/Cross.py,sha256=jVY3cvD6qDWZkJ7q74lFUPO2VdDt4aAjdk2YAfg-ZC4,687
 duckdb_kernel/parser/elements/binary/Difference.py,sha256=ZVRgJHYVMOFwnc97oPvGtKvLvHsjSCsn2Aao6ymxY8Y,742
 duckdb_kernel/parser/elements/binary/Divide.py,sha256=d7mzaOeRYSRO1F-2IHsv_C939TuYtLppbf4-5GSRJXs,265
@@ -63,20 +65,21 @@ duckdb_kernel/parser/elements/unary/Projection.py,sha256=CJ-MIf1-__1ewTjNZVy5hOz
 duckdb_kernel/parser/elements/unary/Rename.py,sha256=Zr2n9EJ3nA476lND0Djz2b6493nnsbSpJ9kkEgk5B_Y,1273
 duckdb_kernel/parser/elements/unary/Selection.py,sha256=TKykDMw0QGQcMFp0r7g6ye4CkjshBTNq14c7qtMkqs4,955
 duckdb_kernel/parser/elements/unary/__init__.py,sha256=48EDygy0pD7l3J_BlXGc-b7HYPaiHQa1-0Mcsj9Xzr0,270
-duckdb_kernel/parser/tokenizer/Token.py,sha256=vwN5hHg11kqzOHLeL5GO1c1BbCTZzYDTuy0QR4kDzew,1800
+duckdb_kernel/parser/tokenizer/Token.py,sha256=gsCzgU_zLiA-yD0FWvd2qS9LQUXbivESYH-34Glffqs,2404
 duckdb_kernel/parser/tokenizer/Tokenizer.py,sha256=PWGgS7gYgpULiKGDho842UbaXuqmwEkccixuF10oi5g,5081
 duckdb_kernel/parser/tokenizer/__init__.py,sha256=EOSmfc2RJwtB5cE1Hhj1JAra97tckxxS8-legybPy60,58
 duckdb_kernel/parser/util/RenamableColumn.py,sha256=LxJhFDMUv_OxYYDLwKn63QGpBRfs08jVvhuJTzRtc9c,704
 duckdb_kernel/parser/util/RenamableColumnList.py,sha256=GfhdGv4KYT64Z9YA9TCn-7hhcEcc3Gu3vI2zMZ52w-8,3015
 duckdb_kernel/parser/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 duckdb_kernel/util/ResultSetComparator.py,sha256=RZDIfjJyx8-eR-HIqQlEYgZd_V1ympbszpVRF4TlA7o,2262
+duckdb_kernel/util/TestError.py,sha256=iwlGHr9j6pFDa2cGxqGyvJ-exrFUtPJjVm_OhHi4n3g,97
 duckdb_kernel/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 duckdb_kernel/util/formatting.py,sha256=cbt0CtERnqtzd97mLrOjeJpqM2Lo6pW96BjAYqrOTD8,793
 duckdb_kernel/visualization/Drawer.py,sha256=D0LkiGMvuJ2v6cQSg_axLTGaM4VXAJEQJAynvedQ3So,296
 duckdb_kernel/visualization/RATreeDrawer.py,sha256=j-Vy1zpYMzwZ3CsphyfPW-J7ou9a9tM6aXXgAlQTgDI,2128
 duckdb_kernel/visualization/SchemaDrawer.py,sha256=9K-TUUmyeGdMYMTFQJ7evIU3p8p2KyMKeizUc7-y8co,3015
 duckdb_kernel/visualization/__init__.py,sha256=5eMJmxJ01XAXcgWDn3t70eSZF2PGaXdNo6GK-x-0H3s,78
-jupyter_duckdb-1.2.0.3.dist-info/METADATA,sha256=O4AGOi_-WpenZIFXf6ZGNNPFGDWAHTNOYOA2CdEB4g0,7980
-jupyter_duckdb-1.2.0.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-jupyter_duckdb-1.2.0.3.dist-info/top_level.txt,sha256=KvRRPMnmkQNuhyBsXoPmwyt26LRDp0O-0HN6u0Dm5jA,14
-jupyter_duckdb-1.2.0.3.dist-info/RECORD,,
+jupyter_duckdb-1.2.0.5.dist-info/METADATA,sha256=7twgLgE-u5--Y9lV-JCtqPoUFSL8GAwdPyVcmu1NzjA,9115
+jupyter_duckdb-1.2.0.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+jupyter_duckdb-1.2.0.5.dist-info/top_level.txt,sha256=KvRRPMnmkQNuhyBsXoPmwyt26LRDp0O-0HN6u0Dm5jA,14
+jupyter_duckdb-1.2.0.5.dist-info/RECORD,,