jupyter-duckdb 0.9.2.2.dev202401171418__tar.gz → 0.9.2.3__tar.gz

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jupyter-duckdb
-Version: 0.9.2.2.dev202401171418
+Version: 0.9.2.3
 Summary: a basic wrapper kernel for DuckDB
 Home-page: https://github.com/erictroebs/jupyter-duckdb
 Author: Eric Tröbs

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/setup.py

@@ -1,13 +1,12 @@
 import os
+import re
 from datetime import datetime
 
 from setuptools import setup, find_packages
 
 # version
-version = '0.9.2.2'
-
-if os.getenv('DEV') == '1':
-    version += '.dev' + datetime.now().strftime('%Y%m%d%H%M')
+version = os.getenv('PACKAGE_VERSION', '1.0')
+version = re.sub(r'[^dev0-9._]', '', version)
 
 # install requires
 install_requires = [
@@ -47,7 +46,10 @@ setup(
     install_requires=install_requires,
     package_data={
         'duckdb_kernel': [
-            'kernel.json'
+            'kernel.json',
+            'db/implementation/duckdb/*.py',
+            'db/implementation/postgres/*.py',
+            'db/implementation/sqlite/*.py'
         ]
     },
     include_package_data=True

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/duckdb_kernel/db/Connection.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Tuple, Any
 
-from . import Table
+from .Table import Table
 
 
 class Connection:
@@ -10,6 +10,17 @@ class Connection:
     def close(self):
         pass
 
+    @staticmethod
+    def plain_explain() -> bool:
+        return False
+
+    @staticmethod
+    def multiple_statements_per_query() -> bool:
+        return True
+
+    def __str__(self) -> str:
+        raise NotImplementedError
+
     def execute(self, query: str) -> Tuple[List[str], List[List[Any]]]:
         raise NotImplementedError
 

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/duckdb_kernel/db/Table.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 
 from . import Column
 from . import ForeignKey
-from .Constraint import Constraint
+from . import Constraint
 
 
 class Table:

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/duckdb_kernel/db/__init__.py

@@ -1,12 +1,6 @@
-from .DatabaseError import DatabaseError
 from .Column import Column
+from .Connection import Connection
 from .Constraint import Constraint
+from .DatabaseError import DatabaseError
 from .ForeignKey import ForeignKey
 from .Table import Table
-
-try:
-    from .duckdb import Connection
-    SQLITE_MODE = False
-except ImportError:
-    from .sqlite import Connection
-    SQLITE_MODE = True

jupyter-duckdb-0.9.2.3/src/duckdb_kernel/db/error/EmptyResultError.py

@@ -0,0 +1,5 @@
+from ..DatabaseError import DatabaseError
+
+
+class EmptyResultError(DatabaseError):
+    pass

jupyter-duckdb-0.9.2.3/src/duckdb_kernel/db/error/__init__.py

@@ -0,0 +1 @@
+from .EmptyResultError import EmptyResultError

{jupyter-duckdb-0.9.2.2.dev202401171418/src/duckdb_kernel/db → jupyter-duckdb-0.9.2.3/src/duckdb_kernel/db/implementation}/duckdb/Connection.py

@@ -2,17 +2,25 @@ from typing import Dict, List, Tuple, Any
 
 import duckdb
 
-from .. import DatabaseError, Column, Constraint, ForeignKey, Table
-from ..Connection import Connection as Base
+from ... import DatabaseError, Column, Constraint, ForeignKey, Table
+from ...Connection import Connection as Base
 
 
 class Connection(Base):
     def __init__(self, path: str):
+        self.path: str = path
         self.con: duckdb.DuckDBPyConnection = duckdb.connect(path, read_only=False)
 
     def close(self):
         self.con.close()
 
+    @staticmethod
+    def plain_explain() -> bool:
+        return True
+
+    def __str__(self) -> str:
+        return f'DuckDB: {self.path}'
+
     def execute(self, query: str) -> Tuple[List[str], List[List[Any]]]:
         with self.con.cursor() as cursor:
             try:

jupyter-duckdb-0.9.2.3/src/duckdb_kernel/db/implementation/postgres/Connection.py

@@ -0,0 +1,226 @@
+import contextlib
+import re
+from typing import Tuple, List, Any, Optional, Dict
+from uuid import uuid4
+
+import psycopg
+
+from ... import DatabaseError, Column, Constraint, ForeignKey, Table
+from ...Connection import Connection as Base
+from ...error import EmptyResultError
+
+
+class Connection(Base):
+    def __init__(self,
+                 host: str, port: int,
+                 username: Optional[str], password: Optional[str],
+                 database_name: Optional[str]):
+        self.host: str = host
+        self.port: int = port
+        self.username: Optional[str] = username
+
+        options = {
+            'host': host,
+            'port': port,
+            'user': username,
+            'password': password,
+            'autocommit': True
+        }
+
+        # If not database name is provided we create one.
+        if database_name is None:
+            # connect to the server without a database name
+            with contextlib.closing(psycopg.connect(**options, dbname='postgres')) as temp_con:
+                # create a database to use
+                database_name: str = 'db_' + str(uuid4()).replace('-', '')
+                temp_con.execute(f'CREATE DATABASE {database_name}')
+
+        # Finally the "real" connection is created using the database name.
+        self.con = psycopg.connect(**options, dbname=database_name)
+        self.database_name: str = database_name
+
+    def close(self):
+        self.con.close()
+
+    def __str__(self) -> str:
+        user = f'{self.username}@' if self.username is not None else ''
+        return f'PostgreSQL: {user}{self.host}:{self.port}/{self.database_name}'
+
+    def execute(self, query: str) -> Tuple[List[str], List[List[Any]]]:
+        with self.con.cursor() as cursor:
+            try:
+                cursor.execute(query)
+            except Exception as e:
+                if isinstance(e, psycopg.Error):
+                    text = str(e)
+                    raise DatabaseError(text)
+                else:
+                    raise e
+
+            # get rows
+            try:
+                rows = cursor.fetchall()
+            except psycopg.ProgrammingError as e:
+                text = str(e)
+                if text == "the last operation didn't produce a result":
+                    raise EmptyResultError()
+                else:
+                    raise e
+
+            # get columns
+            if cursor.description is None:
+                columns = []
+            else:
+                columns = [e[0] for e in cursor.description]
+
+        return columns, rows
+
+    def analyze(self) -> Dict[str, Table]:
+        tables: Dict[str, Table] = {}
+        constraints: Dict[str, Constraint] = {}
+        constraint_index: int = 0
+
+        # First, receive the table names.
+        for table_name, in self.con.execute('''
+            SELECT table_name
+            FROM information_schema.tables
+            WHERE table_schema='public' AND table_type='BASE TABLE'
+        ''').fetchall():
+            table = Table(table_name)
+            tables[table_name] = table
+
+        # Get column names and data types for each table.
+        for table_name, column_name, data_type in self.con.execute('''
+                    SELECT
+                        table_name,
+                        column_name,
+                        data_type
+                    FROM information_schema.columns
+                    ORDER BY ordinal_position ASC
+                ''').fetchall():
+            if table_name in tables:
+                table = tables[table_name]
+
+                column = Column(table, column_name, data_type)
+                table.columns.append(column)
+
+        # Find primary keys.
+        constraints_dict = {}
+
+        for constraint_name, table_name, column_name in self.con.execute('''
+            SELECT tc.constraint_name, tc.table_name, c.column_name
+            FROM information_schema.table_constraints tc
+            JOIN information_schema.constraint_column_usage AS ccu
+                USING (constraint_schema, constraint_name)
+            JOIN information_schema.columns AS c
+                ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name
+            WHERE tc.table_schema = 'public' AND tc.constraint_type = 'PRIMARY KEY'
+            ORDER BY tc.table_name, c.ordinal_position
+        '''):
+            if constraint_name not in constraints_dict:
+                constraints_dict[constraint_name] = (table_name, [])
+
+            constraints_dict[constraint_name][1].append(column_name)
+
+        for constraint_name, (table_name, column_names) in constraints_dict.items():
+            # get table
+            if table_name not in tables:
+                raise AssertionError(f'unknown table {table_name} for constraint {constraint_index}')
+
+            table = tables[table_name]
+
+            # store constraint
+            constraint = Constraint(
+                constraint_index,
+                table,
+                tuple(table.get_column(c) for c in column_names)
+            )
+
+            constraint_index += 1
+
+            # store key
+            if table.primary_key is not None:
+                raise AssertionError(f'discovered second primary key for table {table_name}')
+
+            table.primary_key = constraint
+
+        # Find unqiue keys.
+        constraints_dict = {}
+
+        for constraint_name, table_name, column_name in self.con.execute('''
+            SELECT tc.constraint_name, tc.table_name, c.column_name
+            FROM information_schema.table_constraints tc
+            JOIN information_schema.constraint_column_usage AS ccu
+                USING (constraint_schema, constraint_name)
+            JOIN information_schema.columns AS c
+                ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name
+            WHERE tc.table_schema = 'public' AND tc.constraint_type = 'UNIQUE'
+            ORDER BY tc.table_name, c.ordinal_position
+        '''):
+            if constraint_name not in constraints_dict:
+                constraints_dict[constraint_name] = (table_name, [])
+
+            constraints_dict[constraint_name][1].append(column_name)
+
+        for constraint_name, (table_name, column_names) in constraints_dict.items():
+            # get table
+            if table_name not in tables:
+                raise AssertionError(f'unknown table {table_name} for constraint {constraint_index}')
+
+            table = tables[table_name]
+
+            # store constraint
+            constraint = Constraint(
+                constraint_index,
+                table,
+                tuple(table.get_column(c) for c in column_names)
+            )
+
+            constraint_index += 1
+
+            # store key
+            table.unique_keys.append(constraint)
+
+        # Find foreign keys.
+        for source_table_name, fk_name, fk_def in self.con.execute('''
+            SELECT conrelid::regclass, conname, pg_get_constraintdef(oid)
+            FROM pg_constraint
+            WHERE contype = 'f'
+            AND connamespace = 'public'::regnamespace
+            ORDER BY conrelid::regclass::text, contype DESC
+        '''):
+            # extract information
+            match = re.match(r'^FOREIGN KEY \((.*?)\) REFERENCES (.*?)\((.*?)\)', fk_def, re.IGNORECASE)
+            if match is None:
+                raise AssertionError(f'could not parse foreign key definitions for table {source_table_name}')
+
+            source_table_column_names = [c.strip() for c in match.group(1).split(',')]
+            target_table_name = match.group(2).strip()
+            target_table_column_names = [c.strip() for c in match.group(3).split(',')]
+
+            # get tables
+            if source_table_name not in tables:
+                raise AssertionError(f'unknown table {source_table_name} for foreign key {fk_name}')
+
+            source_table = tables[source_table_name]
+
+            if target_table_name not in tables:
+                raise AssertionError(f'unknown table {target_table_name} for foreign key {fk_name}')
+
+            target_table = tables[target_table_name]
+
+            # store constraint
+            constraint = Constraint(
+                constraint_index,
+                target_table,
+                tuple(target_table.get_column(c) for c in target_table_column_names)
+            )
+
+            constraint_index += 1
+
+            # store key
+            key = ForeignKey(tuple(source_table.get_column(c) for c in source_table_column_names), constraint)
+            source_table.foreign_keys.append(key)
+
+        # return result
+        return tables

{jupyter-duckdb-0.9.2.2.dev202401171418/src/duckdb_kernel/db → jupyter-duckdb-0.9.2.3/src/duckdb_kernel/db/implementation}/sqlite/Connection.py

@@ -2,8 +2,8 @@ import sqlite3
 from contextlib import closing
 from typing import Dict, List, Tuple, Any
 
-from .. import DatabaseError, Column, Constraint, ForeignKey, Table
-from ..Connection import Connection as Base
+from ... import DatabaseError, Column, Constraint, ForeignKey, Table
+from ...Connection import Connection as Base
 
 
 class Connection(Base):
@@ -14,6 +14,13 @@ class Connection(Base):
     def close(self):
         self.con.close()
 
+    @staticmethod
+    def multiple_statements_per_query() -> bool:
+        return False
+
+    def __str__(self) -> str:
+        return f'SQLite: {self.path}'
+
     def execute(self, query: str) -> Tuple[List[str], List[List[Any]]]:
         with closing(self.con.cursor()) as cursor:
             try:

jupyter-duckdb-0.9.2.3/src/duckdb_kernel/db/implementation/sqlite/__init__.py

@@ -0,0 +1 @@
+from .Connection import Connection

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/duckdb_kernel/kernel.py

@@ -8,7 +8,8 @@ from typing import Optional, Dict, List, Tuple
 
 from ipykernel.kernelbase import Kernel
 
-from .db import Connection, DatabaseError, SQLITE_MODE
+from .db import Connection, DatabaseError
+from .db.error import *
 from .magics import *
 from .parser import RAParser, DCParser
 from .util.ResultSetComparator import ResultSetComparator
@@ -20,7 +21,7 @@ class DuckDBKernel(Kernel):
     DEFAULT_MAX_ROWS = 20
 
     implementation = 'DuckDB'
-    implementation_version = '0.9.2'
+    implementation_version = '1.0'
     banner = 'DuckDB Kernel'
     language_info = {
         'name': 'duckdb',
@@ -87,7 +88,41 @@ class DuckDBKernel(Kernel):
     # database related functions
     def _load_database(self, path: str):
         if self._db is None:
-            self._db: Optional[Connection] = Connection(path)
+            # If the provided path looks like a postgres url,
+            # we want to use the postgres driver.
+            if path.startswith(('postgresql://', 'postgres://', 'pgsql://', 'psql://', 'pg://')):
+                # pull data from connection string
+                match = re.fullmatch(r'(postgresql|postgres|pgsql|psql|pg)://((.*?)(:(.*?))?@)?(.*?)(:(\d+))?(/(.*))?', path)
+
+                host = match.group(6)
+                port = int(match.group(8)) if match.group(8) is not None else 5432
+                username = match.group(3)
+                password = match.group(5)
+                database_name = match.group(10)
+
+                # load and create instance
+                try:
+                    from .db.implementation.postgres import Connection as Postgres
+                    self._db = Postgres(host, port, username, password, database_name)
+                except ImportError:
+                    self.print('psycopg could not be found', name='stderr')
+
+            # Otherwise the provided path is used to create an
+            # in-process instance.
+            else:
+                # By default, we try to load DuckDB.
+                try:
+                    from .db.implementation.duckdb import Connection as DuckDB
+                    self._db = DuckDB(path)
+
+                # If DuckDB is not installed or fails to load,
+                # we use SQLite instead.
+                except ImportError:
+                    self.print('DuckDB is not available\n')
+
+                    from .db.implementation.sqlite import Connection as SQLite
+                    self._db = SQLite(path)
+
             return True
         else:
             return False
@@ -107,7 +142,12 @@ class DuckDBKernel(Kernel):
 
         # execute query and store start and end timestamp
         st = time.time()
-        columns, rows = self._db.execute(query)
+
+        try:
+            columns, rows = self._db.execute(query)
+        except EmptyResultError:
+            columns, rows = None, None
+
         et = time.time()
 
         # return result if silent
@@ -115,7 +155,7 @@ class DuckDBKernel(Kernel):
             return columns, rows
 
         # print EXPLAIN queries as raw text if using DuckDB
-        if query.strip().startswith('EXPLAIN') and not SQLITE_MODE:
+        if query.strip().startswith('EXPLAIN') and self._db.plain_explain():
             for ekey, evalue in rows:
                 self.print_data(f'<b>{ekey}</b><br><pre>{evalue}</pre>')
 
@@ -123,34 +163,38 @@ class DuckDBKernel(Kernel):
 
         # print every other query as a table
         else:
-            # table header
-            table_header = ''.join(f'<th>{c}</th>' for c in columns)
-
-            # table data
-            if max_rows is not None and len(rows) > max_rows:
-                table_data = f'''
-                    {rows_table(rows[:math.ceil(max_rows / 2)])}
-                    <tr>
-                        <td colspan="{len(columns)}" 
-                            style="text-align: center"
-                            title="{row_count(len(rows) - max_rows)} omitted">
-                            ...
-                        </td>
-                    </tr>
-                    {rows_table(rows[-math.floor(max_rows // 2):])}
-                '''
-            else:
-                table_data = rows_table(rows)
-
-            # send to client
-            self.print_data(f'''
-                <table class="duckdb-query-result">
-                    {table_header}
-                    {table_data}
-                </table>
-            ''')
+            if columns is not None:
+                # table header
+                table_header = ''.join(f'<th>{c}</th>' for c in columns)
+
+                # table data
+                if max_rows is not None and len(rows) > max_rows:
+                    table_data = f'''
+                        {rows_table(rows[:math.ceil(max_rows / 2)])}
+                        <tr>
+                            <td colspan="{len(columns)}" 
+                                style="text-align: center"
+                                title="{row_count(len(rows) - max_rows)} omitted">
+                                ...
+                            </td>
+                        </tr>
+                        {rows_table(rows[-math.floor(max_rows // 2):])}
+                    '''
+                else:
+                    table_data = rows_table(rows)
+
+                # send to client
+                self.print_data(f'''
+                    <table class="duckdb-query-result">
+                        {table_header}
+                        {table_data}
+                    </table>
+                ''')
+
+                self.print_data(f'{row_count(len(rows))} in {et - st:.3f}s')
 
-            self.print_data(f'{row_count(len(rows))} in {et - st:.3f}s')
+            else:
+                self.print_data(f'statement executed without result in {et - st:.3f}s')
 
         return columns, rows
 
@@ -167,12 +211,8 @@ class DuckDBKernel(Kernel):
             if not silent:
                 self.print('unloaded database\n')
 
-        # print kernel version
-        if not silent:
-            if not SQLITE_MODE:
-                self.print(f'{self.implementation} {self.implementation_version}\n')
-            else:
-                self.print('SQLite (DuckDB is not installed)\n')
+        # No user cares about the kernel version,
+        # so I removed the printing.
 
         # clean path
         if path.startswith(("'", '"')):
@@ -186,7 +226,8 @@ class DuckDBKernel(Kernel):
 
         if self._load_database(path):
             if not silent:
-                self.print(f'loaded database "{path}"\n')
+                # self.print(f'loaded database "{path}"\n')
+                self.print(str(self._db))
 
         # copy data from source database
         if of is not None:
@@ -202,12 +243,12 @@ class DuckDBKernel(Kernel):
                     content = file.read()
 
                     # You can only execute one statement at a time using SQLite.
-                    if SQLITE_MODE:
+                    if not self._db.multiple_statements_per_query():
                         statements = re.split(r';\r?\n', content)
                         for statement in statements:
                             self._db.execute(statement)
 
-                    # DuckDB can execute multiple statements at a time.
+                    # Other DBMS can execute multiple statements at a time.
                     else:
                         self._db.execute(content)
 

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/duckdb_kernel/parser/elements/RAElement.py

@@ -35,9 +35,9 @@ class RAElement:
 
         # if all columns are from the same relation we can skip the relation name
         if len(set(c.table for c in columns)) == 1:
-            column_names = ', '.join(f"{c.current_name} AS '{c.name}'" for c in columns)
+            column_names = ', '.join(f'{c.current_name} AS "{c.name}"' for c in columns)
         else:
-            column_names = ', '.join(f"{c.current_name} AS '{c.full_name}'" for c in columns)
+            column_names = ', '.join(f'{c.current_name} AS "{c.full_name}"' for c in columns)
 
         return f'SELECT {column_names} FROM ({sql}) {self._name()}'
 

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/duckdb_kernel/parser/elements/binary/ConditionalSet.py

@@ -334,10 +334,10 @@ class ConditionalSet:
             target_table_stmt = table_statements[target_name]
             sql_tables += f' LEFT JOIN {target_table_stmt} ON {join_condition}'
 
-        sql_join_filters = '1'
+        sql_join_filters = '1=1'
         for _, _, _, join_filter in sorted_negative_joins:
             sql_join_filters += f' AND {join_filter}'
 
-        sql_condition = condition.to_sql(joined_columns) if condition is not None else 1
+        sql_condition = condition.to_sql(joined_columns) if condition is not None else '1=1'
 
         return f'SELECT DISTINCT {sql_select} FROM {sql_tables} WHERE ({sql_join_filters}) AND ({sql_condition})'

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/jupyter_duckdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jupyter-duckdb
-Version: 0.9.2.2.dev202401171418
+Version: 0.9.2.3
 Summary: a basic wrapper kernel for DuckDB
 Home-page: https://github.com/erictroebs/jupyter-duckdb
 Author: Eric Tröbs

{jupyter-duckdb-0.9.2.2.dev202401171418 → jupyter-duckdb-0.9.2.3}/src/jupyter_duckdb.egg-info/SOURCES.txt

@@ -11,10 +11,14 @@ src/duckdb_kernel/db/DatabaseError.py
 src/duckdb_kernel/db/ForeignKey.py
 src/duckdb_kernel/db/Table.py
 src/duckdb_kernel/db/__init__.py
-src/duckdb_kernel/db/duckdb/Connection.py
-src/duckdb_kernel/db/duckdb/__init__.py
-src/duckdb_kernel/db/sqlite/Connection.py
-src/duckdb_kernel/db/sqlite/__init__.py
+src/duckdb_kernel/db/error/EmptyResultError.py
+src/duckdb_kernel/db/error/__init__.py
+src/duckdb_kernel/db/implementation/duckdb/Connection.py
+src/duckdb_kernel/db/implementation/duckdb/__init__.py
+src/duckdb_kernel/db/implementation/postgres/Connection.py
+src/duckdb_kernel/db/implementation/postgres/__init__.py
+src/duckdb_kernel/db/implementation/sqlite/Connection.py
+src/duckdb_kernel/db/implementation/sqlite/__init__.py
 src/duckdb_kernel/magics/MagicCommand.py
 src/duckdb_kernel/magics/MagicCommandCallback.py
 src/duckdb_kernel/magics/MagicCommandException.py