sqlutilpy 0.23.0__tar.gz → 0.25.0__tar.gz

{sqlutilpy-0.23.0/src/sqlutilpy.egg-info → sqlutilpy-0.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.23.0
+Version: 0.25.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -16,6 +16,7 @@ Requires-Dist: numpy
 Requires-Dist: psycopg[binary]
 Requires-Dist: numpydoc
 Requires-Dist: numpy
+Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
@@ -25,8 +26,8 @@ Requires-Dist: numpy
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
 tables and run join queries involving local arrays and the tables in the DB.
-This module is optimized to be able to deal efficiently with query results with millions of rows.
-The module only works with PostgreSQL, SQLite and DuckDB databases.
+This module is optimized to deal efficiently with query results with millions of rows.
+The module works with PostgreSQL, SQLite and DuckDB databases.
 
 The full documentation is available [here](http://sqlutilpy.readthedocs.io/en/latest/)
 
@@ -40,8 +41,9 @@ pip install sqlutilpy
 ```
 ## Authentication
 
-Throughout this readme, I will assume that the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
-has been created with your login/password details for Postgresql. If that is not the case, all of the 
+Throughout this readme, I will assume that if you are using PostgreSQL, then
+the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
+has been created with your login/password details for Postgresql. If that is not the case, many of the 
 commands given below will also need user='....' and password='...' options
 
 ## Connection information

{sqlutilpy-0.23.0 → sqlutilpy-0.25.0}/README.md

@@ -6,8 +6,8 @@
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
 tables and run join queries involving local arrays and the tables in the DB.
-This module is optimized to be able to deal efficiently with query results with millions of rows.
-The module only works with PostgreSQL, SQLite and DuckDB databases.
+This module is optimized to deal efficiently with query results with millions of rows.
+The module works with PostgreSQL, SQLite and DuckDB databases.
 
 The full documentation is available [here](http://sqlutilpy.readthedocs.io/en/latest/)
 
@@ -21,8 +21,9 @@ pip install sqlutilpy
 ```
 ## Authentication
 
-Throughout this readme, I will assume that the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
-has been created with your login/password details for Postgresql. If that is not the case, all of the 
+Throughout this readme, I will assume that if you are using PostgreSQL, then
+the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
+has been created with your login/password details for Postgresql. If that is not the case, many of the 
 commands given below will also need user='....' and password='...' options
 
 ## Connection information

{sqlutilpy-0.23.0 → sqlutilpy-0.25.0}/src/sqlutilpy/sqlutil.py

@@ -92,6 +92,9 @@ def getConnection(db=None,
         if timeout is None:
             timeout = 5
         conn = sqlite3.connect(db, timeout=timeout)
+    elif driver == 'duckdb':
+        import duckdb
+        conn = duckdb.connect(db)
     else:
         raise Exception("Unknown driver")
     return conn
@@ -137,12 +140,12 @@ def __fromrecords(recList, dtype=None, intNullVal=None):
     """
 
     shape = None
-    descr = sb.dtype((np.core.records.record, dtype))
+    descr = sb.dtype((np.record, dtype))
     try:
         retval = sb.array(recList, dtype=descr)
     except TypeError:  # list of lists instead of list of tuples
         shape = (len(recList), )
-        _array = np.core.records.recarray(shape, descr)
+        _array = np.recarray(shape, descr)
         try:
             for k in range(_array.size):
                 _array[k] = tuple(recList[k])
@@ -170,7 +173,7 @@ def __fromrecords(recList, dtype=None, intNullVal=None):
         if shape is not None and retval.shape != shape:
             retval.shape = shape
 
-    res = retval.view(numpy.core.records.recarray)
+    res = retval.view(numpy.recarray)
 
     return res
 
@@ -210,6 +213,7 @@ def __getDType(row, typeCodes, strLength):
         1022: 'f8',
         1700: 'f8',  # numeric
         1114: '<M8[us]',  # timestamp
+        1184: '<M8[us]',  # timestamp with timezone
         1082: '<M8[us]',  # date
         25: '|U%d',
         1042: '|U%d',  # character()
@@ -339,31 +343,38 @@ def get(query,
         proc = None
         colNames = []
         if driver == 'psycopg2':
-            warnings.warn(
-                'psycopg2 driver is not supported anymore using psycopg instead'
-            )
+            warnings.warn('psycopg2 driver is not supported anymore. '
+                          'We using psycopg instead')
             driver = 'psycopg'
         if driver == 'psycopg':
             try:
-                while (True):
+                while True:
                     # Iterating over the cursor, retrieving batches of results
                     # and then sending them for conversion
                     tups = cur.fetchmany()
                     desc = cur.description
 
+                    # If the is just the start we need to launch the
+                    # thread doing the conversion
+                    no_results = tups == []
+                    if nrec == 0:
+                        typeCodes = [_tmp.type_code for _tmp in desc]
+                        colNames = [_tmp.name for _tmp in cur.description]
+
                     # No more data
-                    if tups == []:
+                    if no_results:
+                        dtype = __getDType([None] * len(typeCodes), typeCodes,
+                                           strLength)
                         break
 
+                    dtype = __getDType(tups[0], typeCodes, strLength)
+
                     # Send the new batch for conversion
                     qIn.put(tups)
 
                     # If the is just the start we need to launch the
                     # thread doing the conversion
                     if nrec == 0:
-                        typeCodes = [_tmp.type_code for _tmp in desc]
-                        colNames = [_tmp.name for _tmp in cur.description]
-                        dtype = __getDType(tups[0], typeCodes, strLength)
                         proc = threading.Thread(target=__converter,
                                                 args=(qIn, qOut, endEvent,
                                                       dtype, intNullVal))
@@ -416,14 +427,14 @@ def get(query,
             tups = cur.fetchall()
             colNames = [_tmp[0] for _tmp in cur.description]
             if len(tups) > 0:
-                res = numpy.core.records.array(tups)
+                res = numpy.rec.array(tups)
             else:
                 return [[]] * len(cur.description)
         elif driver == 'duckdb':
             tups = cur.fetchall()
             colNames = [_tmp[0] for _tmp in cur.description]
             if len(tups) > 0:
-                res = numpy.core.records.array(tups)
+                res = numpy.rec.array(tups)
             else:
                 return [[]] * len(cur.description)
 
@@ -731,9 +742,8 @@ table/pandas/dictionary or provide a separate list of arrays and their names')
         names = ','.join(names)
         for i in range(0, N, nsplit):
             try:
-                with cur.copy(
-                        f'''copy {tableName}({names}) from stdin with delimiter '{delimiter}' '''
-                ) as copy:
+                with cur.copy(f'''copy {tableName}({names}) from stdin
+                        with delimiter '{delimiter}' ''') as copy:
                     __print_arrays([_[i:i + nsplit] for _ in arrays],
                                    copy,
                                    delimiter=delimiter)

sqlutilpy-0.25.0/src/sqlutilpy/version.py

@@ -0,0 +1 @@
+__version__ = '0.25.0'

{sqlutilpy-0.23.0 → sqlutilpy-0.25.0/src/sqlutilpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.23.0
+Version: 0.25.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -16,6 +16,7 @@ Requires-Dist: numpy
 Requires-Dist: psycopg[binary]
 Requires-Dist: numpydoc
 Requires-Dist: numpy
+Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
@@ -25,8 +26,8 @@ Requires-Dist: numpy
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
 tables and run join queries involving local arrays and the tables in the DB.
-This module is optimized to be able to deal efficiently with query results with millions of rows.
-The module only works with PostgreSQL, SQLite and DuckDB databases.
+This module is optimized to deal efficiently with query results with millions of rows.
+The module works with PostgreSQL, SQLite and DuckDB databases.
 
 The full documentation is available [here](http://sqlutilpy.readthedocs.io/en/latest/)
 
@@ -40,8 +41,9 @@ pip install sqlutilpy
 ```
 ## Authentication
 
-Throughout this readme, I will assume that the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
-has been created with your login/password details for Postgresql. If that is not the case, all of the 
+Throughout this readme, I will assume that if you are using PostgreSQL, then
+the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
+has been created with your login/password details for Postgresql. If that is not the case, many of the 
 commands given below will also need user='....' and password='...' options
 
 ## Connection information

{sqlutilpy-0.23.0 → sqlutilpy-0.25.0}/src/sqlutilpy.egg-info/SOURCES.txt

@@ -11,5 +11,6 @@ src/sqlutilpy.egg-info/dependency_links.txt
 src/sqlutilpy.egg-info/requires.txt
 src/sqlutilpy.egg-info/top_level.txt
 tests/killer.py
+tests/test_duckdb.py
 tests/test_pg.py
 tests/test_sqlite.py

sqlutilpy-0.25.0/tests/test_duckdb.py

@@ -0,0 +1,32 @@
+import pytest
+import os
+import sqlutilpy as sqlutil
+import duckdb
+
+
+@pytest.fixture()
+def setup():
+    fname = 'duck.db'
+    kw = dict(db=fname, driver='duckdb')
+    conn = duckdb.connect(fname)
+    cur = conn.cursor()
+    cur.execute('create table tmp (a int, b double precision)')
+    cur.execute('insert into tmp values(1,2), (2,3), (3,4), (4,5);')
+    conn.commit()
+    yield kw
+    os.unlink(fname)
+
+
+def test_simple(setup):
+    a, b = sqlutil.get('select a,b from tmp', **setup)
+    assert (len(a) == 4)
+
+
+def test_preamb(setup):
+    a, b = sqlutil.get('select a,b from tmp', preamb='select 1', **setup)
+    assert (len(a) == 4)
+
+
+def test_empty(setup):
+    a, b = sqlutil.get('select a,b from tmp where a<0', **setup)
+    assert (len(a) == 0)

{sqlutilpy-0.23.0 → sqlutilpy-0.25.0}/tests/test_pg.py

@@ -255,6 +255,12 @@ def test_get_dict(setup):
                      **kw)
     for i, k in enumerate(cols.split(',')):
         assert ((Rd[k] == R0[i]).all())
+    Rempty = sqlutil.get('select %s from sqlutil_test where sicol<-10000' %
+                         (cols, ),
+                         asDict=True,
+                         **kw)
+    for i, k in enumerate(cols.split(',')):
+        assert k in Rempty
 
 
 def test_get_dict_rep(setup):

sqlutilpy-0.23.0/src/sqlutilpy/version.py

@@ -1 +0,0 @@
-__version__ = '0.23.0'