sqlutilpy 0.22.0__tar.gz → 0.24.0__tar.gz

{sqlutilpy-0.22.0/src/sqlutilpy.egg-info → sqlutilpy-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: sqlutilpy
-Version: 0.22.0
+Version: 0.24.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -9,12 +9,13 @@ Keywords: numpy postgresql query sql sqlite array
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Topic :: Utilities
-Requires-Python: >=3.6
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: numpy
-Requires-Dist: psycopg2-binary
+Requires-Dist: psycopg[binary]
 Requires-Dist: numpydoc
+Requires-Dist: numpy
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)

{sqlutilpy-0.22.0 → sqlutilpy-0.24.0}/pyproject.toml

@@ -7,10 +7,11 @@ authors = [
 ]
 dependencies = [
     "numpy",
-    "psycopg2-binary",
+    "psycopg[binary]",
     "numpydoc",
+    "numpy\n"
 ]
-requires-python = ">=3.6"
+requires-python = ">=3.8"
 readme = "README.md"
 license = {text = "BSD"}
 keywords = ["numpy postgresql query sql sqlite array"]

{sqlutilpy-0.22.0 → sqlutilpy-0.24.0}/src/sqlutilpy/sqlutil.py

@@ -3,13 +3,11 @@
 from __future__ import print_function
 import numpy
 import numpy as np
-import psycopg2
+import psycopg
 import threading
 import collections
 import warnings
 from numpy.core import numeric as sb
-from select import select
-from psycopg2.extensions import POLL_OK, POLL_READ, POLL_WRITE
 
 try:
     import astropy.table as atpy
@@ -22,7 +20,6 @@ except ImportError:
     # pandas is not installed
     pandas = None
 
-from io import BytesIO as StringIO
 import queue
 
 _WAIT_SELECT_TIMEOUT = 10
@@ -37,31 +34,6 @@ class SqlUtilException(Exception):
     pass
 
 
-def __wait_select_inter(conn):
-    """ Make the queries interruptible by Ctrl-C
-
-    Taken from http://initd.org/psycopg/articles/2014/07/20/cancelling-postgresql-statements-python/ # noqa
-    """
-    while True:
-        try:
-            state = conn.poll()
-            if state == POLL_OK:
-                break
-            elif state == POLL_READ:
-                select([conn.fileno()], [], [], _WAIT_SELECT_TIMEOUT)
-            elif state == POLL_WRITE:
-                select([], [conn.fileno()], [], _WAIT_SELECT_TIMEOUT)
-            else:
-                raise conn.OperationalError("bad state from poll: %s" % state)
-        except KeyboardInterrupt:
-            conn.cancel()
-            # the loop will be broken by a server error
-            continue
-
-
-psycopg2.extensions.set_wait_callback(__wait_select_inter)
-
-
 def getConnection(db=None,
                   driver=None,
                   user=None,
@@ -80,7 +52,7 @@ def getConnection(db=None,
         The name of the database (in case of PostgreSQL) or filename in
         case of sqlite db
     driver :  string
-        The db driver (either 'psycopg2' or 'sqlite3')
+        The db driver (either 'psycopg' or 'sqlite3')
     user : string, optional
         Username
     password: string, optional
@@ -99,6 +71,10 @@ def getConnection(db=None,
 
     """
     if driver == 'psycopg2':
+        warnings.warn(
+            'psycopg2 driver is not supported anymore using psycopg instead')
+        driver = 'psycopg'
+    if driver == 'psycopg':
         conn_dict = dict()
         if db is not None:
             conn_dict['dbname'] = db
@@ -110,7 +86,7 @@ def getConnection(db=None,
             conn_dict['user'] = user
         if password is not None:
             conn_dict['password'] = password
-        conn = psycopg2.connect(**conn_dict)
+        conn = psycopg.connect(**conn_dict)
     elif driver == 'sqlite3':
         import sqlite3
         if timeout is None:
@@ -126,6 +102,10 @@ def getCursor(conn, driver=None, preamb=None, notNamed=False):
     Retrieve the database cursor
     """
     if driver == 'psycopg2':
+        warnings.warn(
+            'psycopg2 driver is not supported anymore using psycopg instead')
+        driver = 'psycopg'
+    if driver == 'psycopg':
         cur = conn.cursor()
         if preamb is not None:
             cur.execute(preamb)
@@ -267,7 +247,7 @@ def __getDType(row, typeCodes, strLength):
 def get(query,
         params=None,
         db="wsdb",
-        driver="psycopg2",
+        driver="psycopg",
         user=None,
         password=None,
         host=None,
@@ -304,7 +284,7 @@ def get(query,
     db : string
         The name of the database
     driver : string, optional
-        The sql driver to be used (psycopg2 or sqlite3)
+        The sql driver to be used (psycopg or sqlite3)
     user : string, optional
         User name for the DB connection
     password : string, optional
@@ -359,26 +339,38 @@ def get(query,
         proc = None
         colNames = []
         if driver == 'psycopg2':
+            warnings.warn('psycopg2 driver is not supported anymore. '
+                          'We using psycopg instead')
+            driver = 'psycopg'
+        if driver == 'psycopg':
             try:
-                while (True):
+                while True:
                     # Iterating over the cursor, retrieving batches of results
                     # and then sending them for conversion
                     tups = cur.fetchmany()
                     desc = cur.description
 
+                    # If the is just the start we need to launch the
+                    # thread doing the conversion
+                    no_results = tups == []
+                    if nrec == 0:
+                        typeCodes = [_tmp.type_code for _tmp in desc]
+                        colNames = [_tmp.name for _tmp in cur.description]
+
                     # No more data
-                    if tups == []:
+                    if no_results:
+                        dtype = __getDType([None] * len(typeCodes), typeCodes,
+                                           strLength)
                         break
 
+                    dtype = __getDType(tups[0], typeCodes, strLength)
+
                     # Send the new batch for conversion
                     qIn.put(tups)
 
                     # If the is just the start we need to launch the
                     # thread doing the conversion
                     if nrec == 0:
-                        typeCodes = [_tmp.type_code for _tmp in desc]
-                        colNames = [_tmp.name for _tmp in cur.description]
-                        dtype = __getDType(tups[0], typeCodes, strLength)
                         proc = threading.Thread(target=__converter,
                                                 args=(qIn, qOut, endEvent,
                                                       dtype, intNullVal))
@@ -473,7 +465,7 @@ def get(query,
 def execute(query,
             params=None,
             db='wsdb',
-            driver="psycopg2",
+            driver="psycopg",
             user=None,
             password=None,
             host=None,
@@ -493,7 +485,7 @@ def execute(query,
     db : string
         Database name
     driver : string
-        Driver for the DB connection ('psycopg2' or 'sqlite3')
+        Driver for the DB connection ('psycopg' or 'sqlite3')
     user : string, optional
         user name for the DB connection
     password : string, optional
@@ -542,7 +534,7 @@ def __create_schema(tableName, arrays, names, temp=False):
                  (np.uint64, 'bigint'), (np.int16, 'smallint'),
                  (np.uint8, 'smallint'), (np.int8, 'smallint'),
                  (np.float32, 'real'), (np.float64, 'double precision'),
-                 (np.string_, 'varchar'), (np.str_, 'varchar'),
+                 (np.bytes_, 'varchar'), (np.str_, 'varchar'),
                  (np.bool_, 'boolean'), (np.datetime64, 'timestamp')])
     if temp:
         temp = 'temporary'
@@ -568,7 +560,7 @@ def __print_arrays(arrays, f, delimiter=' '):
     format_dict = dict([(np.int32, '%d'), (np.int64, '%d'), (np.int16, '%d'),
                         (np.int8, '%d'), (np.uint8, '%d'),
                         (np.float32, '%.18e'), (np.float64, '%.18e'),
-                        (np.string_, '%s'), (np.str_, '%s'),
+                        (np.bytes_, '%s'), (np.str_, '%s'),
                         (np.datetime64, '%s'), (np.bool_, '%d')])
     fmts = []
     array_mode = False
@@ -626,7 +618,7 @@ def upload(tableName,
            arrays,
            names=None,
            db="wsdb",
-           driver="psycopg2",
+           driver="psycopg",
            user=None,
            password=None,
            host=None,
@@ -653,7 +645,7 @@ def upload(tableName,
     db: string
          Databas name
     driver: string
-         Python database driver "psycopg2",
+         Python database driver "psycopg",
     user: string,
     password: string
     host: string
@@ -743,18 +735,17 @@ table/pandas/dictionary or provide a separate list of arrays and their names')
             cur.execute(query1)
         nsplit = 100000
         N = len(arrays[0])
+        names = ','.join(names)
         for i in range(0, N, nsplit):
-            f = StringIO()
-            __print_arrays([_[i:i + nsplit] for _ in arrays],
-                           f,
-                           delimiter=delimiter)
-            f.seek(0)
             try:
-                thread = psycopg2.extensions.get_wait_callback()
-                psycopg2.extensions.set_wait_callback(None)
-                cur.copy_from(f, tableName, sep=delimiter, columns=names)
+                with cur.copy(f'''copy {tableName}({names}) from stdin
+                        with delimiter '{delimiter}' ''') as copy:
+                    __print_arrays([_[i:i + nsplit] for _ in arrays],
+                                   copy,
+                                   delimiter=delimiter)
+
             finally:
-                psycopg2.extensions.set_wait_callback(thread)
+                pass
     except BaseException:
         failure_cleanup(conn, connSupplied)
         raise
@@ -772,7 +763,7 @@ def local_join(query,
                arrays,
                names,
                db=None,
-               driver="psycopg2",
+               driver="psycopg",
                user=None,
                password=None,
                host=None,

sqlutilpy-0.24.0/src/sqlutilpy/version.py

@@ -0,0 +1 @@
+__version__ = '0.24.0'

{sqlutilpy-0.22.0 → sqlutilpy-0.24.0/src/sqlutilpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: sqlutilpy
-Version: 0.22.0
+Version: 0.24.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -9,12 +9,13 @@ Keywords: numpy postgresql query sql sqlite array
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Topic :: Utilities
-Requires-Python: >=3.6
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: numpy
-Requires-Dist: psycopg2-binary
+Requires-Dist: psycopg[binary]
 Requires-Dist: numpydoc
+Requires-Dist: numpy
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)

sqlutilpy-0.24.0/src/sqlutilpy.egg-info/requires.txt

@@ -0,0 +1,4 @@
+numpy
+psycopg[binary]
+numpydoc
+numpy

{sqlutilpy-0.22.0 → sqlutilpy-0.24.0}/tests/test_pg.py

@@ -1,7 +1,7 @@
 import pytest
 import string
 import os
-import psycopg2
+import psycopg
 import numpy as np
 import sqlutilpy as sqlutil
 import time
@@ -16,7 +16,7 @@ PG_PASS = os.environ['SQLUTIL_TEST_PG_PASS']
 def getrand(N, float=False):
     # simple deterministic pseudo-random number generator
     a, c = 1103515245, 12345
-    m = 2 ^ 31
+    m = 2**31
     arr = np.zeros(N)
     arr[0] = 1
     for i in range(1, N):
@@ -37,8 +37,8 @@ def getconn():
 
 @pytest.fixture
 def setup():
-    conn = psycopg2.connect('dbname=%s user=%s host=%s password=%s' %
-                            (PG_DB, PG_USER, PG_HOST, PG_PASS))
+    conn = psycopg.connect('dbname=%s user=%s host=%s password=%s' %
+                           (PG_DB, PG_USER, PG_HOST, PG_PASS))
     cur = conn.cursor()
     cur.execute('''
     create unlogged table sqlutil_test (sicol smallint, intcol int,
@@ -123,11 +123,14 @@ def test_nocommit(setup):
 
 def test_local_join(setup):
     kw, conn = setup
+    N = 1000
+    rands = np.random.uniform(size=N)
     R, = sqlutil.local_join(
         '''
-    select s.sicol from sqlutil_test as s,  mytab as m
-    where s.sicol = m.id''', 'mytab', [np.arange(10)], ['id'], **kw)
-    assert (len(R) == 1)
+    select xf from sqlutil_test as s,  mytab as m
+    where s.sicol = m.id''', 'mytab', [np.arange(N), rands], ['id', 'xf'],
+        **kw)
+    assert (len(R) == 2)
 
 
 def test_big(setup):
@@ -252,6 +255,12 @@ def test_get_dict(setup):
                      **kw)
     for i, k in enumerate(cols.split(',')):
         assert ((Rd[k] == R0[i]).all())
+    Rempty = sqlutil.get('select %s from sqlutil_test where sicol<-10000' %
+                         (cols, ),
+                         asDict=True,
+                         **kw)
+    for i, k in enumerate(cols.split(',')):
+        assert k in Rempty
 
 
 def test_get_dict_rep(setup):
@@ -392,10 +401,27 @@ def test_upload(setup):
     # try the weird names
     sqlutil.upload(mytab1, (xi16, xi32, xi64, xf32, xf64, xbool),
                    ('xi 16', 'xi(32)', 'xi[64]', 'xf32', 'xf64', 'xbool'),
-                   **kw)
+                   temp=True,
+                   conn=conn)
     yi16, yi32, yi64, yf32, yf64, ybool = sqlutil.get(
         '''select xi_16,xi_32_,xi_64_,xf32,xf64,xbool from %s''' % (mytab1),
-        **kw)
+        conn=conn)
+
+
+def test_upload_big(setup):
+    kw, conn = setup
+    mytab = 'sqlutil_test_tab_big'
+    nrows = 1_000_000
+    xi = np.arange(nrows, dtype=np.int32)
+    xf = getrand(nrows, True)
+    sqlutil.upload(mytab, (xi, xf), ('xi', 'xf'), **kw)
+    yi, yf = sqlutil.get('''select xi,xf from %s order by xi''' % (mytab),
+                         **kw)
+    try:
+        assert ((xi == yi).all())
+        assert (np.allclose(xf, yf))
+    finally:
+        sqlutil.execute('drop table %s' % mytab, **kw)
 
 
 def test_upload_array(setup):

{sqlutilpy-0.22.0 → sqlutilpy-0.24.0}/tests/test_sqlite.py

@@ -1,12 +1,7 @@
 import pytest
-import string
 import os
-import psycopg2
-import numpy as np
 import sqlutilpy as sqlutil
 import sqlite3
-import time
-import killer
 
 
 @pytest.fixture()

sqlutilpy-0.22.0/src/sqlutilpy/version.py

@@ -1 +0,0 @@
-__version__ = '0.22.0'

sqlutilpy-0.22.0/src/sqlutilpy.egg-info/requires.txt

@@ -1,3 +0,0 @@
-numpy
-psycopg2-binary
-numpydoc