sqlutilpy 0.27.0__tar.gz → 0.29.0__tar.gz

{sqlutilpy-0.27.0/src/sqlutilpy.egg-info → sqlutilpy-0.29.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.27.0
+Version: 0.29.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -14,8 +14,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: numpy
 Requires-Dist: psycopg[binary]
-Requires-Dist: numpydoc
-Requires-Dist: numpy
 Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)

{sqlutilpy-0.27.0 → sqlutilpy-0.29.0}/pyproject.toml

@@ -7,9 +7,7 @@ authors = [
 ]
 dependencies = [
     "numpy",
-    "psycopg[binary]",
-    "numpydoc",
-    "numpy\n"
+    "psycopg[binary]"
 ]
 requires-python = ">=3.8"
 readme = "README.md"

{sqlutilpy-0.27.0 → sqlutilpy-0.29.0}/src/sqlutilpy/sqlutil.py

@@ -5,6 +5,7 @@ import numpy as np
 import psycopg
 import collections
 import warnings
+import uuid
 import concurrent.futures
 
 try:
@@ -30,6 +31,26 @@ class SqlUtilException(Exception):
     pass
 
 
+SUPPORTED_DRIVERS = ('psycopg', 'sqlite3', 'duckdb')
+
+
+def __check_driver(driver):
+    """
+    Validate the driver name and return its canonical form.
+    The obsolete 'psycopg2' is accepted with a warning and mapped to
+    'psycopg'; any other unknown driver raises SqlUtilException
+    """
+    if driver == 'psycopg2':
+        warnings.warn(
+            'psycopg2 driver is not supported anymore using psycopg instead')
+        driver = 'psycopg'
+    if driver not in SUPPORTED_DRIVERS:
+        raise SqlUtilException(
+            'Unknown driver %r; supported drivers are %s' %
+            (driver, ', '.join(SUPPORTED_DRIVERS)))
+    return driver
+
+
 def getConnection(db=None,
                   driver=None,
                   user=None,
@@ -66,10 +87,7 @@ def getConnection(db=None,
          Database Connection
 
     """
-    if driver == 'psycopg2':
-        warnings.warn(
-            'psycopg2 driver is not supported anymore using psycopg instead')
-        driver = 'psycopg'
+    driver = __check_driver(driver)
     if driver == 'psycopg':
         conn_dict = dict()
         if db is not None:
@@ -91,8 +109,6 @@ def getConnection(db=None,
     elif driver == 'duckdb':
         import duckdb
         conn = duckdb.connect(db)
-    else:
-        raise Exception("Unknown driver")
     return conn
 
 
@@ -100,10 +116,7 @@ def getCursor(conn, driver=None, preamb=None, named=True):
     """
     Retrieve the database cursor
     """
-    if driver == 'psycopg2':
-        warnings.warn(
-            'psycopg2 driver is not supported anymore using psycopg instead')
-        driver = 'psycopg'
+    driver = __check_driver(driver)
     if driver == 'psycopg':
         cur = conn.cursor(scrollable=False)
         if preamb is not None:
@@ -114,7 +127,9 @@ def getCursor(conn, driver=None, preamb=None, named=True):
             # different plan
         if not named:
             return cur
-        cur = conn.cursor(name='sqlutilcursor')
+        # the unique cursor name allows several batched queries
+        # to run over the same connection
+        cur = conn.cursor(name='sqlutilcursor_' + uuid.uuid4().hex)
         cur.arraysize = config.arraysize
     elif driver == 'sqlite3':
         cur = conn.cursor()
@@ -124,8 +139,6 @@ def getCursor(conn, driver=None, preamb=None, named=True):
         cur = conn.cursor()
         if preamb is not None:
             cur.execute(preamb)
-    else:
-        raise SqlUtilException('unrecognized driver')
     return cur
 
 
@@ -187,6 +200,7 @@ def __fromrecords(recList, dtype=None, intNullVal=None, strNullVal='None'):
             arr[k] = tuple(recList[k])
 
     # Apply conversions
+    keep_object = []
     for name, val in converters.items():
         col = arr[name]
         # Check for None efficiently
@@ -194,6 +208,16 @@ def __fromrecords(recList, dtype=None, intNullVal=None, strNullVal='None'):
         if np.any(mask):
             if val is not None:
                 col[mask] = val
+            else:
+                # no null replacement value was given, so we keep
+                # the column as an object column with Nones in it
+                keep_object.append(name)
+
+    if keep_object:
+        descr = np.dtype((np.record, [
+            (_n, object if _n in keep_object else dtype.fields[_n][0])
+            for _n in names
+        ]))
 
     # Cast to final dtype
     retval = arr.astype(descr)
@@ -208,7 +232,6 @@ def __getDType(row, typeCodes, strLength):
         20: 'i8',
         21: 'i2',
         23: 'i4',
-        1007: 'i4',
         700: 'f4',
         701: 'f8',
         1000: bool,
@@ -281,6 +304,70 @@ def __get_representative_row(batch, type_codes):
     return first_row
 
 
+def __fix_object_column(col, intNullVal, strNullVal):
+    """
+    Convert an object column into a typed numpy array if possible,
+    replacing the Nones in integer/string columns by intNullVal/strNullVal
+    and by NaNs in float columns.
+    The columns that cannot be converted are returned unchanged.
+    """
+    # Find first non-None value to guess type
+    # (we avoid col != None here as that fails for columns
+    # containing arrays)
+    valid_mask = np.fromiter((_tmp is not None for _tmp in col),
+                             dtype=bool,
+                             count=len(col))
+    if not np.any(valid_mask):
+        # All None.
+        # We can't determine type. Return as is (Object filled with
+        # None)
+        return col
+
+    # Take a sample (first valid)
+    first_val = col[valid_mask][0]
+
+    # INTEGER
+    if isinstance(
+            first_val,
+        (int, np.integer)) and not isinstance(first_val, (bool, np.bool_)):
+        # Verify all valid are integers
+        valid_vals = col[valid_mask]
+        is_valid = all(
+            isinstance(x, (int, np.integer))
+            and not isinstance(x, (bool, np.bool_)) for x in valid_vals)
+        if is_valid:
+            # Convert
+            temp_col = np.copy(col)
+            temp_col[~valid_mask] = intNullVal
+            try:
+                return temp_col.astype(int)
+            except Exception:
+                pass
+
+    # FLOAT
+    if isinstance(first_val, (float, np.floating)):
+        if all(
+                isinstance(x, (float, np.floating, int, np.integer))
+                for x in col[valid_mask]):
+            temp_col = np.copy(col)
+            temp_col[~valid_mask] = np.nan
+            try:
+                return temp_col.astype(float)
+            except Exception:
+                pass
+
+    # STRING
+    if isinstance(first_val, str):
+        if all(isinstance(x, str) for x in col[valid_mask]):
+            # Replicate PG behavior: None -> strNullVal
+            temp_col = np.copy(col)
+            temp_col[~valid_mask] = strNullVal
+            # Let numpy infer string length
+            return np.array(list(temp_col))
+
+    return col
+
+
 def __fix_numpy_record_array(res, intNullVal, strNullVal):
     if res.size == 0:
         return res
@@ -290,76 +377,53 @@ def __fix_numpy_record_array(res, intNullVal, strNullVal):
 
     for name in names:
         col = res[name]
-        dtype = col.dtype
 
-        if dtype.kind == 'O':
-            # Check for None (vectorized)
+        if col.dtype.kind == 'O':
             # We treat everything as candidates if it's Object type
-
-            # Find first non-None value to guess type
-            valid_mask = (col != None)
-            if not np.any(valid_mask):
-                # All None.
-                # We can't determine type. Return as is (Object filled with
-                # None)
-                new_arrays.append(col)
-                continue
-
-            # Take a sample (first valid)
-            first_val = col[valid_mask][0]
-
-            # INTEGER
-            if isinstance(
-                    first_val,
-                (int, np.integer)) and not isinstance(first_val,
-                                                      (bool, np.bool_)):
-                # Verify all valid are integers
-                valid_vals = col[valid_mask]
-                is_valid = all(
-                    isinstance(x, (int, np.integer))
-                    and not isinstance(x, (bool, np.bool_))
-                    for x in valid_vals)
-                if is_valid:
-                    # Convert
-                    temp_col = np.copy(col)
-                    temp_col[~valid_mask] = intNullVal
-                    try:
-                        new_col = temp_col.astype(int)
-                        new_arrays.append(new_col)
-                        continue
-                    except Exception:
-                        pass
-
-            # FLOAT
-            if isinstance(first_val, (float, np.floating)):
-                if all(
-                        isinstance(x, (float, np.floating, int, np.integer))
-                        for x in col[valid_mask]):
-                    temp_col = np.copy(col)
-                    temp_col[~valid_mask] = np.nan
-                    try:
-                        new_col = temp_col.astype(float)
-                        new_arrays.append(new_col)
-                        continue
-                    except Exception:
-                        pass
-
-            # STRING
-            if isinstance(first_val, str):
-                if all(isinstance(x, str) for x in col[valid_mask]):
-                    # Replicate PG behavior: None -> strNullVal
-                    temp_col = np.copy(col)
-                    temp_col[~valid_mask] = strNullVal
-                    # Let numpy infer string length
-                    new_col = np.array(list(temp_col))
-                    new_arrays.append(new_col)
-                    continue
-
-        new_arrays.append(col)
+            new_arrays.append(__fix_object_column(col, intNullVal, strNullVal))
+        else:
+            new_arrays.append(col)
 
     return numpy.rec.fromarrays(new_arrays, names=names)
 
 
+def __process_duckdb_column(col, intNullVal, strNullVal):
+    """
+    Convert a single column returned by duckdb's fetchnumpy() into
+    the sqlutilpy convention, i.e. nulls in integer/string columns
+    replaced by intNullVal/strNullVal, by NaNs in float columns and
+    by NaTs in datetime columns; strings converted to fixed-width arrays
+    """
+    if isinstance(col, np.ma.MaskedArray):
+        kind = col.dtype.kind
+        if kind in ('i', 'u') and intNullVal is not None:
+            info = np.iinfo(col.dtype)
+            if not (info.min <= intNullVal <= info.max):
+                # intNullVal does not fit into the column type (e.g. a
+                # negative value for an unsigned or a tinyint column),
+                # so the column is widened to int64 first.
+                # (we check explicitly rather than catch an exception,
+                # because numpy<2 silently wraps the value instead)
+                col = col.astype(np.int64)
+            col = col.filled(intNullVal)
+        elif kind == 'f':
+            col = col.filled(np.nan)
+        elif kind == 'M':
+            col = col.filled(np.array('NaT', dtype=col.dtype))
+        else:
+            # object columns (i.e. strings), booleans and integers
+            # without a null replacement value become object columns
+            # with Nones
+            # (note that col.filled(None) cannot be used, as None
+            # there means the default fill value)
+            new_col = col.data.astype(object)
+            new_col[np.ma.getmaskarray(col)] = None
+            col = new_col
+    if col.dtype.kind == 'O':
+        col = __fix_object_column(col, intNullVal, strNullVal)
+    return col
+
+
 def get(query,
         params=None,
         db="wsdb",
@@ -394,8 +458,9 @@ def get(query,
         Flag whether to retrieve the results as a dictionary with column
         names as keys
     strLength : integer
-        The maximum length of the string.
-        Strings will be truncated to this length
+        The minimum length allocated for string columns. If longer
+        strings are encountered, the allocated length is extended
+        to accommodate them
     intNullVal : integer, optional
         All the integer columns with nulls will have null replaced by
         this value
@@ -443,6 +508,7 @@ def get(query,
     >>> a, b = sqlutilpy.get('select ra,dec from rc3 where name=?', "NGC 3166")
     '''
 
+    driver = __check_driver(driver)
     connSupplied = (conn is not None)
     if not connSupplied:
         conn = getConnection(db=db,
@@ -453,10 +519,6 @@ def get(query,
                              port=port,
                              timeout=timeout)
     try:
-        if driver == 'psycopg2':
-            warnings.warn('psycopg2 driver is not supported anymore. '
-                          'We using psycopg instead')
-            driver = 'psycopg'
         if batched:
             # This is related to postgresql server cursors vs
             # application cursors
@@ -540,20 +602,35 @@ def get(query,
 
         elif driver == 'sqlite3':
             tups = cur.fetchall()
-            colNames = [_tmp[0] for _tmp in cur.description]
-            if len(tups) > 0:
-                res = numpy.rec.array(tups)
-                res = __fix_numpy_record_array(res, intNullVal, strNullVal)
+            if cur.description is None:
+                # no result set (e.g. create/insert statement)
+                colNames = []
+                res = []
             else:
-                res = [numpy.array([]) for _ in cur.description]
+                colNames = [_tmp[0] for _tmp in cur.description]
+                if len(tups) > 0:
+                    res = numpy.rec.array(tups)
+                    res = __fix_numpy_record_array(res, intNullVal, strNullVal)
+                else:
+                    res = [numpy.array([]) for _ in cur.description]
         elif driver == 'duckdb':
-            tups = cur.fetchall()
-            colNames = [_tmp[0] for _tmp in cur.description]
-            if len(tups) > 0:
-                res = numpy.rec.array(tups)
-                res = __fix_numpy_record_array(res, intNullVal, strNullVal)
+            if cur.description is None:
+                # no result set
+                colNames = []
+                res = []
             else:
-                res = [numpy.array([]) for _ in cur.description]
+                # fetchnumpy() is much faster than fetchall() followed
+                # by a conversion to numpy.
+                # It returns a dict of numpy (possibly masked) arrays;
+                # in the case of repeated column names the dict keys are
+                # deduplicated, so we take the column names from the
+                # description instead
+                col_dict = cur.fetchnumpy()
+                colNames = [_tmp[0] for _tmp in cur.description]
+                res = [
+                    __process_duckdb_column(_tmp, intNullVal, strNullVal)
+                    for _tmp in col_dict.values()
+                ]
 
         if isinstance(res, numpy.ndarray) and res.dtype.names:
             res = [res[tmp] for tmp in res.dtype.names]
@@ -595,6 +672,7 @@ def execute(query,
             user=None,
             password=None,
             host=None,
+            port=None,
             conn=None,
             preamb=None,
             timeout=None,
@@ -630,6 +708,7 @@ def execute(query,
     >>> sqlutil.execute('create table mytab (a int)', db='mydb')
 
     """
+    driver = __check_driver(driver)
     connSupplied = (conn is not None)
     if not connSupplied:
         conn = getConnection(db=db,
@@ -637,6 +716,7 @@ def execute(query,
                              user=user,
                              password=password,
                              host=host,
+                             port=port,
                              timeout=timeout)
     try:
         cur = getCursor(conn, driver=driver, preamb=preamb, named=False)
@@ -679,6 +759,23 @@ def __create_schema(tableName, arrays, names, temp=False):
     return outp + '(' + ','.join(outp1) + ')'
 
 
+def __escape_strings_for_copy(arr, delimiter):
+    """
+    Escape the characters that are special in the text format of
+    PostgreSQL's COPY FROM (backslash, newline, carriage return, tab
+    and the column delimiter) in a numpy unicode string array
+    """
+    specials = ['\\', '\n', '\r', '\t']
+    # the backslash must be escaped first
+    if delimiter not in specials:
+        specials.append(delimiter)
+    replacements = {'\n': '\\n', '\r': '\\r', '\t': '\\t'}
+    for ch in specials:
+        rep = replacements.get(ch, '\\' + ch)
+        arr = np.char.replace(arr, ch, rep)
+    return arr
+
+
 def __print_arrays(arrays, f, delimiter=' '):
     """
     print the input arrays into the open file separated by a delimiter
@@ -703,6 +800,16 @@ def __print_arrays(arrays, f, delimiter=' '):
                 raise RuntimeError(
                     'Unsupported column type %s. Please report' %
                     (x.dtype.type))
+    # bytes arrays are decoded (as utf-8) into unicode arrays, since
+    # otherwise they would be written as the b'...' python literals
+    arrays = [
+        np.char.decode(x, 'utf-8') if x.dtype.kind == 'S' else x
+        for x in arrays
+    ]
+    arrays = [
+        __escape_strings_for_copy(x, delimiter) if x.dtype.kind == 'U' else x
+        for x in arrays
+    ]
     recarr = np.rec.fromarrays(arrays)
     if not array_mode:
         np.savetxt(f, recarr, fmt=fmts, delimiter=delimiter)
@@ -748,6 +855,7 @@ def upload(tableName,
            user=None,
            password=None,
            host=None,
+           port=None,
            conn=None,
            preamb=None,
            timeout=None,
@@ -775,6 +883,8 @@ def upload(tableName,
     user: string,
     password: string
     host: string
+    port: integer
+         Port of the database
     conn: object
          SQL connection
     preamb: string
@@ -790,7 +900,7 @@ def upload(tableName,
          if True the table will be created before uploading (default)
     delimiter: string
          the string used for delimiting the input data when ingesting into
-         the db (default is |)
+         the db (default is ``|``)
 
     Examples
     --------
@@ -801,6 +911,7 @@ def upload(tableName,
     >>> T = astropy.Table({'x':[1, 2, 3], 'y':['a', 'b', 'c'])
     >>> sqlutilpy.upload('mytable', T)
     """
+    driver = __check_driver(driver)
     connSupplied = (conn is not None)
     if not connSupplied:
         conn = getConnection(db=db,
@@ -808,6 +919,7 @@ def upload(tableName,
                              user=user,
                              password=password,
                              host=host,
+                             port=port,
                              timeout=timeout)
     if names is None:
         for i in range(1):
@@ -872,11 +984,11 @@ table/pandas/dictionary or provide a separate list of arrays and their names')
 
             finally:
                 pass
+        if analyze:
+            cur.execute('analyze %s' % tableName)
     except BaseException:
         failure_cleanup(conn, connSupplied)
         raise
-    if analyze:
-        cur.execute('analyze %s' % tableName)
     cur.close()
     if not noCommit:
         conn.commit()
@@ -926,6 +1038,7 @@ def local_join(query,
     ... 'mytable', (x, y), ('x', 'y'))
     """
 
+    driver = __check_driver(driver)
     connSupplied = (conn is not None)
     if not connSupplied:
         conn = getConnection(db=db,

sqlutilpy-0.29.0/src/sqlutilpy/version.py

@@ -0,0 +1 @@
+__version__ = '0.29.0'

{sqlutilpy-0.27.0 → sqlutilpy-0.29.0/src/sqlutilpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.27.0
+Version: 0.29.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -14,8 +14,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: numpy
 Requires-Dist: psycopg[binary]
-Requires-Dist: numpydoc
-Requires-Dist: numpy
 Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)

{sqlutilpy-0.27.0 → sqlutilpy-0.29.0}/src/sqlutilpy.egg-info/requires.txt

@@ -1,4 +1,2 @@
 numpy
 psycopg[binary]
-numpydoc
-numpy

{sqlutilpy-0.27.0 → sqlutilpy-0.29.0}/tests/test_consistency.py

@@ -8,6 +8,7 @@ PG_DB = os.environ.get('SQLUTIL_TEST_PG_DB')
 PG_HOST = os.environ.get('SQLUTIL_TEST_PG_HOST')
 PG_USER = os.environ.get('SQLUTIL_TEST_PG_USER')
 PG_PASS = os.environ.get('SQLUTIL_TEST_PG_PASS', "")
+PG_PORT = int(os.environ.get('SQLUTIL_TEST_PG_PORT') or 5432)
 
 
 @pytest.fixture(scope="module")
@@ -18,6 +19,7 @@ def pg_conn():
         host=PG_HOST,
         user=PG_USER,
         db=PG_DB,
+        port=PG_PORT,
         password=PG_PASS,
         driver='psycopg')
     yield conn