sqlutilpy 0.24.0__tar.gz → 0.26.0__tar.gz

{sqlutilpy-0.24.0/src/sqlutilpy.egg-info → sqlutilpy-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.24.0
+Version: 0.26.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -16,6 +16,7 @@ Requires-Dist: numpy
 Requires-Dist: psycopg[binary]
 Requires-Dist: numpydoc
 Requires-Dist: numpy
+Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
@@ -25,8 +26,8 @@ Requires-Dist: numpy
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
 tables and run join queries involving local arrays and the tables in the DB.
-This module is optimized to be able to deal efficiently with query results with millions of rows.
-The module only works with PostgreSQL, SQLite and DuckDB databases.
+This module is optimized to deal efficiently with query results with millions of rows.
+The module works with PostgreSQL, SQLite and DuckDB databases.
 
 The full documentation is available [here](http://sqlutilpy.readthedocs.io/en/latest/)
 
@@ -40,13 +41,14 @@ pip install sqlutilpy
 ```
 ## Authentication
 
-Throughout this readme, I will assume that the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
-has been created with your login/password details for Postgresql. If that is not the case, all of the 
-commands given below will also need user='....' and password='...' options
+Throughout this readme, I will assume that if you are using PostgreSQL, then
+the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
+has been created with your login/password details for Postgresql. If that is not the case, many of the 
+commands given below will also need user='....' and password='...' options.
 
 ## Connection information
 
-Most of the sqlutilpy commands require hostname, database name, user etc. 
+Most of the `sqlutilpy` commands require hostname, database name.  
 If you don't want to always type it, you can use standard PostgreSQL environment variables
 like PGPORT, PGDATABASE, PGUSER, PGHOST for the port, database name, user name and hostname
 of the connection. 
@@ -54,7 +56,8 @@ of the connection.
 
 ## Querying the database and retrieving the results
 
-This command will run the query and put the columns into variables ra,dec
+This command will run the query and put the columns into variables ra,dec:
+
 ```python
 import sqlutilpy
 ra,dec = squtilpy.get('select ra,dec from mytable', 
@@ -62,23 +65,28 @@ ra,dec = squtilpy.get('select ra,dec from mytable',
                  db='THE_NAME_OF_MY_DB')
 ```
 
-By default sqlutilpy.get executes the query and returns the tuple with 
-results. You can return the results as dictionary using asDict option.
+By default `sqlutilpy.get` executes the query and returns the tuple with 
+arrays. One array for each column in the query result. 
+You can return the results as dictionary using `asDict` option.
 
 ## Uploading your arrays as column in a table
 
+You can use `sqlutilpy.upload` to upload your arrays as columns in a table.
+
 ```python
 x = np.arange(10)                                                   
 y = x**.5                                                           
 sqlutilpy.upload('mytable',(x,y),('xcol','ycol'))    
 ``` 
-This will create a table called mytable with columns xcol and ycol 
+This will create a table called `mytable` with columns `xcol` and `ycol` 
 
 ## Join query involving your local data and the database table
 
-Imagine you have arrays myid and y and you want to to extract all the 
-information from somebigtable for objects with id=myid. In principle
-you could upload the arrays in the DB and run a query, but local_join function does that for you.
+Sometimes it is beneficial to run a join query involving your local data and the data in the database.
+
+Imagine you have arrays `myid` and `y` and you want to extract all the 
+information from `somebigtable` for objects with `id=myid`. In principle,
+you could upload the arrays in the DB and run a query, but `local_join` function does that for you.
 
 ```python
 myid = np.arange(10)
@@ -89,15 +97,14 @@ R=sqlutilpy.local_join('''select * from mytmptable as m,
            'mytmptable',(myid, y),('myid','ycol'))
 ```
 
-It executes a query as if you arrays were in mytmptable. (behind the scenes
-it uploads the data to the db and runs a query)
+It executes a query as if your arrays were in `mytmptable`. What happens behind the scenes
+is that it uploads the data to the database and runs a query against it.
 
 ## Keeping the connection open. 
 
 Often it is beneficial to preserve an open connection to the database. You can do that if you first 
-obtain the connection using sqlutilpy.getConnection() and then provide it directly
-to sqlutil.get() and friends using conn=conn argument
-```python
+obtain the connection using `sqlutilpy.getConnection()` and then provide it directly
+to `sqlutil.get()` and similar commands using `conn=` keyword:
 conn = sqlutilpy.getConnection(db='mydb', user='meuser', password='something', host='hostname')
 R= sqlutilpy.get('select 1', conn=conn)
 R1= sqlutilpy.get('select 1', conn=conn)
@@ -105,5 +112,5 @@ R1= sqlutilpy.get('select 1', conn=conn)
 
 # How to cite the software
 
-If you use this package, please cite it through zenodo https://doi.org/10.5281/zenodo.6867957
+If you use this package, please cite it through Zenodo https://doi.org/10.5281/zenodo.5160118
 

{sqlutilpy-0.24.0 → sqlutilpy-0.26.0}/README.md

@@ -6,8 +6,8 @@
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
 tables and run join queries involving local arrays and the tables in the DB.
-This module is optimized to be able to deal efficiently with query results with millions of rows.
-The module only works with PostgreSQL, SQLite and DuckDB databases.
+This module is optimized to deal efficiently with query results with millions of rows.
+The module works with PostgreSQL, SQLite and DuckDB databases.
 
 The full documentation is available [here](http://sqlutilpy.readthedocs.io/en/latest/)
 
@@ -21,13 +21,14 @@ pip install sqlutilpy
 ```
 ## Authentication
 
-Throughout this readme, I will assume that the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
-has been created with your login/password details for Postgresql. If that is not the case, all of the 
-commands given below will also need user='....' and password='...' options
+Throughout this readme, I will assume that if you are using PostgreSQL, then
+the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
+has been created with your login/password details for Postgresql. If that is not the case, many of the 
+commands given below will also need user='....' and password='...' options.
 
 ## Connection information
 
-Most of the sqlutilpy commands require hostname, database name, user etc. 
+Most of the `sqlutilpy` commands require hostname, database name.  
 If you don't want to always type it, you can use standard PostgreSQL environment variables
 like PGPORT, PGDATABASE, PGUSER, PGHOST for the port, database name, user name and hostname
 of the connection. 
@@ -35,7 +36,8 @@ of the connection.
 
 ## Querying the database and retrieving the results
 
-This command will run the query and put the columns into variables ra,dec
+This command will run the query and put the columns into variables ra,dec:
+
 ```python
 import sqlutilpy
 ra,dec = squtilpy.get('select ra,dec from mytable', 
@@ -43,23 +45,28 @@ ra,dec = squtilpy.get('select ra,dec from mytable',
                  db='THE_NAME_OF_MY_DB')
 ```
 
-By default sqlutilpy.get executes the query and returns the tuple with 
-results. You can return the results as dictionary using asDict option.
+By default `sqlutilpy.get` executes the query and returns the tuple with 
+arrays. One array for each column in the query result. 
+You can return the results as dictionary using `asDict` option.
 
 ## Uploading your arrays as column in a table
 
+You can use `sqlutilpy.upload` to upload your arrays as columns in a table.
+
 ```python
 x = np.arange(10)                                                   
 y = x**.5                                                           
 sqlutilpy.upload('mytable',(x,y),('xcol','ycol'))    
 ``` 
-This will create a table called mytable with columns xcol and ycol 
+This will create a table called `mytable` with columns `xcol` and `ycol` 
 
 ## Join query involving your local data and the database table
 
-Imagine you have arrays myid and y and you want to to extract all the 
-information from somebigtable for objects with id=myid. In principle
-you could upload the arrays in the DB and run a query, but local_join function does that for you.
+Sometimes it is beneficial to run a join query involving your local data and the data in the database.
+
+Imagine you have arrays `myid` and `y` and you want to extract all the 
+information from `somebigtable` for objects with `id=myid`. In principle,
+you could upload the arrays in the DB and run a query, but `local_join` function does that for you.
 
 ```python
 myid = np.arange(10)
@@ -70,15 +77,14 @@ R=sqlutilpy.local_join('''select * from mytmptable as m,
            'mytmptable',(myid, y),('myid','ycol'))
 ```
 
-It executes a query as if you arrays were in mytmptable. (behind the scenes
-it uploads the data to the db and runs a query)
+It executes a query as if your arrays were in `mytmptable`. What happens behind the scenes
+is that it uploads the data to the database and runs a query against it.
 
 ## Keeping the connection open. 
 
 Often it is beneficial to preserve an open connection to the database. You can do that if you first 
-obtain the connection using sqlutilpy.getConnection() and then provide it directly
-to sqlutil.get() and friends using conn=conn argument
-```python
+obtain the connection using `sqlutilpy.getConnection()` and then provide it directly
+to `sqlutil.get()` and similar commands using `conn=` keyword:
 conn = sqlutilpy.getConnection(db='mydb', user='meuser', password='something', host='hostname')
 R= sqlutilpy.get('select 1', conn=conn)
 R1= sqlutilpy.get('select 1', conn=conn)
@@ -86,5 +92,5 @@ R1= sqlutilpy.get('select 1', conn=conn)
 
 # How to cite the software
 
-If you use this package, please cite it through zenodo https://doi.org/10.5281/zenodo.6867957
+If you use this package, please cite it through Zenodo https://doi.org/10.5281/zenodo.5160118
 

{sqlutilpy-0.24.0 → sqlutilpy-0.26.0}/src/sqlutilpy/sqlutil.py

@@ -7,7 +7,6 @@ import psycopg
 import threading
 import collections
 import warnings
-from numpy.core import numeric as sb
 
 try:
     import astropy.table as atpy
@@ -92,6 +91,9 @@ def getConnection(db=None,
         if timeout is None:
             timeout = 5
         conn = sqlite3.connect(db, timeout=timeout)
+    elif driver == 'duckdb':
+        import duckdb
+        conn = duckdb.connect(db)
     else:
         raise Exception("Unknown driver")
     return conn
@@ -137,12 +139,12 @@ def __fromrecords(recList, dtype=None, intNullVal=None):
     """
 
     shape = None
-    descr = sb.dtype((np.core.records.record, dtype))
+    descr = np.dtype((np.record, dtype))
     try:
-        retval = sb.array(recList, dtype=descr)
+        retval = np.array(recList, dtype=descr)
     except TypeError:  # list of lists instead of list of tuples
         shape = (len(recList), )
-        _array = np.core.records.recarray(shape, descr)
+        _array = np.recarray(shape, descr)
         try:
             for k in range(_array.size):
                 _array[k] = tuple(recList[k])
@@ -170,7 +172,7 @@ def __fromrecords(recList, dtype=None, intNullVal=None):
         if shape is not None and retval.shape != shape:
             retval.shape = shape
 
-    res = retval.view(numpy.core.records.recarray)
+    res = retval.view(numpy.recarray)
 
     return res
 
@@ -210,6 +212,7 @@ def __getDType(row, typeCodes, strLength):
         1022: 'f8',
         1700: 'f8',  # numeric
         1114: '<M8[us]',  # timestamp
+        1184: '<M8[us]',  # timestamp with timezone
         1082: '<M8[us]',  # date
         25: '|U%d',
         1042: '|U%d',  # character()
@@ -407,7 +410,8 @@ def get(query,
                     proc.join(0.2)
                     # in the converter process
                     if proc.is_alive():
-                        proc.terminate()
+                        # could not kill
+                        pass
                 raise
             if proc is not None:
                 proc.join()
@@ -423,14 +427,14 @@ def get(query,
             tups = cur.fetchall()
             colNames = [_tmp[0] for _tmp in cur.description]
             if len(tups) > 0:
-                res = numpy.core.records.array(tups)
+                res = numpy.rec.array(tups)
             else:
                 return [[]] * len(cur.description)
         elif driver == 'duckdb':
             tups = cur.fetchall()
             colNames = [_tmp[0] for _tmp in cur.description]
             if len(tups) > 0:
-                res = numpy.core.records.array(tups)
+                res = numpy.rec.array(tups)
             else:
                 return [[]] * len(cur.description)
 
@@ -735,7 +739,7 @@ table/pandas/dictionary or provide a separate list of arrays and their names')
             cur.execute(query1)
         nsplit = 100000
         N = len(arrays[0])
-        names = ','.join(names)
+        names = ','.join(['"%s"' % _ for _ in names])
         for i in range(0, N, nsplit):
             try:
                 with cur.copy(f'''copy {tableName}({names}) from stdin

sqlutilpy-0.26.0/src/sqlutilpy/version.py

@@ -0,0 +1 @@
+__version__ = '0.26.0'

{sqlutilpy-0.24.0 → sqlutilpy-0.26.0/src/sqlutilpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.24.0
+Version: 0.26.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
 License: BSD
@@ -16,6 +16,7 @@ Requires-Dist: numpy
 Requires-Dist: psycopg[binary]
 Requires-Dist: numpydoc
 Requires-Dist: numpy
+Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
@@ -25,8 +26,8 @@ Requires-Dist: numpy
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
 tables and run join queries involving local arrays and the tables in the DB.
-This module is optimized to be able to deal efficiently with query results with millions of rows.
-The module only works with PostgreSQL, SQLite and DuckDB databases.
+This module is optimized to deal efficiently with query results with millions of rows.
+The module works with PostgreSQL, SQLite and DuckDB databases.
 
 The full documentation is available [here](http://sqlutilpy.readthedocs.io/en/latest/)
 
@@ -40,13 +41,14 @@ pip install sqlutilpy
 ```
 ## Authentication
 
-Throughout this readme, I will assume that the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
-has been created with your login/password details for Postgresql. If that is not the case, all of the 
-commands given below will also need user='....' and password='...' options
+Throughout this readme, I will assume that if you are using PostgreSQL, then
+the .pgpass file ( https://www.postgresql.org/docs/11/libpq-pgpass.html ) 
+has been created with your login/password details for Postgresql. If that is not the case, many of the 
+commands given below will also need user='....' and password='...' options.
 
 ## Connection information
 
-Most of the sqlutilpy commands require hostname, database name, user etc. 
+Most of the `sqlutilpy` commands require hostname, database name.  
 If you don't want to always type it, you can use standard PostgreSQL environment variables
 like PGPORT, PGDATABASE, PGUSER, PGHOST for the port, database name, user name and hostname
 of the connection. 
@@ -54,7 +56,8 @@ of the connection.
 
 ## Querying the database and retrieving the results
 
-This command will run the query and put the columns into variables ra,dec
+This command will run the query and put the columns into variables ra,dec:
+
 ```python
 import sqlutilpy
 ra,dec = squtilpy.get('select ra,dec from mytable', 
@@ -62,23 +65,28 @@ ra,dec = squtilpy.get('select ra,dec from mytable',
                  db='THE_NAME_OF_MY_DB')
 ```
 
-By default sqlutilpy.get executes the query and returns the tuple with 
-results. You can return the results as dictionary using asDict option.
+By default `sqlutilpy.get` executes the query and returns the tuple with 
+arrays. One array for each column in the query result. 
+You can return the results as dictionary using `asDict` option.
 
 ## Uploading your arrays as column in a table
 
+You can use `sqlutilpy.upload` to upload your arrays as columns in a table.
+
 ```python
 x = np.arange(10)                                                   
 y = x**.5                                                           
 sqlutilpy.upload('mytable',(x,y),('xcol','ycol'))    
 ``` 
-This will create a table called mytable with columns xcol and ycol 
+This will create a table called `mytable` with columns `xcol` and `ycol` 
 
 ## Join query involving your local data and the database table
 
-Imagine you have arrays myid and y and you want to to extract all the 
-information from somebigtable for objects with id=myid. In principle
-you could upload the arrays in the DB and run a query, but local_join function does that for you.
+Sometimes it is beneficial to run a join query involving your local data and the data in the database.
+
+Imagine you have arrays `myid` and `y` and you want to extract all the 
+information from `somebigtable` for objects with `id=myid`. In principle,
+you could upload the arrays in the DB and run a query, but `local_join` function does that for you.
 
 ```python
 myid = np.arange(10)
@@ -89,15 +97,14 @@ R=sqlutilpy.local_join('''select * from mytmptable as m,
            'mytmptable',(myid, y),('myid','ycol'))
 ```
 
-It executes a query as if you arrays were in mytmptable. (behind the scenes
-it uploads the data to the db and runs a query)
+It executes a query as if your arrays were in `mytmptable`. What happens behind the scenes
+is that it uploads the data to the database and runs a query against it.
 
 ## Keeping the connection open. 
 
 Often it is beneficial to preserve an open connection to the database. You can do that if you first 
-obtain the connection using sqlutilpy.getConnection() and then provide it directly
-to sqlutil.get() and friends using conn=conn argument
-```python
+obtain the connection using `sqlutilpy.getConnection()` and then provide it directly
+to `sqlutil.get()` and similar commands using `conn=` keyword:
 conn = sqlutilpy.getConnection(db='mydb', user='meuser', password='something', host='hostname')
 R= sqlutilpy.get('select 1', conn=conn)
 R1= sqlutilpy.get('select 1', conn=conn)
@@ -105,5 +112,5 @@ R1= sqlutilpy.get('select 1', conn=conn)
 
 # How to cite the software
 
-If you use this package, please cite it through zenodo https://doi.org/10.5281/zenodo.6867957
+If you use this package, please cite it through Zenodo https://doi.org/10.5281/zenodo.5160118
 

{sqlutilpy-0.24.0 → sqlutilpy-0.26.0}/src/sqlutilpy.egg-info/SOURCES.txt

@@ -11,5 +11,6 @@ src/sqlutilpy.egg-info/dependency_links.txt
 src/sqlutilpy.egg-info/requires.txt
 src/sqlutilpy.egg-info/top_level.txt
 tests/killer.py
+tests/test_duckdb.py
 tests/test_pg.py
 tests/test_sqlite.py

sqlutilpy-0.26.0/tests/test_duckdb.py

@@ -0,0 +1,32 @@
+import pytest
+import os
+import sqlutilpy as sqlutil
+import duckdb
+
+
+@pytest.fixture()
+def setup():
+    fname = 'duck.db'
+    kw = dict(db=fname, driver='duckdb')
+    conn = duckdb.connect(fname)
+    cur = conn.cursor()
+    cur.execute('create table tmp (a int, b double precision)')
+    cur.execute('insert into tmp values(1,2), (2,3), (3,4), (4,5);')
+    conn.commit()
+    yield kw
+    os.unlink(fname)
+
+
+def test_simple(setup):
+    a, b = sqlutil.get('select a,b from tmp', **setup)
+    assert (len(a) == 4)
+
+
+def test_preamb(setup):
+    a, b = sqlutil.get('select a,b from tmp', preamb='select 1', **setup)
+    assert (len(a) == 4)
+
+
+def test_empty(setup):
+    a, b = sqlutil.get('select a,b from tmp where a<0', **setup)
+    assert (len(a) == 0)

{sqlutilpy-0.24.0 → sqlutilpy-0.26.0}/tests/test_pg.py

@@ -340,12 +340,12 @@ def test_upload(setup):
         'xi32': xi32,
         'xi64': xi64,
         'xf32': xf32,
-        'xf64': xf64,
+        'XF64': xf64,  # one uppercase
         'xbool': xbool
     })
     sqlutil.upload(mytab, astroTab, **kw)
     yi16, yi32, yi64, yf32, yf64, ybool = sqlutil.get(
-        '''select xi16,xi32,xi64,xf32,xf64,xbool from %s''' % (mytab), **kw)
+        '''select xi16,xi32,xi64,xf32,"XF64",xbool from %s''' % (mytab), **kw)
     try:
         assert ((xi16 == yi16).all())
         assert ((xi32 == yi32).all())
@@ -357,7 +357,7 @@ def test_upload(setup):
         sqlutil.execute('drop table %s' % mytab, **kw)
     sqlutil.upload(mytab, astroTab.to_pandas(), **kw)
     yi16, yi32, yi64, yf32, yf64, ybool = sqlutil.get(
-        '''select xi16,xi32,xi64,xf32,xf64,xbool from %s''' % (mytab), **kw)
+        '''select xi16,xi32,xi64,xf32,"XF64",xbool from %s''' % (mytab), **kw)
     try:
         assert ((xi16 == yi16).all())
         assert ((xi32 == yi32).all())

sqlutilpy-0.24.0/src/sqlutilpy/version.py

@@ -1 +0,0 @@
-__version__ = '0.24.0'