sqlutilpy 0.28.0__tar.gz → 0.30.0__tar.gz

sqlutilpy-0.30.0/CHANGELOG.md

@@ -0,0 +1,55 @@
+Changelog
+
+# 0.30.0
+* All the options of get(), execute(), upload(), local_join(), getConnection() and getCursor() are now keyword-only and listed in the same order in every function (db, driver, user, password, host, port, conn, preamb, timeout, then the function specific options). Only the query/parameters and the table name/arrays/column names can be given positionally. This stops the argument order changes between releases (e.g. the port argument added in 0.28.0 before conn in execute() and upload()) from silently shifting positional arguments
+* get() and getCursor() accept again the notNamed option removed in 0.27.0, as a deprecated alias of batched=not notNamed / named=not notNamed. It emits a DeprecationWarning and will be removed in a future release
+
+# 0.29.0
+* get() with duckdb: fix the null replacement in narrow and unsigned integer columns (tinyint, utinyint, uinteger, ubigint) with numpy<2, where intNullVal was silently wrapped (e.g. -9999 became -15 in a tinyint column) instead of the column being widened to int64
+* This is the last release supporting numpy<2
+* An unknown driver name now raises SqlUtilException from get(), execute(), upload(), local_join(), getConnection() and getCursor() before any connection is attempted (it was added in 0.28.0)
+
+# 0.28.0
+* upload(): strings containing the delimiter, backslashes, newlines, tabs or carriage returns are now escaped and survive the upload unchanged
+* upload(): numpy bytes arrays (dtype S) are now decoded as utf-8 and uploaded as strings, rather than being stored as the b'...' python literals
+* get(): intNullVal=None keeps the nulls in integer columns as None (the column is returned as an object array) instead of failing
+* get(): statements that do not return a result set no longer fail with sqlite3/duckdb
+* get(): use a unique server-side cursor name so that several batched queries can be run over the same connection
+* get() with duckdb now uses the much faster fetchnumpy() interface. As a result the integer/float columns keep the width of the database type (e.g. int -> int32, float -> float32, rather than always int64/float64), dates, timestamps with time zone and intervals are returned as datetime64/timedelta64, decimals and hugeints as float64, blobs as object arrays of bytearrays, and list columns are now supported
+* add the port argument to execute() and upload()
+* upload(): analyze is now run inside the try block, so a failure there rolls back the transaction
+* fix the strLength docstring (strings are not truncated, the allocated length is extended as needed)
+* packaging: remove numpydoc and a duplicate numpy from the dependencies; the readthedocs build uses docs/requirements.txt and python 3.12
+
+# 0.27.0
+* Introduce the non-batched mode with batched=False that allows (at the cost of more memory use) using the parallel PostgreSQL plans, which can speed up queries by a factor of few
+* remove the notNamed option of sqlutil.get
+* refactor the threading for converting data, to use the threadpool
+* I added more tests when strings are read, so hopefully the truncation into 20 characters should be more rare now.
+
+# 0.26.0
+* Preserve uppercase columns when ingestng data
+* Remove obsolet numpy functionality
+
+# 0.25.0
+* Allow querying timestamp with timezone types
+* Allow sqlutilpy.connect() to the duckdb db
+
+# 0.24.0
+* In the case sqlutilpy.get() returns no rows and asDict=True, the column names will still be there
+
+# 0.23.0
+* Switch to psycopg (i.e. third version of psycopg)
+* Make the code compatible with numpy 2.0
+
+# 0.22.0
+* This version migrated to pyproject.toml
+
+# 0.21.0
+* Allow uploading int8 columns
+* Allow uploading data where some columns have arrays
+
+# 0.20.0
+
+* Increase default string length in local_join
+* add intNullVal option to local_join

sqlutilpy-0.30.0/CITATION.cff

@@ -0,0 +1,26 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it using the metadata from this file."
+type: software
+title: "sqlutilpy"
+abstract: "Database query code returning numpy arrays. A thin layer over psycopg, sqlite3 and duckdb that runs SQL queries and returns the results as numpy arrays, and uploads numpy arrays into PostgreSQL tables."
+authors:
+  - family-names: "Koposov"
+    given-names: "Sergey E."
+    email: "skoposov@ed.ac.uk"
+    orcid: "https://orcid.org/0000-0003-2644-135X"
+    affiliation: "Institute for Astronomy, University of Edinburgh"
+license: "BSD-3-Clause"
+repository-code: "https://github.com/segasai/sqlutilpy"
+url: "https://sqlutilpy.readthedocs.io"
+doi: "10.5281/zenodo.5160118"
+identifiers:
+  - type: doi
+    value: "10.5281/zenodo.5160118"
+    description: "Zenodo concept DOI resolving to the latest release"
+keywords:
+  - numpy
+  - postgresql
+  - sqlite
+  - duckdb
+  - sql
+  - astronomy

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2018, Sergey Koposov
+Copyright (c) 2026, Sergey Koposov
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -8,17 +8,17 @@ modification, are permitted provided that the following conditions are met:
     * Redistributions in binary form must reproduce the above copyright
       notice, this list of conditions and the following disclaimer in the
       documentation and/or other materials provided with the distribution.
-    * Neither the name of the <organization> nor the
+    * Neither the name of Sergey Koposov nor the
       names of its contributors may be used to endorse or promote products
       derived from this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DISCLAIMED. IN NO EVENT SHALL the copyright holders BE LIABLE FOR ANY
 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

sqlutilpy-0.30.0/MANIFEST.in

@@ -0,0 +1,6 @@
+include README.md
+include LICENSE.txt
+include CHANGELOG.md
+include CITATION.cff
+include MANIFEST.in
+recursive-include tests *.py

{sqlutilpy-0.28.0/src/sqlutilpy.egg-info → sqlutilpy-0.30.0}/PKG-INFO

@@ -1,25 +1,29 @@
 Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.28.0
+Version: 0.30.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
-License: BSD
+License-Expression: BSD-3-Clause
 Project-URL: Homepage, https://github.com/segasai/sqlutilpy
-Keywords: numpy postgresql query sql sqlite array
-Classifier: Development Status :: 3 - Alpha
-Classifier: License :: OSI Approved :: BSD License
+Project-URL: Documentation, https://sqlutilpy.readthedocs.io
+Project-URL: Changelog, https://github.com/segasai/sqlutilpy/blob/master/CHANGELOG.md
+Keywords: numpy,postgresql,sqlite,duckdb,sql,query,array
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
 Classifier: Topic :: Utilities
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: numpy
+Requires-Dist: numpy>=2
 Requires-Dist: psycopg[binary]
 Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/segasai/sqlutilpy/badge.svg?branch=master)](https://coveralls.io/github/segasai/sqlutilpy?branch=master)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6867957.svg)](https://doi.org/10.5281/zenodo.6867957)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5160118.svg)](https://doi.org/10.5281/zenodo.5160118)
 
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
@@ -58,7 +62,7 @@ This command will run the query and put the columns into variables ra,dec:
 
 ```python
 import sqlutilpy
-ra,dec = squtilpy.get('select ra,dec from mytable', 
+ra,dec = sqlutilpy.get('select ra,dec from mytable', 
                  host='HOST_NAME_OF_MY_PG_SERVER', 
                  db='THE_NAME_OF_MY_DB')
 ```
@@ -67,7 +71,19 @@ By default `sqlutilpy.get` executes the query and returns the tuple with
 arrays. One array for each column in the query result. 
 You can return the results as dictionary using `asDict` option.
 
-## Uploading your arrays as column in a table
+## SQLite and DuckDB
+
+The same `get()` and `execute()` functions work with SQLite and DuckDB
+files, by selecting the driver and giving the file name as the database:
+
+```python
+a, b = sqlutilpy.get('select a, b from mytable', db='mydata.db', driver='sqlite3')
+a, b = sqlutilpy.get('select a, b from mytable', db='mydata.duckdb', driver='duckdb')
+```
+
+Uploading tables and `local_join` are only available with PostgreSQL.
+
+## Uploading your arrays as columns in a table (PostgreSQL)
 
 You can use `sqlutilpy.upload` to upload your arrays as columns in a table.
 
@@ -78,7 +94,7 @@ sqlutilpy.upload('mytable',(x,y),('xcol','ycol'))
 ``` 
 This will create a table called `mytable` with columns `xcol` and `ycol` 
 
-## Join query involving your local data and the database table
+## Join query involving your local data and the database table (PostgreSQL)
 
 Sometimes it is beneficial to run a join query involving your local data and the data in the database.
 
@@ -102,7 +118,8 @@ is that it uploads the data to the database and runs a query against it.
 
 Often it is beneficial to preserve an open connection to the database. You can do that if you first 
 obtain the connection using `sqlutilpy.getConnection()` and then provide it directly
-to `sqlutil.get()` and similar commands using `conn=` keyword:
+to `sqlutilpy.get()` and similar commands using `conn=` keyword:
+```python
 conn = sqlutilpy.getConnection(db='mydb', user='meuser', password='something', host='hostname')
 R= sqlutilpy.get('select 1', conn=conn)
 R1= sqlutilpy.get('select 1', conn=conn)

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/README.md

@@ -1,7 +1,7 @@
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/segasai/sqlutilpy/badge.svg?branch=master)](https://coveralls.io/github/segasai/sqlutilpy?branch=master)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6867957.svg)](https://doi.org/10.5281/zenodo.6867957)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5160118.svg)](https://doi.org/10.5281/zenodo.5160118)
 
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
@@ -40,7 +40,7 @@ This command will run the query and put the columns into variables ra,dec:
 
 ```python
 import sqlutilpy
-ra,dec = squtilpy.get('select ra,dec from mytable', 
+ra,dec = sqlutilpy.get('select ra,dec from mytable', 
                  host='HOST_NAME_OF_MY_PG_SERVER', 
                  db='THE_NAME_OF_MY_DB')
 ```
@@ -49,7 +49,19 @@ By default `sqlutilpy.get` executes the query and returns the tuple with
 arrays. One array for each column in the query result. 
 You can return the results as dictionary using `asDict` option.
 
-## Uploading your arrays as column in a table
+## SQLite and DuckDB
+
+The same `get()` and `execute()` functions work with SQLite and DuckDB
+files, by selecting the driver and giving the file name as the database:
+
+```python
+a, b = sqlutilpy.get('select a, b from mytable', db='mydata.db', driver='sqlite3')
+a, b = sqlutilpy.get('select a, b from mytable', db='mydata.duckdb', driver='duckdb')
+```
+
+Uploading tables and `local_join` are only available with PostgreSQL.
+
+## Uploading your arrays as columns in a table (PostgreSQL)
 
 You can use `sqlutilpy.upload` to upload your arrays as columns in a table.
 
@@ -60,7 +72,7 @@ sqlutilpy.upload('mytable',(x,y),('xcol','ycol'))
 ``` 
 This will create a table called `mytable` with columns `xcol` and `ycol` 
 
-## Join query involving your local data and the database table
+## Join query involving your local data and the database table (PostgreSQL)
 
 Sometimes it is beneficial to run a join query involving your local data and the data in the database.
 
@@ -84,7 +96,8 @@ is that it uploads the data to the database and runs a query against it.
 
 Often it is beneficial to preserve an open connection to the database. You can do that if you first 
 obtain the connection using `sqlutilpy.getConnection()` and then provide it directly
-to `sqlutil.get()` and similar commands using `conn=` keyword:
+to `sqlutilpy.get()` and similar commands using `conn=` keyword:
+```python
 conn = sqlutilpy.getConnection(db='mydb', user='meuser', password='something', host='hostname')
 R= sqlutilpy.get('select 1', conn=conn)
 R1= sqlutilpy.get('select 1', conn=conn)

sqlutilpy-0.30.0/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "sqlutilpy"
+dynamic = ["version"]
+description = "Database query code returning numpy arrays"
+authors = [
+    {name = "Sergey E. Koposov", email = "skoposov@ed.ac.uk"},
+]
+dependencies = [
+    "numpy>=2",
+    "psycopg[binary]"
+]
+requires-python = ">=3.10"
+readme = "README.md"
+license = "BSD-3-Clause"
+license-files = ["LICENSE.txt"]
+keywords = ["numpy", "postgresql", "sqlite", "duckdb", "sql", "query", "array"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Topic :: Database",
+    "Topic :: Utilities",
+]
+
+[project.urls]
+Homepage = "https://github.com/segasai/sqlutilpy"
+Documentation = "https://sqlutilpy.readthedocs.io"
+Changelog = "https://github.com/segasai/sqlutilpy/blob/master/CHANGELOG.md"
+
+[build-system]
+requires = ["setuptools>=77", "setuptools-git-versioning"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools-git-versioning]
+enabled = true

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/src/sqlutilpy/sqlutil.py

@@ -1,4 +1,6 @@
-"""Sqlutilpy module to access SQL databases
+"""Query SQL databases (PostgreSQL, SQLite, DuckDB) and get numpy
+arrays back, upload numpy arrays as tables and join local arrays with
+database tables
 """
 import numpy
 import numpy as np
@@ -45,13 +47,13 @@ def __check_driver(driver):
             'psycopg2 driver is not supported anymore using psycopg instead')
         driver = 'psycopg'
     if driver not in SUPPORTED_DRIVERS:
-        raise SqlUtilException(
-            'Unknown driver %r; supported drivers are %s' %
-            (driver, ', '.join(SUPPORTED_DRIVERS)))
+        raise SqlUtilException('Unknown driver %r; supported drivers are %s' %
+                               (driver, ', '.join(SUPPORTED_DRIVERS)))
     return driver
 
 
-def getConnection(db=None,
+def getConnection(*,
+                  db=None,
                   driver=None,
                   user=None,
                   password=None,
@@ -64,28 +66,37 @@ def getConnection(db=None,
 
     Parameters
     ----------
-
     db : string
-        The name of the database (in case of PostgreSQL) or filename in
-        case of sqlite db
-    driver :  string
-        The db driver (either 'psycopg' or 'sqlite3')
+        The name of the database (PostgreSQL) or the filename of the
+        database (SQLite, DuckDB). For DuckDB ':memory:' gives an
+        in-memory database. For PostgreSQL, if this and the other
+        connection details are not given, the libpq defaults and
+        environment variables (PGDATABASE, PGHOST, PGPORT, PGUSER) are used
+    driver : string
+        The database driver: 'psycopg' (PostgreSQL), 'sqlite3' or 'duckdb'
     user : string, optional
-        Username
-    password: string, optional
-        Password
+        Username (PostgreSQL only)
+    password : string, optional
+        Password (PostgreSQL only)
     host : string, optional
-        Host-name
-    port : integer
-        Connection port (by default 5432 for PostgreSQL)
-    timeout : integer
-        Connection timeout for sqlite
+        Hostname (PostgreSQL only)
+    port : integer, optional
+        Connection port (PostgreSQL only, by default 5432)
+    timeout : integer, optional
+        How long to wait for a locked database, in seconds
+        (SQLite only, by default 5)
 
     Returns
     -------
     conn : object
-         Database Connection
+        Database connection
 
+    Examples
+    --------
+    >>> conn = sqlutilpy.getConnection(db='mydb', host='myhost',
+    ...                                driver='psycopg')
+    >>> conn = sqlutilpy.getConnection(db='mydata.db', driver='sqlite3')
+    >>> conn = sqlutilpy.getConnection(db=':memory:', driver='duckdb')
     """
     driver = __check_driver(driver)
     if driver == 'psycopg':
@@ -112,10 +123,39 @@ def getConnection(db=None,
     return conn
 
 
-def getCursor(conn, driver=None, preamb=None, named=True):
+def getCursor(conn, *, driver=None, preamb=None, named=True, notNamed=None):
     """
     Retrieve the database cursor
+
+    Parameters
+    ----------
+    conn : object
+        Database connection
+    driver : string
+        The database driver: 'psycopg', 'sqlite3' or 'duckdb'
+    preamb : string, optional
+        SQL to be executed on the cursor before anything else
+    named : bool
+        If True (the default) and the driver is 'psycopg', a server-side
+        (named) cursor is returned, so that the results can be fetched in
+        batches; this is what get(batched=True) uses. If False a
+        client-side cursor is returned, as for get(batched=False).
+        Ignored for the other drivers
+    notNamed : bool, optional
+        Deprecated spelling from before 0.27: notNamed=True is the same as
+        named=False. It will be removed in a future release
+
+    Returns
+    -------
+    cur : object
+        Database cursor
     """
+    if notNamed is not None:
+        # the pre-0.27 spelling of named=not notNamed
+        warnings.warn('notNamed is deprecated; use named=not notNamed instead',
+                      DeprecationWarning,
+                      stacklevel=2)
+        named = not notNamed
     driver = __check_driver(driver)
     if driver == 'psycopg':
         cur = conn.cursor(scrollable=False)
@@ -397,13 +437,15 @@ def __process_duckdb_column(col, intNullVal, strNullVal):
     if isinstance(col, np.ma.MaskedArray):
         kind = col.dtype.kind
         if kind in ('i', 'u') and intNullVal is not None:
-            try:
-                col = col.filled(intNullVal)
-            except (TypeError, OverflowError):
+            info = np.iinfo(col.dtype)
+            if not (info.min <= intNullVal <= info.max):
                 # intNullVal does not fit into the column type (e.g. a
                 # negative value for an unsigned or a tinyint column),
-                # so the column is widened to int64 first
-                col = col.astype(np.int64).filled(intNullVal)
+                # so the column is widened to int64 first.
+                # (we check explicitly rather than catch an exception,
+                # because numpy<2 silently wraps the value instead)
+                col = col.astype(np.int64)
+            col = col.filled(intNullVal)
         elif kind == 'f':
             col = col.filled(np.nan)
         elif kind == 'M':
@@ -424,88 +466,123 @@ def __process_duckdb_column(col, intNullVal, strNullVal):
 
 def get(query,
         params=None,
+        *,
         db="wsdb",
         driver="psycopg",
         user=None,
         password=None,
         host=None,
-        preamb=None,
-        conn=None,
         port=None,
-        strLength=STRLEN_DEFAULT,
+        conn=None,
+        preamb=None,
         timeout=None,
+        strLength=STRLEN_DEFAULT,
         batched=True,
         asDict=False,
         intNullVal=-9999,
         strNullVal='None',
-        nthreads=1):
-    '''
-    Executes the sql query and returns the tuple or dictionary
-    with the numpy arrays.
+        nthreads=1,
+        notNamed=None):
+    """
+    Execute the SQL query and return the result as a tuple or dictionary
+    of numpy arrays, one array per column.
 
     Parameters
     ----------
     query : string
-        Query you want to execute, can include question
-        marks to refer to query parameters
-    params : tuple
+        The query to execute. It can contain placeholders for the query
+        parameters: %s for PostgreSQL, ? for SQLite and DuckDB
+    params : tuple, optional
         Query parameters
-    conn : object
-        The connection object to the DB (optional) to avoid reconnecting
-    asDict : boolean
-        Flag whether to retrieve the results as a dictionary with column
-        names as keys
-    strLength : integer
-        The minimum length allocated for string columns. If longer
-        strings are encountered, the allocated length is extended
-        to accommodate them
-    intNullVal : integer, optional
-        All the integer columns with nulls will have null replaced by
-        this value
-    strNullVal : string, optional
-        All the string columns with nulls will have null replaced by
-        this value
     db : string
-        The name of the database
+        The name of the database (PostgreSQL) or the filename of the
+        database (SQLite, DuckDB)
     driver : string, optional
-        The sql driver to be used (psycopg or sqlite3)
+        The database driver: 'psycopg' (PostgreSQL, the default),
+        'sqlite3' or 'duckdb'
     user : string, optional
-        User name for the DB connection
+        User name for the DB connection (PostgreSQL only)
     password : string, optional
-        DB connection password
+        DB connection password (PostgreSQL only)
     host : string, optional
-        Hostname of the database
+        Hostname of the database (PostgreSQL only)
     port : integer, optional
-        Port of the database
-    preamb : string
-        SQL code to be executed before the query
+        Port of the database (PostgreSQL only)
+    conn : object, optional
+        An existing connection to the DB (see getConnection()), to avoid
+        reconnecting. The connection is left open afterwards. The driver
+        argument must still match the connection
+    preamb : string, optional
+        SQL to be executed before the query
+    timeout : integer, optional
+        How long to wait for a locked database, in seconds (SQLite only)
+    strLength : integer, optional
+        The minimum width allocated for string columns (PostgreSQL only).
+        If longer strings are encountered the width is extended to
+        accommodate them. With SQLite and DuckDB the width is always
+        inferred from the data
     batched : bool, optional
-        Whether to use named or not-named cursor with PostgreSQL
-        The default setting of True leads to retrieval of results in batches
-        and on the fly conversion of
-        retrieved results into numpy, thus it will not use more memory than
-        needed to store the results of the query. The
-        The batched=False will use at least twice the amount of memory
-        needed to store the results, but with the benefit of faster query
-        execution, because these queries use PostgreSQL parallelism and thus
-        can be ~ a factor of few faster
+        PostgreSQL only. With the default of True the results are
+        retrieved through a server-side (named) cursor in batches of
+        config.arraysize rows and converted to numpy on the fly, so the
+        memory use is about the size of the final arrays. With False a
+        client-side cursor is used and the whole result is fetched at
+        once, which needs about twice the memory, but lets PostgreSQL use
+        parallel query plans, which can be several times faster for heavy
+        queries. The server-side cursor only accepts SELECT, VALUES,
+        TABLE and WITH statements, so statements such EXPLAIN, SHOW
+        must be run with batched=False
+    asDict : bool, optional
+        If True, return an ordered dictionary keyed by column name instead
+        of a tuple of arrays. Repeated column names get a numeric suffix
+    intNullVal : integer or None, optional
+        The value that replaces nulls in integer columns (-9999 by
+        default). If None, integer columns with nulls are returned as
+        object arrays with None in place of the nulls
+    strNullVal : string, optional
+        The value that replaces nulls in string columns ('None' by
+        default)
+    nthreads : integer, optional
+        The number of threads used to convert the batches to numpy
+        in the batched PostgreSQL mode
+    notNamed : bool, optional
+        Deprecated spelling from before 0.27: notNamed=True is the same as
+        batched=False (a client-side cursor). It will be removed in a
+        future release
+
     Returns
     -------
-    ret : Tuple or dictionary
-        By default you get a tuple with numpy arrays for each column
-        in your query.
-        If you specified asDict keyword then you get an ordered dictionary with
-        your columns.
+    ret : tuple or dictionary
+        A tuple with one numpy array per column of the query result, or,
+        with asDict=True, an ordered dictionary mapping the column names
+        to the arrays. Nulls in floating point columns become NaN, in
+        timestamp columns NaT; see intNullVal and strNullVal for the
+        integer and string columns
 
     Examples
     --------
-    >>> a, b, c = sqlutilpy.get('select ra,dec,d25 from rc3')
+    >>> ra, dec, d25 = sqlutilpy.get('select ra, dec, d25 from rc3',
+    ...                              host='myhost', db='mydb')
+
+    With query parameters:
 
-    You can also use the parameters in your query:
+    >>> ra, dec = sqlutilpy.get('select ra, dec from rc3 where name = %s',
+    ...                         ('NGC 3166', ), host='myhost', db='mydb')
 
-    >>> a, b = sqlutilpy.get('select ra,dec from rc3 where name=?', "NGC 3166")
-    '''
+    From a SQLite file:
+
+    >>> a, b = sqlutilpy.get('select a, b from tmp where a > ?', (3, ),
+    ...                      db='mydata.db', driver='sqlite3')
+    """
 
+    if notNamed is not None:
+        # the pre-0.27 spelling: notNamed=True asked for a client-side
+        # cursor, which is what batched=False does now
+        warnings.warn(
+            'notNamed is deprecated; use batched=not notNamed instead',
+            DeprecationWarning,
+            stacklevel=2)
+        batched = not notNamed
     driver = __check_driver(driver)
     connSupplied = (conn is not None)
     if not connSupplied:
@@ -517,13 +594,11 @@ def get(query,
                              port=port,
                              timeout=timeout)
     try:
-        if batched:
-            # This is related to postgresql server cursors vs
-            # application cursors
-            namedCursor = True
-        else:
-            namedCursor = False
-        cur = getCursor(conn, driver=driver, preamb=preamb, named=namedCursor)
+        # batched=True means a server-side (named) PostgreSQL cursor,
+        # read with fetchmany() in chunks of config.arraysize rows;
+        # batched=False means a client-side cursor read with one
+        # fetchall(). The cursor type is irrelevant for the other drivers
+        cur = getCursor(conn, driver=driver, preamb=preamb, named=batched)
 
         if params is None:
             res = cur.execute(query)
@@ -665,6 +740,7 @@ def get(query,
 
 def execute(query,
             params=None,
+            *,
             db='wsdb',
             driver="psycopg",
             user=None,
@@ -676,35 +752,45 @@ def execute(query,
             timeout=None,
             noCommit=False):
     """
-    Execute a given SQL command without returning the results
+    Execute an SQL statement without returning any results
 
     Parameters
     ----------
-    query: string
-        The query or command you are executing
-    params: tuple, optional
-        Optional parameters of your query
+    query : string
+        The statement to execute. It can contain placeholders for the
+        parameters: %s for PostgreSQL, ? for SQLite and DuckDB
+    params : tuple, optional
+        Parameters of the statement
     db : string
-        Database name
-    driver : string
-        Driver for the DB connection ('psycopg' or 'sqlite3')
+        The name of the database (PostgreSQL) or the filename of the
+        database (SQLite, DuckDB)
+    driver : string, optional
+        The database driver: 'psycopg' (PostgreSQL, the default),
+        'sqlite3' or 'duckdb'
     user : string, optional
-        user name for the DB connection
+        User name for the DB connection (PostgreSQL only)
     password : string, optional
-        DB connection password
+        DB connection password (PostgreSQL only)
     host : string, optional
-        Hostname of the database
+        Hostname of the database (PostgreSQL only)
     port : integer, optional
-        Port of the database
-    noCommit: bool
-        By default execute() will commit your command.
-        If you say noCommit, the commit won't be issued.
+        Port of the database (PostgreSQL only)
+    conn : object, optional
+        An existing connection to the DB (see getConnection()), to avoid
+        reconnecting. The connection is left open afterwards
+    preamb : string, optional
+        SQL to be executed before the statement
+    timeout : integer, optional
+        How long to wait for a locked database, in seconds (SQLite only)
+    noCommit : bool, optional
+        By default the statement is committed. With noCommit=True the
+        commit is not issued
 
     Examples
     --------
-    >>> sqlutil.execute('drop table mytab', conn=conn)
-    >>> sqlutil.execute('create table mytab (a int)', db='mydb')
-
+    >>> sqlutilpy.execute('drop table mytab', conn=conn)
+    >>> sqlutilpy.execute('create table mytab (a int)', db='mydb')
+    >>> sqlutilpy.execute('insert into mytab values (%s)', (1, ), db='mydb')
     """
     driver = __check_driver(driver)
     connSupplied = (conn is not None)
@@ -848,6 +934,7 @@ def failure_cleanup(conn, connSupplied):
 def upload(tableName,
            arrays,
            names=None,
+           *,
            db="wsdb",
            driver="psycopg",
            user=None,
@@ -863,42 +950,59 @@ def upload(tableName,
            createTable=True,
            delimiter='|'):
     """
-    Upload the data stored in the tuple of arrays in the DB
+    Upload the data stored in a tuple of arrays (or a table) into a new
+    table in the DB. PostgreSQL only.
 
     Parameters
     ----------
     tableName : string
         The name of the table where the data will be uploaded
-    arrays_or_table : tuple
-        Tuple of arrays that will be columns of the new table
-        If names are not specified, I this parameter can be pandas or
-        astropy table
-    names : tuple
-    db: string
-         Databas name
-    driver: string
-         Python database driver "psycopg",
-    user: string,
-    password: string
-    host: string
-    port: integer
-         Port of the database
-    conn: object
-         SQL connection
-    preamb: string
-         The string/query to be executed before your command
-    noCommit: bool
-         If true, the commit is not executed and the table will go away
-         after the disconnect
-    temp: bool
-         If true a temporary table will be created
-    analyze: bool
-         if True, the table will be analyzed after the upload
-    createTable: bool
-         if True the table will be created before uploading (default)
-    delimiter: string
-         the string used for delimiting the input data when ingesting into
-         the db (default is ``|``)
+    arrays : tuple, astropy.table.Table, pandas.DataFrame or dict
+        A tuple of arrays that will become the columns of the table, or
+        an astropy Table, pandas DataFrame or dictionary of arrays, in
+        which case the column names are taken from it and the names
+        argument must be omitted.
+        Integer, floating point, boolean, string (unicode or bytes;
+        bytes are decoded as utf-8), and datetime64 arrays are supported,
+        as well as object arrays holding numpy arrays, which become
+        array columns
+    names : tuple, optional
+        The column names (one per array). Characters that are not allowed
+        in identifiers (spaces, brackets, dashes) are replaced by
+        underscores, with a warning
+    db : string
+        Database name
+    driver : string
+        The database driver; only 'psycopg' is supported
+    user : string, optional
+        User name for the DB connection
+    password : string, optional
+        DB connection password
+    host : string, optional
+        Hostname of the database
+    port : integer, optional
+        Port of the database
+    conn : object, optional
+        An existing connection to the DB (see getConnection()), to avoid
+        reconnecting. The connection is left open afterwards
+    preamb : string, optional
+        SQL to be executed before the upload
+    timeout : integer, optional
+        Unused for PostgreSQL
+    noCommit : bool, optional
+        If True, the commit is not executed and the table goes away
+        when the connection is closed
+    temp : bool, optional
+        If True, a temporary table is created
+    analyze : bool, optional
+        If True, the table is analyzed after the upload
+    createTable : bool, optional
+        If True (the default), the table is created before uploading.
+        With False the data is appended to an existing table
+    delimiter : string, optional
+        The column delimiter used when sending the data to the DB
+        (the default is ``|``). Strings containing the delimiter are
+        escaped, so it can be any character
 
     Examples
     --------
@@ -906,7 +1010,7 @@ def upload(tableName,
     >>> y = x**.5
     >>> sqlutilpy.upload('mytable', (x, y), ('xcol', 'ycol'))
 
-    >>> T = astropy.Table({'x':[1, 2, 3], 'y':['a', 'b', 'c'])
+    >>> T = astropy.table.Table({'x': [1, 2, 3], 'y': ['a', 'b', 'c']})
     >>> sqlutilpy.upload('mytable', T)
     """
     driver = __check_driver(driver)
@@ -998,6 +1102,7 @@ def local_join(query,
                tableName,
                arrays,
                names,
+               *,
                db=None,
                driver="psycopg",
                user=None,
@@ -1008,31 +1113,71 @@ def local_join(query,
                preamb=None,
                timeout=None,
                strLength=STRLEN_DEFAULT,
+               asDict=False,
                intNullVal=-9999,
-               strNullVal='None',
-               asDict=False):
+               strNullVal='None'):
     """
-    Join your local data in python with the data in the database
-    This command first uploads the data in the DB creating a temporary table
-    and then runs a user specified query that can your local data.
+    Join your local data in python with the data in the database.
+    PostgreSQL only.
+
+    This uploads the arrays into a temporary table and then runs the
+    query, which can refer to that table. The temporary table exists
+    only for the duration of the call.
 
     Parameters
     ----------
-    query : String with the query to be executed
-    tableName : The name of the temporary table that is going to be created
-    arrays : The tuple with list of arrays with the data to be loaded in the DB
-    names : The tuple with the column names for the user table
+    query : string
+        The query to be executed; it can refer to the table tableName
+    tableName : string
+        The name of the temporary table holding your data
+    arrays : tuple
+        The tuple of arrays with the data to be loaded in the DB
+    names : tuple
+        The column names of the temporary table (one per array)
+    db : string
+        Database name
+    driver : string
+        The database driver; only 'psycopg' is supported
+    user : string, optional
+        User name for the DB connection
+    password : string, optional
+        DB connection password
+    host : string, optional
+        Hostname of the database
+    port : integer, optional
+        Port of the database
+    conn : object, optional
+        An existing connection to the DB (see getConnection()), to avoid
+        reconnecting. The connection is left open afterwards
+    preamb : string, optional
+        SQL to be executed before the query
+    timeout : integer, optional
+        Unused for PostgreSQL
+    strLength : integer, optional
+        The minimum width allocated for string columns (see get())
+    asDict : bool, optional
+        If True, return an ordered dictionary keyed by column name instead
+        of a tuple of arrays
+
+    intNullVal : integer or None, optional
+        The value that replaces nulls in integer columns (see get())
+    strNullVal : string, optional
+        The value that replaces nulls in string columns (see get())
+    Returns
+    -------
+    ret : tuple or dictionary
+        The result of the query, as returned by get()
 
     Examples
     --------
-    This will extract the rows from the table sometable matching
-    to the provided array x
+    This extracts the rows of sometable matching the values of the
+    array x:
 
     >>> x = np.arange(10)
     >>> y = x**.5
     >>> sqlutilpy.local_join('''
-    ... SELECT s.* FROM mytable AS m LEFT JOIN sometable AS s
-    ... ON s.x = m.x ORDER BY m.xcol''',
+    ... select s.* from mytable as m left join sometable as s
+    ... on s.x = m.x order by m.x''',
     ... 'mytable', (x, y), ('x', 'y'))
     """
 

sqlutilpy-0.30.0/src/sqlutilpy/version.py

@@ -0,0 +1 @@
+__version__ = '0.30.0'

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0/src/sqlutilpy.egg-info}/PKG-INFO

@@ -1,25 +1,29 @@
 Metadata-Version: 2.4
 Name: sqlutilpy
-Version: 0.28.0
+Version: 0.30.0
 Summary: Database query code returning numpy arrays
 Author-email: "Sergey E. Koposov" <skoposov@ed.ac.uk>
-License: BSD
+License-Expression: BSD-3-Clause
 Project-URL: Homepage, https://github.com/segasai/sqlutilpy
-Keywords: numpy postgresql query sql sqlite array
-Classifier: Development Status :: 3 - Alpha
-Classifier: License :: OSI Approved :: BSD License
+Project-URL: Documentation, https://sqlutilpy.readthedocs.io
+Project-URL: Changelog, https://github.com/segasai/sqlutilpy/blob/master/CHANGELOG.md
+Keywords: numpy,postgresql,sqlite,duckdb,sql,query,array
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
 Classifier: Topic :: Utilities
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: numpy
+Requires-Dist: numpy>=2
 Requires-Dist: psycopg[binary]
 Dynamic: license-file
 
 [![Build Status](https://github.com/segasai/sqlutilpy/workflows/Testing/badge.svg)](https://github.com/segasai/sqlutilpy/actions)
 [![Documentation Status](https://readthedocs.org/projects/sqlutilpy/badge/?version=latest)](http://sqlutilpy.readthedocs.io/en/latest/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/segasai/sqlutilpy/badge.svg?branch=master)](https://coveralls.io/github/segasai/sqlutilpy?branch=master)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6867957.svg)](https://doi.org/10.5281/zenodo.6867957)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5160118.svg)](https://doi.org/10.5281/zenodo.5160118)
 
 # sqlutilpy
 Python module to query SQL databases and return numpy arrays, upload
@@ -58,7 +62,7 @@ This command will run the query and put the columns into variables ra,dec:
 
 ```python
 import sqlutilpy
-ra,dec = squtilpy.get('select ra,dec from mytable', 
+ra,dec = sqlutilpy.get('select ra,dec from mytable', 
                  host='HOST_NAME_OF_MY_PG_SERVER', 
                  db='THE_NAME_OF_MY_DB')
 ```
@@ -67,7 +71,19 @@ By default `sqlutilpy.get` executes the query and returns the tuple with
 arrays. One array for each column in the query result. 
 You can return the results as dictionary using `asDict` option.
 
-## Uploading your arrays as column in a table
+## SQLite and DuckDB
+
+The same `get()` and `execute()` functions work with SQLite and DuckDB
+files, by selecting the driver and giving the file name as the database:
+
+```python
+a, b = sqlutilpy.get('select a, b from mytable', db='mydata.db', driver='sqlite3')
+a, b = sqlutilpy.get('select a, b from mytable', db='mydata.duckdb', driver='duckdb')
+```
+
+Uploading tables and `local_join` are only available with PostgreSQL.
+
+## Uploading your arrays as columns in a table (PostgreSQL)
 
 You can use `sqlutilpy.upload` to upload your arrays as columns in a table.
 
@@ -78,7 +94,7 @@ sqlutilpy.upload('mytable',(x,y),('xcol','ycol'))
 ``` 
 This will create a table called `mytable` with columns `xcol` and `ycol` 
 
-## Join query involving your local data and the database table
+## Join query involving your local data and the database table (PostgreSQL)
 
 Sometimes it is beneficial to run a join query involving your local data and the data in the database.
 
@@ -102,7 +118,8 @@ is that it uploads the data to the database and runs a query against it.
 
 Often it is beneficial to preserve an open connection to the database. You can do that if you first 
 obtain the connection using `sqlutilpy.getConnection()` and then provide it directly
-to `sqlutil.get()` and similar commands using `conn=` keyword:
+to `sqlutilpy.get()` and similar commands using `conn=` keyword:
+```python
 conn = sqlutilpy.getConnection(db='mydb', user='meuser', password='something', host='hostname')
 R= sqlutilpy.get('select 1', conn=conn)
 R1= sqlutilpy.get('select 1', conn=conn)

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/src/sqlutilpy.egg-info/SOURCES.txt

@@ -1,3 +1,5 @@
+CHANGELOG.md
+CITATION.cff
 LICENSE.txt
 MANIFEST.in
 README.md

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/src/sqlutilpy.egg-info/requires.txt

@@ -1,2 +1,2 @@
-numpy
+numpy>=2
 psycopg[binary]

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/tests/test_pg.py

@@ -728,3 +728,22 @@ def test_string_growth(setup, batched):
 
     finally:
         sqlutil.execute(f"drop table if exists {table_name}", conn=conn)
+
+
+def test_notNamed_deprecated(setup):
+    # notNamed=True is a deprecated alias of batched=False
+    kw, conn = setup
+    with pytest.warns(DeprecationWarning, match='notNamed'):
+        a, = sqlutil.get('select sicol from sqlutil_test order by sicol',
+                         notNamed=True,
+                         conn=conn)
+    assert list(a) == [1, 11]
+    with pytest.warns(DeprecationWarning, match='notNamed'):
+        cur = sqlutil.getCursor(conn, driver='psycopg', notNamed=True)
+    assert not isinstance(cur, psycopg.ServerCursor)
+    cur.close()
+    with pytest.warns(DeprecationWarning, match='notNamed'):
+        cur = sqlutil.getCursor(conn, driver='psycopg', notNamed=False)
+    assert isinstance(cur, psycopg.ServerCursor)
+    cur.close()
+    conn.rollback()

{sqlutilpy-0.28.0 → sqlutilpy-0.30.0}/tests/test_sqlite.py

@@ -71,3 +71,35 @@ def test_unknown_driver(setup):
     with pytest.raises(sqlutil.SqlUtilException):
         sqlutil.getConnection(db='sql.db', driver='nosuchdriver')
     conn.close()
+
+
+def test_notNamed_deprecated(setup):
+    # notNamed is a deprecated alias of batched=not notNamed
+    with pytest.warns(DeprecationWarning, match='notNamed'):
+        a, b = sqlutil.get('select a,b from tmp', notNamed=True, **setup)
+    assert (len(a) == 4)
+    conn = sqlite3.connect('sql.db')
+    with pytest.warns(DeprecationWarning, match='notNamed'):
+        sqlutil.getCursor(conn, driver='sqlite3', notNamed=True)
+    conn.close()
+
+
+def test_keyword_only(setup):
+    # the options can only be given as keywords
+    import numpy as np
+    with pytest.raises(TypeError):
+        sqlutil.get('select a,b from tmp', None, 'sql.db', 'sqlite3')
+    with pytest.raises(TypeError):
+        sqlutil.execute('select 1', None, 'sql.db', 'sqlite3')
+    with pytest.raises(TypeError):
+        sqlutil.upload('tmp4', (np.arange(2), ), ('a', ), 'sql.db',
+                       'sqlite3')
+    with pytest.raises(TypeError):
+        sqlutil.local_join('select 1', 'tmp5', (np.arange(2), ), ('a', ),
+                           'sql.db', 'sqlite3')
+    with pytest.raises(TypeError):
+        sqlutil.getConnection('sql.db', 'sqlite3')
+    conn = sqlite3.connect('sql.db')
+    with pytest.raises(TypeError):
+        sqlutil.getCursor(conn, 'sqlite3')
+    conn.close()

sqlutilpy-0.28.0/MANIFEST.in

@@ -1,9 +0,0 @@
-include README.md
-include requirements.txt
-include setup.py
-include setup.cfg
-include LICENSE
-include MANIFEST.in
-recursive-include tests *py
-recursive-include py *py
-

sqlutilpy-0.28.0/pyproject.toml

@@ -1,31 +0,0 @@
-[project]
-name = "sqlutilpy"
-dynamic = ["version"] 
-description = "Database query code returning numpy arrays"
-authors = [
-    {name = "Sergey E. Koposov", email = "skoposov@ed.ac.uk"},
-]
-dependencies = [
-    "numpy",
-    "psycopg[binary]"
-]
-requires-python = ">=3.8"
-readme = "README.md"
-license = {text = "BSD"}
-keywords = ["numpy postgresql query sql sqlite array"]
-classifiers = [
-    "Development Status :: 3 - Alpha",
-    "License :: OSI Approved :: BSD License",
-    "Topic :: Utilities",
-]
-
-[build-system]
-requires = ["setuptools", "wheel", "setuptools-git-versioning"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools-git-versioning]
-enabled = true
-
-[project.urls]
-Homepage = "https://github.com/segasai/sqlutilpy"
-

sqlutilpy-0.28.0/src/sqlutilpy/version.py

@@ -1 +0,0 @@
-__version__ = '0.28.0'