singlestoredb 1.0.4__tar.gz → 1.2.0__tar.gz

{singlestoredb-1.0.4/singlestoredb.egg-info → singlestoredb-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: singlestoredb
-Version: 1.0.4
+Version: 1.2.0
 Summary: Interface to the SingleStoreDB database and workspace management APIs
 Home-page: https://github.com/singlestore-labs/singlestoredb-python
 Author: SingleStore

{singlestoredb-1.0.4 → singlestoredb-1.2.0}/accel.c

@@ -16,6 +16,10 @@
 #define ACCEL_OUT_STRUCTSEQUENCES 1
 #define ACCEL_OUT_DICTS 2
 #define ACCEL_OUT_NAMEDTUPLES 3
+#define ACCEL_OUT_NUMPY 4
+#define ACCEL_OUT_PANDAS 5
+#define ACCEL_OUT_POLARS 6
+#define ACCEL_OUT_ARROW 7
 
 #define NUMPY_BOOL 1
 #define NUMPY_INT8 2
@@ -77,6 +81,21 @@
 #define MYSQL_TYPE_STRING 254
 #define MYSQL_TYPE_GEOMETRY 255
 
+// SingleStoreDB extended types
+#define MYSQL_TYPE_BSON 1001
+#define MYSQL_TYPE_FLOAT32_VECTOR_JSON 2001
+#define MYSQL_TYPE_FLOAT64_VECTOR_JSON 2002
+#define MYSQL_TYPE_INT8_VECTOR_JSON 2003
+#define MYSQL_TYPE_INT16_VECTOR_JSON 2004
+#define MYSQL_TYPE_INT32_VECTOR_JSON 2005
+#define MYSQL_TYPE_INT64_VECTOR_JSON 2006
+#define MYSQL_TYPE_FLOAT32_VECTOR 3001
+#define MYSQL_TYPE_FLOAT64_VECTOR 3002
+#define MYSQL_TYPE_INT8_VECTOR 3003
+#define MYSQL_TYPE_INT16_VECTOR 3004
+#define MYSQL_TYPE_INT32_VECTOR 3005
+#define MYSQL_TYPE_INT64_VECTOR 3006
+
 #define MYSQL_TYPE_CHAR MYSQL_TYPE_TINY
 #define MYSQL_TYPE_INTERVAL MYSQL_TYPE_ENUM
 
@@ -329,6 +348,8 @@ typedef struct {
 inline int IMAX(int a, int b) { return((a) > (b) ? a : b); }
 inline int IMIN(int a, int b) { return((a) < (b) ? a : b); }
 
+static PyObject *create_numpy_array(PyObject *py_memview, char *data_format, int data_type, PyObject *py_objs);
+
 char *_PyUnicode_AsUTF8(PyObject *unicode) {
     PyObject *bytes = PyUnicode_AsEncodedString(unicode, "utf-8", "strict");
     if (!bytes) return NULL;
@@ -389,6 +410,17 @@ typedef struct {
     PyObject *Series;
     PyObject *array;
     PyObject *vectorize;
+    PyObject *DataFrame;
+    PyObject *Table;
+    PyObject *from_pylist;
+    PyObject *int8;
+    PyObject *int16;
+    PyObject *int32;
+    PyObject *int64;
+    PyObject *float32;
+    PyObject *float64;
+    PyObject *unpack;
+    PyObject *decode;
 } PyStrings;
 
 static PyStrings PyStr = {0};
@@ -406,6 +438,12 @@ typedef struct {
     PyObject *collections_namedtuple;
     PyObject *numpy_array;
     PyObject *numpy_vectorize;
+    PyObject *pandas_DataFrame;
+    PyObject *polars_DataFrame;
+    PyObject *pyarrow_Table;
+    PyObject *pyarrow_Table_from_pylist;
+    PyObject *struct_unpack;
+    PyObject *bson_decode;
 } PyFunctions;
 
 static PyFunctions PyFunc = {0};
@@ -417,6 +455,9 @@ typedef struct {
     PyObject *namedtuple_kwargs;
     PyObject *create_numpy_array_args;
     PyObject *create_numpy_array_kwargs;
+    PyObject *create_numpy_array_kwargs_vector[7];
+    PyObject *struct_unpack_args;
+    PyObject *bson_decode_args;
 } PyObjects;
 
 static PyObjects PyObj = {0};
@@ -466,10 +507,111 @@ typedef struct {
     char *encoding_errors;
 } StateObject;
 
-static void read_options(MySQLAccelOptions *options, PyObject *dict);
+static int read_options(MySQLAccelOptions *options, PyObject *dict);
 
 #define DESTROY(x) do { if (x) { free((void*)x); (x) = NULL; } } while (0)
 
+int ensure_numpy() {
+    if (PyFunc.numpy_array && PyFunc.numpy_vectorize) goto exit;
+
+    // Import numpy if it exists
+    PyObject *numpy_mod = PyImport_ImportModule("numpy");
+    if (!numpy_mod) goto error;
+
+    PyFunc.numpy_array = PyObject_GetAttr(numpy_mod, PyStr.array);
+    if (!PyFunc.numpy_array) goto error;
+
+    PyFunc.numpy_vectorize = PyObject_GetAttr(numpy_mod, PyStr.vectorize);
+    if (!PyFunc.numpy_vectorize) goto error;
+
+exit:
+    return 0;
+
+error:
+    PyErr_Clear();
+    return -1;
+}
+
+
+int ensure_pandas() {
+    if (PyFunc.pandas_DataFrame) goto exit;
+
+    // Import pandas if it exists
+    PyObject *pandas_mod = PyImport_ImportModule("pandas");
+    if (!pandas_mod) goto error;
+
+    PyFunc.pandas_DataFrame = PyObject_GetAttr(pandas_mod, PyStr.DataFrame);
+    if (!PyFunc.pandas_DataFrame) goto error;
+
+exit:
+    return 0;
+
+error:
+    PyErr_Clear();
+    return -1;
+}
+
+
+int ensure_polars() {
+    if (PyFunc.polars_DataFrame) goto exit;
+
+    // Import polars if it exists
+    PyObject *polars_mod = PyImport_ImportModule("polars");
+    if (!polars_mod) goto error;
+
+    PyFunc.polars_DataFrame = PyObject_GetAttr(polars_mod, PyStr.DataFrame);
+    if (!PyFunc.polars_DataFrame) goto error;
+
+exit:
+    return 0;
+
+error:
+    PyErr_Clear();
+    return -1;
+}
+
+
+int ensure_pyarrow() {
+    if (PyFunc.pyarrow_Table_from_pylist) goto exit;
+
+    // Import pyarrow if it exists
+    PyObject *pyarrow_mod = PyImport_ImportModule("pyarrow");
+    if (!pyarrow_mod) goto error;
+
+    PyFunc.pyarrow_Table = PyObject_GetAttr(pyarrow_mod, PyStr.Table);
+    if (!PyFunc.pyarrow_Table) goto error;
+
+    PyFunc.pyarrow_Table_from_pylist = PyObject_GetAttr(PyFunc.pyarrow_Table, PyStr.from_pylist);
+    if (!PyFunc.pyarrow_Table_from_pylist) goto error;
+
+exit:
+    return 0;
+
+error:
+    PyErr_Clear();
+    return -1;
+}
+
+
+int ensure_bson() {
+    if (PyFunc.bson_decode) goto exit;
+
+    // Import bson if it exists
+    PyObject *bson_mod = PyImport_ImportModule("bson");
+    if (!bson_mod) goto error;
+
+    PyFunc.bson_decode = PyObject_GetAttr(bson_mod, PyStr.decode);
+    if (!PyFunc.bson_decode) goto error;
+
+exit:
+    return 0;
+
+error:
+    PyErr_Clear();
+    return -1;
+}
+
+
 static void State_clear_fields(StateObject *self) {
     if (!self) return;
     DESTROY(self->offsets);
@@ -679,9 +821,10 @@ static int State_init(StateObject *self, PyObject *args, PyObject *kwds) {
                                       NULL : py_converter;
         Py_XINCREF(self->py_invalid_values[i]);
 
-        self->py_converters[i] = (!py_converter
-                                  || py_converter == Py_None
-                                  || py_converter == py_default_converter) ?
+        self->py_converters[i] = ((!py_converter || py_converter == py_default_converter)
+                                   // TODO: Need C accelerated converters for extended types
+                                   // && self->type_codes[i] < 256
+                                   ) ?
                                  NULL : py_converter;
         Py_XINCREF(self->py_converters[i]);
     }
@@ -709,7 +852,8 @@ static int State_init(StateObject *self, PyObject *args, PyObject *kwds) {
     Py_XDECREF(py_next_seq_id);
 
     if (py_options && PyDict_Check(py_options)) {
-        read_options(&self->options, py_options);
+        rc = read_options(&self->options, py_options);
+        if (rc) goto error;
     }
 
     switch (self->options.results_type) {
@@ -825,12 +969,13 @@ static PyType_Spec StateType_spec = {
 // End State
 //
 
-static void read_options(MySQLAccelOptions *options, PyObject *dict) {
-    if (!options || !dict) return;
+static int read_options(MySQLAccelOptions *options, PyObject *dict) {
+    if (!options || !dict) return 0;
 
     PyObject *key = NULL;
     PyObject *value = NULL;
     Py_ssize_t pos = 0;
+    int rc = 0;
 
     while (PyDict_Next(dict, &pos, &key, &value)) {
         if (PyUnicode_CompareWithASCIIString(key, "results_type") == 0) {
@@ -846,6 +991,23 @@ static void read_options(MySQLAccelOptions *options, PyObject *dict) {
                      PyUnicode_CompareWithASCIIString(value, "structsequences") == 0) {
                 options->results_type = ACCEL_OUT_STRUCTSEQUENCES;
             }
+            else if (PyUnicode_CompareWithASCIIString(value, "numpy") == 0) {
+                options->results_type = ACCEL_OUT_NUMPY;
+                rc = ensure_numpy();
+            }
+            else if (PyUnicode_CompareWithASCIIString(value, "pandas") == 0) {
+                options->results_type = ACCEL_OUT_PANDAS;
+                rc = ensure_pandas();
+            }
+            else if (PyUnicode_CompareWithASCIIString(value, "polars") == 0) {
+                options->results_type = ACCEL_OUT_POLARS;
+                rc = ensure_polars();
+            }
+            else if (PyUnicode_CompareWithASCIIString(value, "arrow") == 0 ||
+                     PyUnicode_CompareWithASCIIString(value, "pyarrow") == 0) {
+                options->results_type = ACCEL_OUT_ARROW;
+                rc = ensure_pyarrow();
+            }
             else {
                 options->results_type = ACCEL_OUT_TUPLES;
             }
@@ -857,6 +1019,8 @@ static void read_options(MySQLAccelOptions *options, PyObject *dict) {
             }
         }
     }
+
+    return rc;
 }
 
 static void raise_exception(
@@ -1310,7 +1474,10 @@ static PyObject *read_row_from_packet(
     PyObject *py_result = NULL;
     PyObject *py_item = NULL;
     PyObject *py_str = NULL;
+    PyObject *py_memview = NULL;
     char end = '\0';
+    char *cast_type_codes[] = {"", "f", "d", "b", "h", "l", "q"};
+    int item_type_lengths[] = {0, 4, 8, 1, 2, 4, 8};
 
     int sign = 1;
     int year = 0;
@@ -1323,9 +1490,10 @@ static PyObject *read_row_from_packet(
 
     switch (py_state->options.results_type) {
     case ACCEL_OUT_DICTS:
+    case ACCEL_OUT_ARROW:
         py_result = PyDict_New();
         break;
-   case ACCEL_OUT_STRUCTSEQUENCES: {
+    case ACCEL_OUT_STRUCTSEQUENCES: {
         if (!py_state->structsequence) goto error;
         py_result = PyStructSequence_New(py_state->structsequence);
         break;
@@ -1362,8 +1530,12 @@ static PyObject *read_row_from_packet(
                     py_str = PyUnicode_Decode(out, out_l, py_state->encodings[i], py_state->encoding_errors);
                     if (!py_str) goto error;
                 }
-                py_item = PyObject_CallFunctionObjArgs(py_state->py_converters[i], py_str, NULL);
-                Py_CLEAR(py_str);
+                if (py_state->py_converters[i] == Py_None) {
+                    py_item = py_str;
+                } else {
+                    py_item = PyObject_CallFunctionObjArgs(py_state->py_converters[i], py_str, NULL);
+                    Py_CLEAR(py_str);
+                }
                 if (!py_item) goto error;
             }
 
@@ -1550,6 +1722,12 @@ static PyObject *read_row_from_packet(
                 case MYSQL_TYPE_VARCHAR:
                 case MYSQL_TYPE_VAR_STRING:
                 case MYSQL_TYPE_STRING:
+                case MYSQL_TYPE_FLOAT32_VECTOR_JSON:
+                case MYSQL_TYPE_FLOAT64_VECTOR_JSON:
+                case MYSQL_TYPE_INT8_VECTOR_JSON:
+                case MYSQL_TYPE_INT16_VECTOR_JSON:
+                case MYSQL_TYPE_INT32_VECTOR_JSON:
+                case MYSQL_TYPE_INT64_VECTOR_JSON:
                     if (!py_state->encodings[i]) {
                         py_item = PyBytes_FromStringAndSize(out, out_l);
                         if (!py_item) goto error;
@@ -1560,13 +1738,86 @@ static PyObject *read_row_from_packet(
                     if (!py_item) goto error;
 
                     // Parse JSON string.
-                    if (py_state->type_codes[i] == MYSQL_TYPE_JSON && py_state->options.parse_json) {
+                    if ((py_state->type_codes[i] == MYSQL_TYPE_JSON && py_state->options.parse_json)
+                        || (py_state->type_codes[i] >= MYSQL_TYPE_FLOAT32_VECTOR_JSON
+                            && py_state->type_codes[i] <= MYSQL_TYPE_INT64_VECTOR_JSON)) {
                         py_str = py_item;
                         py_item = PyObject_CallFunctionObjArgs(PyFunc.json_loads, py_str, NULL);
                         Py_CLEAR(py_str);
                         if (!py_item) goto error;
                     }
 
+                    if (ensure_numpy() == 0) {
+                        switch (py_state->type_codes[i]) {
+                        case MYSQL_TYPE_FLOAT32_VECTOR_JSON:
+                        case MYSQL_TYPE_FLOAT64_VECTOR_JSON:
+                        case MYSQL_TYPE_INT8_VECTOR_JSON:
+                        case MYSQL_TYPE_INT16_VECTOR_JSON:
+                        case MYSQL_TYPE_INT32_VECTOR_JSON:
+                        case MYSQL_TYPE_INT64_VECTOR_JSON:
+                            CHECKRC(PyTuple_SetItem(PyObj.create_numpy_array_args, 0, py_item));
+                            py_item = PyObject_Call(
+                                PyFunc.numpy_array,
+                                PyObj.create_numpy_array_args,
+                                PyObj.create_numpy_array_kwargs_vector[py_state->type_codes[i] % 1000]
+                            );
+                            if (!py_item) goto error;
+                        }
+                    }
+
+                    break;
+
+                case MYSQL_TYPE_FLOAT32_VECTOR:
+                case MYSQL_TYPE_FLOAT64_VECTOR:
+                case MYSQL_TYPE_INT8_VECTOR:
+                case MYSQL_TYPE_INT16_VECTOR:
+                case MYSQL_TYPE_INT32_VECTOR:
+                case MYSQL_TYPE_INT64_VECTOR:
+                    if (ensure_numpy() == 0) {
+                        py_memview = PyMemoryView_FromMemory(out, out_l, PyBUF_WRITE);
+                        if (!py_memview) goto error;
+
+                        py_item = create_numpy_array(
+                            py_memview,
+                            cast_type_codes[py_state->type_codes[i] % 1000],
+                            py_state->type_codes[i],
+                            NULL
+                        );
+                        Py_CLEAR(py_memview);
+                        if (!py_item) goto error;
+
+                    } else {
+                        py_memview = PyBytes_FromStringAndSize(out, out_l);
+                        if (!py_memview) goto error;
+
+                        CHECKRC(PyTuple_SetItem(PyObj.struct_unpack_args, 0,
+                                                PyUnicode_FromFormat("<%l%s", out_l / item_type_lengths[i], cast_type_codes[i])));
+                        CHECKRC(PyTuple_SetItem(PyObj.struct_unpack_args, 1, py_memview));
+
+                        py_item = PyObject_Call(
+                            PyFunc.struct_unpack,
+                            PyObj.struct_unpack_args,
+                            NULL
+                        );
+                        if (!py_item) goto error;
+                    }
+
+                    break;
+
+                case MYSQL_TYPE_BSON:
+                    py_item = PyBytes_FromStringAndSize(out, out_l);
+                    if (!py_item) goto error;
+
+                    if (ensure_bson() == 0) {
+                        CHECKRC(PyTuple_SetItem(PyObj.bson_decode_args, 0, py_item));
+                        py_item = PyObject_Call(
+                            PyFunc.bson_decode,
+                            PyObj.bson_decode_args,
+                            NULL
+                        );
+                        if (!py_item) goto error;
+                    }
+
                     break;
 
                 default:
@@ -1586,6 +1837,7 @@ static PyObject *read_row_from_packet(
             PyStructSequence_SetItem(py_result, i, py_item);
             break;
         case ACCEL_OUT_DICTS:
+        case ACCEL_OUT_ARROW:
             PyDict_SetItem(py_result, py_state->py_names[i], py_item);
             Py_INCREF(py_state->py_names[i]);
             Py_DECREF(py_item);
@@ -1847,27 +2099,6 @@ error:
 }
 
 
-int ensure_numpy() {
-    if (PyFunc.numpy_array && PyFunc.numpy_vectorize) goto exit;
-
-    // Import numpy if it exists
-    PyObject *numpy_mod = PyImport_ImportModule("numpy");
-    if (!numpy_mod) goto error;
-
-    PyFunc.numpy_array = PyObject_GetAttr(numpy_mod, PyStr.array);
-    if (!PyFunc.numpy_array) goto error;
-
-    PyFunc.numpy_vectorize = PyObject_GetAttr(numpy_mod, PyStr.vectorize);
-    if (!PyFunc.numpy_vectorize) goto error;
-
-exit:
-    return 0;
-
-error:
-    return -1;
-}
-
-
 static PyObject *load_rowdat_1_numpy(PyObject *self, PyObject *args, PyObject *kwargs) {
     PyObject *py_data = NULL;
     PyObject *py_out = NULL;
@@ -4372,6 +4603,17 @@ PyMODINIT_FUNC PyInit__singlestoredb_accel(void) {
     PyStr.Series = PyUnicode_FromString("Series");
     PyStr.array = PyUnicode_FromString("array");
     PyStr.vectorize = PyUnicode_FromString("vectorize");
+    PyStr.DataFrame = PyUnicode_FromString("DataFrame");
+    PyStr.Table = PyUnicode_FromString("Table");
+    PyStr.from_pylist = PyUnicode_FromString("from_pylist");
+    PyStr.int8 = PyUnicode_FromString("int8");
+    PyStr.int16 = PyUnicode_FromString("int16");
+    PyStr.int32 = PyUnicode_FromString("int32");
+    PyStr.int64 = PyUnicode_FromString("int64");
+    PyStr.float32 = PyUnicode_FromString("float32");
+    PyStr.float64 = PyUnicode_FromString("float64");
+    PyStr.unpack = PyUnicode_FromString("unpack");
+    PyStr.decode = PyUnicode_FromString("decode");
 
     PyObject *decimal_mod = PyImport_ImportModule("decimal");
     if (!decimal_mod) goto error;
@@ -4381,6 +4623,8 @@ PyMODINIT_FUNC PyInit__singlestoredb_accel(void) {
     if (!json_mod) goto error;
     PyObject *collections_mod = PyImport_ImportModule("collections");
     if (!collections_mod) goto error;
+    PyObject *struct_mod = PyImport_ImportModule("struct");
+    if (!struct_mod) goto error;
 
     PyFunc.decimal_Decimal = PyObject_GetAttr(decimal_mod, PyStr.Decimal);
     if (!PyFunc.decimal_Decimal) goto error;
@@ -4396,6 +4640,8 @@ PyMODINIT_FUNC PyInit__singlestoredb_accel(void) {
     if (!PyFunc.json_loads) goto error;
     PyFunc.collections_namedtuple = PyObject_GetAttr(collections_mod, PyStr.namedtuple);
     if (!PyFunc.collections_namedtuple) goto error;
+    PyFunc.struct_unpack = PyObject_GetAttr(struct_mod, PyStr.unpack);
+    if (!PyFunc.struct_unpack) goto error;
 
     PyObj.namedtuple_kwargs = PyDict_New();
     if (!PyObj.namedtuple_kwargs) goto error;
@@ -4412,6 +4658,43 @@ PyMODINIT_FUNC PyInit__singlestoredb_accel(void) {
         goto error;
     }
 
+    PyObj.create_numpy_array_kwargs_vector[1] = PyDict_New();
+    if (!PyObj.create_numpy_array_kwargs_vector[1]) goto error;
+    if (PyDict_SetItemString(PyObj.create_numpy_array_kwargs_vector[1], "dtype", PyStr.float32)) {
+        goto error;
+    }
+    PyObj.create_numpy_array_kwargs_vector[2] = PyDict_New();
+    if (!PyObj.create_numpy_array_kwargs_vector[2]) goto error;
+    if (PyDict_SetItemString(PyObj.create_numpy_array_kwargs_vector[2], "dtype", PyStr.float64)) {
+        goto error;
+    }
+    PyObj.create_numpy_array_kwargs_vector[3] = PyDict_New();
+    if (!PyObj.create_numpy_array_kwargs_vector[3]) goto error;
+    if (PyDict_SetItemString(PyObj.create_numpy_array_kwargs_vector[3], "dtype", PyStr.int8)) {
+        goto error;
+    }
+    PyObj.create_numpy_array_kwargs_vector[4] = PyDict_New();
+    if (!PyObj.create_numpy_array_kwargs_vector[4]) goto error;
+    if (PyDict_SetItemString(PyObj.create_numpy_array_kwargs_vector[4], "dtype", PyStr.int16)) {
+        goto error;
+    }
+    PyObj.create_numpy_array_kwargs_vector[5] = PyDict_New();
+    if (!PyObj.create_numpy_array_kwargs_vector[5]) goto error;
+    if (PyDict_SetItemString(PyObj.create_numpy_array_kwargs_vector[5], "dtype", PyStr.int32)) {
+        goto error;
+    }
+    PyObj.create_numpy_array_kwargs_vector[6] = PyDict_New();
+    if (!PyObj.create_numpy_array_kwargs_vector[6]) goto error;
+    if (PyDict_SetItemString(PyObj.create_numpy_array_kwargs_vector[6], "dtype", PyStr.int64)) {
+        goto error;
+    }
+
+    PyObj.struct_unpack_args = PyTuple_New(2);
+    if (!PyObj.struct_unpack_args) goto error;
+
+    PyObj.bson_decode_args = PyTuple_New(1);
+    if (!PyObj.bson_decode_args) goto error;
+
     return PyModule_Create(&_singlestoredb_accelmodule);
 
 error:

{singlestoredb-1.0.4 → singlestoredb-1.2.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = singlestoredb
-version = 1.0.4
+version = 1.2.0
 description = Interface to the SingleStoreDB database and workspace management APIs
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -26,6 +26,7 @@ install_requires =
 	setuptools
 	sqlparams
 	wheel
+	tomli>=1.1.0;python_version < '3.11'
 python_requires = >=3.8
 tests_require = 
 	coverage
@@ -77,7 +78,7 @@ exclude =
 	docs/*
 	resources/*
 	licenses/*
-max-complexity = 30
+max-complexity = 35
 max-line-length = 90
 per-file-ignores = 
 	singlestoredb/__init__.py:F401

{singlestoredb-1.0.4 → singlestoredb-1.2.0}/singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.0.4'
+__version__ = '1.2.0'
 
 from typing import Any