arraykit 0.8.1__tar.gz → 0.8.3__tar.gz

arraykit-0.8.3/MANIFEST.in

@@ -0,0 +1,2 @@
+include src/__init__.pyi src/py.typed
+include src/*.h

{arraykit-0.8.1/arraykit.egg-info → arraykit-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arraykit
-Version: 0.8.1
+Version: 0.8.3
 Summary: Array utilities for StaticFrame
 Home-page: https://github.com/static-frame/arraykit
 Author: Christopher Ariza, Brandt Bucher, Charles Burkland

{arraykit-0.8.1 → arraykit-0.8.3}/README.rst

@@ -37,11 +37,22 @@ ArrayKit requires the following:
 What is New in ArrayKit
 -------------------------
 
+0.8.3
+............
+
+Corrected incorrect component of ``_TLabel`` in ``__init__.pyi``.
+
+
+0.8.2
+............
+
+Updated build configuration.
+
 
 0.8.1
 ............
 
-Updated build ``include_dirs`` configuration.
+Updated build configuration.
 
 
 0.8.0

{arraykit-0.8.1 → arraykit-0.8.3/arraykit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arraykit
-Version: 0.8.1
+Version: 0.8.3
 Summary: Array utilities for StaticFrame
 Home-page: https://github.com/static-frame/arraykit
 Author: Christopher Ariza, Brandt Bucher, Charles Burkland

{arraykit-0.8.1 → arraykit-0.8.3}/arraykit.egg-info/SOURCES.txt

@@ -11,12 +11,19 @@ src/__init__.py
 src/__init__.pyi
 src/_arraykit.c
 src/array_go.c
+src/array_go.h
 src/array_to_tuple.c
+src/array_to_tuple.h
 src/block_index.c
+src/block_index.h
 src/delimited_to_arrays.c
+src/delimited_to_arrays.h
 src/methods.c
+src/methods.h
 src/py.typed
 src/tri_map.c
+src/tri_map.h
+src/utilities.h
 test/test_array_go.py
 test/test_block_index.py
 test/test_delimited_to_arrays.py

{arraykit-0.8.1 → arraykit-0.8.3}/setup.py

@@ -5,7 +5,7 @@ from setuptools import Extension  # type: ignore
 from setuptools import setup
 from pathlib import Path
 
-AK_VERSION = '0.8.1'
+AK_VERSION = '0.8.3'
 
 def get_long_description() -> str:
     return '''The ArrayKit library provides utilities for creating and transforming NumPy arrays, implementing performance-critical StaticFrame operations as Python C extensions.

{arraykit-0.8.1 → arraykit-0.8.3}/src/__init__.pyi

@@ -23,7 +23,7 @@ _TLabel = tp.Union[
         np.timedelta64,
         datetime.date,
         datetime.datetime,
-        tp.Tuple['_TLabel'],
+        tp.Tuple['_TLabel', ...],
 ]
 
 class ErrorInitTypeBlocks(RuntimeError):

arraykit-0.8.3/src/array_go.h

@@ -0,0 +1,8 @@
+# ifndef ARRAYKIT_SRC_ARRAY_GO_H_
+# define ARRAYKIT_SRC_ARRAY_GO_H_
+
+# include "Python.h"
+
+extern PyTypeObject ArrayGOType;
+
+# endif /* ARRAYKIT_SRC_ARRAY_GO_H_ */

arraykit-0.8.3/src/array_to_tuple.h

@@ -0,0 +1,14 @@
+# ifndef ARRAYKIT_SRC_ARRAY_TO_TUPLE_H_
+# define ARRAYKIT_SRC_ARRAY_TO_TUPLE_H_
+
+# include "Python.h"
+
+// Given a 1D or 2D array, return a 1D object array of tuples.
+PyObject *
+array_to_tuple_array(PyObject *Py_UNUSED(m), PyObject *a);
+
+// Given a 2D array, return an iterator of row tuples.
+PyObject *
+array_to_tuple_iter(PyObject *Py_UNUSED(m), PyObject *a);
+
+# endif // ARRAYKIT_SRC_ARRAY_TO_TUPLE_H_

arraykit-0.8.3/src/block_index.h

@@ -0,0 +1,15 @@
+# ifndef ARRAYKIT_SRC_BLOCK_INDEX_H_
+# define ARRAYKIT_SRC_BLOCK_INDEX_H_
+
+# include "Python.h"
+
+extern PyTypeObject BlockIndexType;
+extern PyObject * ErrorInitTypeBlocks;
+extern PyTypeObject BIIterType;
+extern PyTypeObject BIIterSeqType;
+extern PyTypeObject BIIterSliceType;
+extern PyTypeObject BIIterBoolType;
+extern PyTypeObject BIIterContiguousType;
+extern PyTypeObject BIIterBlockType;
+
+# endif /* ARRAYKIT_SRC_BLOCK_INDEX_H_ */

arraykit-0.8.3/src/delimited_to_arrays.h

@@ -0,0 +1,16 @@
+# ifndef ARRAYKIT_SRC_DELIMITED_TO_ARRAYS_H_
+# define ARRAYKIT_SRC_DELIMITED_TO_ARRAYS_H_
+
+# include "Python.h"
+
+// NOTE: implement skip_header, skip_footer in client Python, not here.
+PyObject *
+delimited_to_arrays(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kwargs);
+
+PyObject *
+iterable_str_to_array_1d(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kwargs);
+
+PyObject *
+split_after_count(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kwargs);
+
+# endif /* ARRAYKIT_SRC_DELIMITED_TO_ARRAYS_H_ */

arraykit-0.8.3/src/methods.h

@@ -0,0 +1,72 @@
+# ifndef ARRAYKIT_SRC_METHODS_H
+# define ARRAYKIT_SRC_METHODS_H
+# include "Python.h"
+
+// A fast counter of unsized iterators
+PyObject *
+count_iteration(PyObject *Py_UNUSED(m), PyObject *iterable);
+
+// Reshape if necessary a row that might be 2D or 1D is returned as a 1D array.
+PyObject *
+row_1d_filter(PyObject *Py_UNUSED(m), PyObject *a);
+
+// Convert any slice to an ascending slice that covers the same values.
+PyObject *
+slice_to_ascending_slice(PyObject *Py_UNUSED(m), PyObject *args);
+
+// Reshape if necessary a flat ndim 1 array into a 2D array with one columns and rows of length.
+// related example: https://github.com/RhysU/ar/blob/master/ar-python.cpp
+PyObject *
+column_2d_filter(PyObject *Py_UNUSED(m), PyObject *a);
+
+// Reshape if necessary a column that might be 2D or 1D is returned as a 1D array.
+PyObject *
+column_1d_filter(PyObject *Py_UNUSED(m), PyObject *a);
+
+// Represent a 1D array as a 2D array with length as rows of a single-column array.
+// https://stackoverflow.com/questions/56182259/how-does-one-acces-numpy-multidimensionnal-array-in-c-extensions
+PyObject *
+shape_filter(PyObject *Py_UNUSED(m), PyObject *a);
+
+PyObject *
+name_filter(PyObject *Py_UNUSED(m), PyObject *n);
+
+// Return the integer version of the pointer to underlying data-buffer of array.
+PyObject *
+mloc(PyObject *Py_UNUSED(m), PyObject *a);
+
+PyObject *
+immutable_filter(PyObject *Py_UNUSED(m), PyObject *a);
+
+PyObject *
+resolve_dtype(PyObject *Py_UNUSED(m), PyObject *args);
+
+PyObject *
+resolve_dtype_iter(PyObject *Py_UNUSED(m), PyObject *arg);
+
+PyObject *
+nonzero_1d(PyObject *Py_UNUSED(m), PyObject *a);
+
+PyObject *
+first_true_1d(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kwargs);
+
+PyObject *
+first_true_2d(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kwargs);
+
+PyObject *
+dtype_from_element(PyObject *Py_UNUSED(m), PyObject *arg);
+
+PyObject *
+isna_element(PyObject *m, PyObject *args, PyObject *kwargs);
+
+PyObject *
+get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kwargs);
+
+// Specialized array deepcopy that stores immutable arrays in an optional memo dict that can be provided with kwargs.
+PyObject *
+array_deepcopy(PyObject *m, PyObject *args, PyObject *kwargs);
+
+PyObject *
+immutable_filter(PyObject *Py_UNUSED(m), PyObject *a);
+
+# endif /* ARRAYKIT_SRC_METHODS_H */

arraykit-0.8.3/src/tri_map.h

@@ -0,0 +1,8 @@
+# ifndef ARRAYKIT_SRC_TRI_MAP_H_
+# define ARRAYKIT_SRC_TRI_MAP_H_
+
+# include "Python.h"
+
+extern PyTypeObject TriMapType;
+
+# endif /* ARRAYKIT_SRC_TRI_MAP_H_ */

arraykit-0.8.3/src/utilities.h

@@ -0,0 +1,321 @@
+#ifndef ARRAYKIT_SRC_UTILITIES_H_
+#define ARRAYKIT_SRC_UTILITIES_H_
+
+# include "Python.h"
+# include "stdbool.h"
+
+# define NO_IMPORT_ARRAY
+# define PY_ARRAY_UNIQUE_SYMBOL AK_ARRAY_API
+# define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+# include "numpy/arrayobject.h"
+
+static const size_t UCS4_SIZE = sizeof(Py_UCS4);
+
+//------------------------------------------------------------------------------
+// Macros
+//------------------------------------------------------------------------------
+
+// Given a PyObject, raise if not an array.
+# define AK_CHECK_NUMPY_ARRAY(O)                 \
+    if (!PyArray_Check(O)) {                     \
+        return PyErr_Format(PyExc_TypeError,     \
+                "Expected NumPy array, not %s.", \
+                Py_TYPE(O)->tp_name);            \
+    }
+
+// Given a PyObject, raise if not an array or is not one or two dimensional.
+# define AK_CHECK_NUMPY_ARRAY_1D_2D(O)                    \
+    do {                                                  \
+        AK_CHECK_NUMPY_ARRAY(O)                           \
+        int ndim = PyArray_NDIM((PyArrayObject *)O);      \
+        if (ndim != 1 && ndim != 2) {                     \
+            return PyErr_Format(PyExc_NotImplementedError,\
+                    "Expected 1D or 2D array, not %i.",   \
+                    ndim);                                \
+        }                                                 \
+    } while (0)
+
+// Given a PyObject, raise if not an array or is not two dimensional.
+# define AK_CHECK_NUMPY_ARRAY_2D(O)                       \
+    do {                                                  \
+        AK_CHECK_NUMPY_ARRAY(O)                           \
+        int ndim = PyArray_NDIM((PyArrayObject *)O);      \
+        if (ndim != 2) {                                  \
+            return PyErr_Format(PyExc_NotImplementedError,\
+                    "Expected a 2D array, not %i.",       \
+                    ndim);                                \
+        }                                                 \
+    } while (0)
+
+# if defined __GNUC__ || defined __clang__
+# define AK_LIKELY(X) __builtin_expect(!!(X), 1)
+# define AK_UNLIKELY(X) __builtin_expect(!!(X), 0)
+# else
+# define AK_LIKELY(X) (!!(X))
+# define AK_UNLIKELY(X) (!!(X))
+# endif
+
+// Placeholder of not implemented pathways / debugging.
+# define AK_NOT_IMPLEMENTED(msg)                        \
+    do {                                                \
+        PyErr_SetString(PyExc_NotImplementedError, msg);\
+        return NULL;                                    \
+    } while (0)
+
+# define _AK_DEBUG_BEGIN() \
+    do {                   \
+        fprintf(stderr, "--- %s: %i: %s: ", __FILE__, __LINE__, __FUNCTION__);
+
+# define _AK_DEBUG_END()       \
+        fprintf(stderr, "\n"); \
+        fflush(stderr);        \
+    } while (0)
+
+# define AK_DEBUG(msg)          \
+    _AK_DEBUG_BEGIN();          \
+        fprintf(stderr, #msg);  \
+    _AK_DEBUG_END()
+
+# define AK_DEBUG_MSG_OBJ(msg, obj)     \
+    _AK_DEBUG_BEGIN();                  \
+        fprintf(stderr, #msg " ");      \
+        PyObject_Print(obj, stderr, 0); \
+    _AK_DEBUG_END()
+
+# define AK_DEBUG_MSG_REFCNT(msg, obj)     \
+    _AK_DEBUG_BEGIN();                  \
+        fprintf(stderr, #msg " ref count: ");      \
+        PyObject_Print(PyLong_FromSsize_t(Py_REFCNT((PyObject*)obj)), stderr, 0); \
+    _AK_DEBUG_END()
+
+# define AK_DEBUG_OBJ(obj)              \
+    _AK_DEBUG_BEGIN();                  \
+        fprintf(stderr, #obj " = ");    \
+        PyObject_Print(obj, stderr, 0); \
+    _AK_DEBUG_END()
+
+// Takes and returns a PyArrayObject, optionally copying a mutable array and setting it as immutable
+static inline PyArrayObject *
+AK_immutable_filter(PyArrayObject *a)
+{
+    // https://numpy.org/devdocs/reference/c-api/array.html#array-flags
+    if (PyArray_FLAGS(a) & NPY_ARRAY_WRITEABLE) {
+        if ((a = (PyArrayObject *)PyArray_NewCopy(a, NPY_ANYORDER))) {
+            PyArray_CLEARFLAGS(a, NPY_ARRAY_WRITEABLE);
+        }
+        return a;
+    }
+    Py_INCREF(a);
+    return a;
+}
+
+// Returns NULL on error.
+static inline PyArray_Descr *
+AK_resolve_dtype(PyArray_Descr *d1, PyArray_Descr *d2)
+{
+    if (PyArray_EquivTypes(d1, d2)) {
+        Py_INCREF(d1);
+        return d1;
+    }
+    if (PyDataType_ISOBJECT(d1) || PyDataType_ISOBJECT(d2)
+        || PyDataType_ISBOOL(d1) || PyDataType_ISBOOL(d2)
+        || (PyDataType_ISSTRING(d1) != PyDataType_ISSTRING(d2))
+        || ((PyDataType_ISDATETIME(d1) || PyDataType_ISDATETIME(d2))
+            // PyDataType_ISDATETIME matches NPY_DATETIME or NPY_TIMEDELTA, so
+            // we need to make sure we didn't get one of each:
+            && !PyArray_EquivTypenums(d1->type_num, d2->type_num)))
+    {
+        return PyArray_DescrFromType(NPY_OBJECT);
+    }
+    PyArray_Descr *d = PyArray_PromoteTypes(d1, d2);
+    if (!d) {
+        PyErr_Clear();
+        return PyArray_DescrFromType(NPY_OBJECT);
+    }
+    return d;
+}
+
+// Returns NULL on error. Returns a new reference.
+static inline PyObject *
+AK_build_pair_ssize_t(Py_ssize_t a, Py_ssize_t b)
+{
+    PyObject* t = PyTuple_New(2);
+    if (t == NULL) {
+        return NULL;
+    }
+    PyObject* py_a = PyLong_FromSsize_t(a);
+    if (py_a == NULL) {
+        Py_DECREF(t);
+        return NULL;
+    }
+    PyObject* py_b = PyLong_FromSsize_t(b);
+    if (py_b == NULL) {
+        Py_DECREF(t);
+        Py_DECREF(py_a);
+        return NULL;
+    }
+    // steals refs
+    PyTuple_SET_ITEM(t, 0, py_a);
+    PyTuple_SET_ITEM(t, 1, py_b);
+    return t;
+}
+
+// Returns a new ref; returns NULL on error. Any start or stop less than 0 will be set to NULL.
+static inline PyObject *
+AK_build_slice(Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step)
+{
+    PyObject* py_start = NULL;
+    PyObject* py_stop = NULL;
+    PyObject* py_step = NULL;
+
+    if (start >= 0) {
+        py_start = PyLong_FromSsize_t(start);
+        if (py_start == NULL) {return NULL;}
+    }
+    if (stop >= 0) {
+        py_stop = PyLong_FromSsize_t(stop);
+        if (py_stop == NULL) {return NULL;}
+    }
+    // do not set a step if not necessary
+    if (step != 0 && step != 1) {
+        py_step = PyLong_FromSsize_t(step);
+        if (py_step == NULL) {return NULL;}
+    }
+
+    // might be NULL, let return
+    PyObject* new = PySlice_New(py_start, py_stop, py_step);
+
+    Py_XDECREF(py_start);
+    Py_XDECREF(py_stop);
+    Py_XDECREF(py_step);
+
+    return new;
+}
+
+// Utility function for converting slices; returns NULL on error; returns a new reference.
+static inline PyObject *
+AK_slice_to_ascending_slice(PyObject* slice, Py_ssize_t size)
+{
+    Py_ssize_t step_count = -1;
+    Py_ssize_t start = 0;
+    Py_ssize_t stop = 0;
+    Py_ssize_t step = 0;
+
+    if (PySlice_Unpack(slice, &start, &stop, &step)) {
+        return NULL;
+    }
+    if (step > 0) {
+        Py_INCREF(slice);
+        return slice;
+    }
+    step_count = PySlice_AdjustIndices(
+            size,
+            &start,
+            &stop,
+            step);
+
+    // step will be negative; shift original start value down to find new start
+    return AK_build_slice(
+            start + (step * (step_count - 1)),
+            start + 1,
+            -step);
+}
+
+// Given a Boolean, contiguous 1D array, return the index positions in an int64 array. Through experimentation it has been verified that doing full-size allocation of memory provides the best performance at all scales. Using NpyIter, or using, bit masks does not improve performance over pointer arithmetic. Prescanning for all empty is very effective. Note that NumPy benefits from first counting the nonzeros, then allocating only enough data for the expexted number of indices.
+static inline PyObject *
+AK_nonzero_1d(PyArrayObject* array) {
+    PyObject* final;
+    npy_intp count_max = PyArray_SIZE(array);
+
+    if (count_max == 0) { // return empty array
+        npy_intp dims = {count_max};
+        final = PyArray_SimpleNew(1, &dims, NPY_INT64);
+        PyArray_CLEARFLAGS((PyArrayObject*)final, NPY_ARRAY_WRITEABLE);
+        return final;
+    }
+    lldiv_t size_div = lldiv((long long)count_max, 8); // quot, rem
+
+    Py_ssize_t count = 0;
+    // the maximum number of collected integers is equal to or less than count_max
+    Py_ssize_t capacity = count_max;
+    npy_int64* indices = (npy_int64*)malloc(sizeof(npy_int64) * capacity);
+
+    NPY_BEGIN_THREADS_DEF;
+    NPY_BEGIN_THREADS;
+
+    if (PyArray_IS_C_CONTIGUOUS(array)) {
+        npy_bool* p_start = (npy_bool*)PyArray_DATA(array);
+        npy_bool* p = p_start;
+        npy_bool* p_end = p + count_max;
+        npy_bool* p_end_roll = p_end - size_div.rem;
+
+        while (p < p_end_roll) {
+            if (*(npy_uint64*)p == 0) {
+                p += 8; // no true within this 8 byte roll region
+                continue;
+            }
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+        }
+        while (p < p_end) {
+            if (*p) {indices[count++] = p - p_start;}
+            p++;
+        }
+    }
+    else {
+        npy_intp i = 0; // position within Boolean array
+        npy_intp i_end = count_max;
+        npy_intp i_end_roll = count_max - size_div.rem;
+        while (i < i_end_roll) {
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+        }
+        while (i < i_end) {
+            if (*(npy_bool*)PyArray_GETPTR1(array, i)) {indices[count++] = i;}
+            i++;
+        }
+    }
+    NPY_END_THREADS;
+
+    npy_intp dims = {count};
+    final = PyArray_SimpleNewFromData(1, &dims, NPY_INT64, (void*)indices);
+    if (!final) {
+        free(indices);
+        return NULL;
+    }
+    // This ensures that the array frees the indices array; this has been tested by calling free(indices) and observing segfault
+    PyArray_ENABLEFLAGS((PyArrayObject*)final, NPY_ARRAY_OWNDATA);
+    PyArray_CLEARFLAGS((PyArrayObject*)final, NPY_ARRAY_WRITEABLE);
+    return final;
+}
+
+#endif  /* ARRAYKIT_SRC_UTILITIES_H_ */

arraykit-0.8.1/MANIFEST.in

@@ -1 +0,0 @@
-include arraykit/__init__.pyi arraykit/py.typed