tskit 1.0.0b3__tar.gz → 1.0.1__tar.gz

{tskit-1.0.0b3/tskit.egg-info → tskit-1.0.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: tskit
-Version: 1.0.0b3
+Version: 1.0.1
 Summary: The tree sequence toolkit.
 Author-email: Tskit Developers <admin@tskit.dev>
-License: MIT
+License-Expression: MIT
 Project-URL: Homepage, https://tskit.dev/tskit
 Project-URL: Documentation, https://tskit.dev/tskit/docs/stable
 Project-URL: Changelog, https://tskit.dev/tskit/docs/stable/changelogs.html
@@ -17,11 +17,11 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Other Environment
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: POSIX
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: Microsoft :: Windows
@@ -39,9 +39,9 @@ Requires-Dist: dendropy==5.0.1; extra == "test"
 Requires-Dist: kastore==0.3.3; extra == "test"
 Requires-Dist: lshmm==0.0.8; extra == "test"
 Requires-Dist: msgpack==1.1.0; extra == "test"
-Requires-Dist: msprime==1.3.4; extra == "test"
+Requires-Dist: msprime==1.4.0b2; extra == "test"
 Requires-Dist: networkx==3.2.1; extra == "test"
-Requires-Dist: numba==0.61.2; extra == "test"
+Requires-Dist: numba==0.63.1; extra == "test"
 Requires-Dist: portion==2.6.0; extra == "test"
 Requires-Dist: pytest==8.3.5; extra == "test"
 Requires-Dist: pytest-cov==6.0.0; extra == "test"
@@ -57,8 +57,8 @@ Requires-Dist: breathe==4.35.0; extra == "docs"
 Requires-Dist: sphinx-autodoc-typehints==2.3.0; extra == "docs"
 Requires-Dist: sphinx-issues==5.0.0; extra == "docs"
 Requires-Dist: sphinx-argparse==0.5.2; extra == "docs"
-Requires-Dist: msprime==1.3.3; extra == "docs"
-Requires-Dist: numba==0.61.2; extra == "docs"
+Requires-Dist: msprime==1.4.0b2; extra == "docs"
+Requires-Dist: numba==0.63.1; extra == "docs"
 Requires-Dist: sphinx-book-theme; extra == "docs"
 Requires-Dist: pandas==2.2.3; extra == "docs"
 Provides-Extra: dev
@@ -84,7 +84,7 @@ Requires-Dist: tszip; extra == "dev"
 Requires-Dist: xmlunittest; extra == "dev"
 Requires-Dist: newick; extra == "dev"
 Requires-Dist: zarr<3; extra == "dev"
-Requires-Dist: jupyter-book; extra == "dev"
+Requires-Dist: jupyter-book<2; extra == "dev"
 Requires-Dist: breathe; extra == "dev"
 Requires-Dist: sphinx-autodoc-typehints; extra == "dev"
 Requires-Dist: sphinx-issues; extra == "dev"

{tskit-1.0.0b3 → tskit-1.0.1}/_tskitmodule.c

@@ -4347,15 +4347,18 @@ TableCollection_union(TableCollection *self, PyObject *args, PyObject *kwds)
     npy_intp *shape;
     tsk_flags_t options = 0;
     int check_shared = true;
+    int all_edges = false;
+    int all_mutations = false;
     int add_populations = true;
     static char *kwlist[] = { "other", "other_node_mapping", "check_shared_equality",
-        "add_populations", NULL };
+        "add_populations", "all_edges", "all_mutations", NULL };
 
     if (TableCollection_check_state(self) != 0) {
         goto out;
     }
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O|ii", kwlist, &TableCollectionType,
-            &other, &other_node_mapping, &check_shared, &add_populations)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O|iiii", kwlist,
+            &TableCollectionType, &other, &other_node_mapping, &check_shared,
+            &add_populations, &all_edges, &all_mutations)) {
         goto out;
     }
     nmap_array = (PyArrayObject *) PyArray_FROMANY(
@@ -4370,6 +4373,12 @@ TableCollection_union(TableCollection *self, PyObject *args, PyObject *kwds)
             " number of nodes in the other tree sequence.");
         goto out;
     }
+    if (all_edges) {
+        options |= TSK_UNION_ALL_EDGES;
+    }
+    if (all_mutations) {
+        options |= TSK_UNION_ALL_MUTATIONS;
+    }
     if (!check_shared) {
         options |= TSK_UNION_NO_CHECK_SHARED;
     }
@@ -5335,6 +5344,69 @@ out:
     return ret;
 }
 
+static PyObject *
+TreeSequence_link_ancestors(TreeSequence *self, PyObject *args, PyObject *kwds)
+{
+    int err;
+    PyObject *ret = NULL;
+    PyObject *samples = NULL;
+    PyObject *ancestors = NULL;
+    PyArrayObject *samples_array = NULL;
+    PyArrayObject *ancestors_array = NULL;
+    npy_intp *shape;
+    tsk_size_t num_samples, num_ancestors;
+    EdgeTable *result = NULL;
+    PyObject *result_args = NULL;
+    static char *kwlist[] = { "samples", "ancestors", NULL };
+
+    if (TreeSequence_check_state(self) != 0) {
+        goto out;
+    }
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &samples, &ancestors)) {
+        goto out;
+    }
+
+    samples_array = (PyArrayObject *) PyArray_FROMANY(
+        samples, NPY_INT32, 1, 1, NPY_ARRAY_IN_ARRAY);
+    if (samples_array == NULL) {
+        goto out;
+    }
+    shape = PyArray_DIMS(samples_array);
+    num_samples = (tsk_size_t) shape[0];
+
+    ancestors_array = (PyArrayObject *) PyArray_FROMANY(
+        ancestors, NPY_INT32, 1, 1, NPY_ARRAY_IN_ARRAY);
+    if (ancestors_array == NULL) {
+        goto out;
+    }
+    shape = PyArray_DIMS(ancestors_array);
+    num_ancestors = (tsk_size_t) shape[0];
+
+    result_args = PyTuple_New(0);
+    if (result_args == NULL) {
+        goto out;
+    }
+    result = (EdgeTable *) PyObject_CallObject((PyObject *) &EdgeTableType, result_args);
+    if (result == NULL) {
+        goto out;
+    }
+    err = tsk_table_collection_link_ancestors(self->tree_sequence->tables,
+        PyArray_DATA(samples_array), num_samples, PyArray_DATA(ancestors_array),
+        num_ancestors, 0, result->table);
+    if (err != 0) {
+        handle_library_error(err);
+        goto out;
+    }
+    ret = (PyObject *) result;
+    result = NULL;
+out:
+    Py_XDECREF(samples_array);
+    Py_XDECREF(ancestors_array);
+    Py_XDECREF(result);
+    Py_XDECREF(result_args);
+    return ret;
+}
+
 static PyObject *
 TreeSequence_load(TreeSequence *self, PyObject *args, PyObject *kwds)
 {
@@ -6070,6 +6142,102 @@ out:
     return ret;
 }
 
+static PyObject *
+TreeSequence_decode_alignments(TreeSequence *self, PyObject *args, PyObject *kwds)
+{
+    int err;
+    PyObject *ret = NULL;
+    PyObject *py_ref, *py_nodes, *py_missing;
+    PyArrayObject *nodes_array = NULL;
+    const char *ref_seq;
+    Py_ssize_t ref_len, missing_len;
+    tsk_id_t *nodes;
+    tsk_size_t num_nodes;
+    double left, right;
+    char missing_char;
+    const char *missing_utf8;
+    int isolated_as_missing = 1;
+    tsk_flags_t options = 0;
+    PyObject *buf_obj = NULL;
+    char *buf = NULL;
+
+    static char *kwlist[] = { "reference_sequence", "nodes", "left", "right",
+        "missing_data_character", "isolated_as_missing", NULL };
+
+    if (TreeSequence_check_state(self) != 0) {
+        goto out;
+    }
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOddOp", kwlist, &py_ref, &py_nodes,
+            &left, &right, &py_missing, &isolated_as_missing)) {
+        goto out;
+    }
+
+    if (!PyBytes_Check(py_ref)) {
+        PyErr_SetString(PyExc_TypeError, "reference_sequence must be bytes");
+        goto out;
+    }
+    if (PyBytes_AsStringAndSize(py_ref, (char **) &ref_seq, &ref_len) < 0) {
+        goto out;
+    }
+
+    if (!PyUnicode_Check(py_missing)) {
+        PyErr_SetString(
+            PyExc_TypeError, "missing_data_character must be a (length 1) string");
+        goto out;
+    }
+    missing_utf8 = PyUnicode_AsUTF8AndSize(py_missing, &missing_len);
+    if (missing_utf8 == NULL) {
+        goto out;
+    }
+    if (missing_len != 1) {
+        PyErr_SetString(
+            PyExc_TypeError, "missing_data_character must be a single character");
+        goto out;
+    }
+    missing_char = missing_utf8[0];
+
+    if (!isolated_as_missing) {
+        options |= TSK_ISOLATED_NOT_MISSING;
+    }
+
+    nodes_array = (PyArrayObject *) PyArray_FROMANY(
+        py_nodes, NPY_INT32, 1, 1, NPY_ARRAY_IN_ARRAY);
+    if (nodes_array == NULL) {
+        goto out;
+    }
+    num_nodes = (tsk_size_t) PyArray_DIM(nodes_array, 0);
+    nodes = PyArray_DATA(nodes_array);
+
+    buf_obj = PyBytes_FromStringAndSize(
+        NULL, (Py_ssize_t)(num_nodes * (tsk_size_t)(right - left)));
+    if (buf_obj == NULL) {
+        goto out;
+    }
+    buf = PyBytes_AS_STRING(buf_obj);
+
+    // clang-format off
+    Py_BEGIN_ALLOW_THREADS 
+    err = tsk_treeseq_decode_alignments(self->tree_sequence,
+        ref_seq, (tsk_size_t) ref_len, nodes, num_nodes, left, right, missing_char, buf,
+        options);
+    Py_END_ALLOW_THREADS
+        // clang-format on
+        if (err != 0)
+    {
+        handle_library_error(err);
+        goto out;
+    }
+
+    ret = buf_obj;
+    buf_obj = NULL;
+
+out:
+    Py_XDECREF(nodes_array);
+    Py_XDECREF(buf_obj);
+    return ret;
+}
+
 static PyObject *
 TreeSequence_get_mutations_edge(TreeSequence *self)
 {
@@ -8519,6 +8687,10 @@ static PyMethodDef TreeSequence_methods[] = {
         .ml_meth = (PyCFunction) TreeSequence_dump_tables,
         .ml_flags = METH_VARARGS | METH_KEYWORDS,
         .ml_doc = "Dumps the tree sequence to the specified set of tables" },
+    { .ml_name = "link_ancestors",
+        .ml_meth = (PyCFunction) TreeSequence_link_ancestors,
+        .ml_flags = METH_VARARGS | METH_KEYWORDS,
+        .ml_doc = "Returns an EdgeTable linking the specified samples and ancestors." },
     { .ml_name = "get_node",
         .ml_meth = (PyCFunction) TreeSequence_get_node,
         .ml_flags = METH_VARARGS,
@@ -8651,6 +8823,10 @@ static PyMethodDef TreeSequence_methods[] = {
         .ml_meth = (PyCFunction) TreeSequence_get_individuals_nodes,
         .ml_flags = METH_NOARGS,
         .ml_doc = "Returns an array of the node ids for each individual" },
+    { .ml_name = "decode_alignments",
+        .ml_meth = (PyCFunction) TreeSequence_decode_alignments,
+        .ml_flags = METH_VARARGS | METH_KEYWORDS,
+        .ml_doc = "Decode full alignments for given nodes and interval." },
     { .ml_name = "get_mutations_edge",
         .ml_meth = (PyCFunction) TreeSequence_get_mutations_edge,
         .ml_flags = METH_NOARGS,

{tskit-1.0.0b3 → tskit-1.0.1}/lib/tskit/core.c

@@ -584,6 +584,14 @@ tsk_strerror_internal(int err)
             ret = "Must have at least one allele when specifying an allele map. "
                   "(TSK_ERR_ZERO_ALLELES)";
             break;
+        case TSK_ERR_BAD_ALLELE_LENGTH:
+            ret = "Alleles used when decoding alignments must have length one. "
+                  "(TSK_ERR_BAD_ALLELE_LENGTH)";
+            break;
+        case TSK_ERR_MISSING_CHAR_COLLISION:
+            ret = "Alleles used when decoding alignments must not match the missing "
+                  "data character. (TSK_ERR_MISSING_CHAR_COLLISION)";
+            break;
 
         /* Distance metric errors */
         case TSK_ERR_SAMPLE_SIZE_MISMATCH:
@@ -1033,7 +1041,7 @@ FILE *
 tsk_get_debug_stream(void)
 {
     if (_tsk_debug_stream == NULL) {
-        _tsk_debug_stream = stdout;
+        _tsk_debug_stream = TSK_DEFAULT_DEBUG_STREAM;
     }
     return _tsk_debug_stream;
 }
@@ -1260,16 +1268,16 @@ tsk_avl_tree_int_ordered_nodes(const tsk_avl_tree_int_t *self, tsk_avl_node_int_
 }
 
 // Bit Array implementation. Allows us to store unsigned integers in a compact manner.
-// Currently implemented as an array of 32-bit unsigned integers for ease of counting.
+// Currently implemented as an array of 32-bit unsigned integers.
 
 int
-tsk_bit_array_init(tsk_bit_array_t *self, tsk_size_t num_bits, tsk_size_t length)
+tsk_bitset_init(tsk_bitset_t *self, tsk_size_t num_bits, tsk_size_t length)
 {
     int ret = 0;
 
-    self->size = (num_bits >> TSK_BIT_ARRAY_CHUNK)
-                 + (num_bits % TSK_BIT_ARRAY_NUM_BITS ? 1 : 0);
-    self->data = tsk_calloc(self->size * length, sizeof(*self->data));
+    self->row_len = (num_bits / TSK_BITSET_BITS) + (num_bits % TSK_BITSET_BITS ? 1 : 0);
+    self->len = length;
+    self->data = tsk_calloc(self->row_len * length, sizeof(*self->data));
     if (self->data == NULL) {
         ret = tsk_trace_error(TSK_ERR_NO_MEMORY);
         goto out;
@@ -1278,96 +1286,111 @@ out:
     return ret;
 }
 
-void
-tsk_bit_array_get_row(const tsk_bit_array_t *self, tsk_size_t row, tsk_bit_array_t *out)
-{
-    out->size = self->size;
-    out->data = self->data + (row * self->size);
-}
+#define BITSET_DATA_ROW(bs, row) ((bs)->data + (row) * (bs)->row_len)
 
 void
-tsk_bit_array_intersect(
-    const tsk_bit_array_t *self, const tsk_bit_array_t *other, tsk_bit_array_t *out)
+tsk_bitset_intersect(const tsk_bitset_t *self, tsk_size_t self_row,
+    const tsk_bitset_t *other, tsk_size_t other_row, tsk_bitset_t *out)
 {
-    for (tsk_size_t i = 0; i < self->size; i++) {
-        out->data[i] = self->data[i] & other->data[i];
+    const tsk_bitset_val_t *restrict self_d = BITSET_DATA_ROW(self, self_row);
+    const tsk_bitset_val_t *restrict other_d = BITSET_DATA_ROW(other, other_row);
+    tsk_bitset_val_t *restrict out_d = out->data;
+    for (tsk_size_t i = 0; i < self->row_len; i++) {
+        out_d[i] = self_d[i] & other_d[i];
     }
 }
 
 void
-tsk_bit_array_subtract(tsk_bit_array_t *self, const tsk_bit_array_t *other)
+tsk_bitset_subtract(tsk_bitset_t *self, tsk_size_t self_row, const tsk_bitset_t *other,
+    tsk_size_t other_row)
 {
-    for (tsk_size_t i = 0; i < self->size; i++) {
-        self->data[i] &= ~(other->data[i]);
+    tsk_bitset_val_t *restrict self_d = BITSET_DATA_ROW(self, self_row);
+    const tsk_bitset_val_t *restrict other_d = BITSET_DATA_ROW(other, other_row);
+    for (tsk_size_t i = 0; i < self->row_len; i++) {
+        self_d[i] &= ~(other_d[i]);
     }
 }
 
 void
-tsk_bit_array_add(tsk_bit_array_t *self, const tsk_bit_array_t *other)
+tsk_bitset_union(tsk_bitset_t *self, tsk_size_t self_row, const tsk_bitset_t *other,
+    tsk_size_t other_row)
 {
-    for (tsk_size_t i = 0; i < self->size; i++) {
-        self->data[i] |= other->data[i];
+    tsk_bitset_val_t *restrict self_d = BITSET_DATA_ROW(self, self_row);
+    const tsk_bitset_val_t *restrict other_d = BITSET_DATA_ROW(other, other_row);
+    for (tsk_size_t i = 0; i < self->row_len; i++) {
+        self_d[i] |= other_d[i];
     }
 }
 
 void
-tsk_bit_array_add_bit(tsk_bit_array_t *self, const tsk_bit_array_value_t bit)
+tsk_bitset_set_bit(tsk_bitset_t *self, tsk_size_t row, const tsk_bitset_val_t bit)
 {
-    tsk_bit_array_value_t i = bit >> TSK_BIT_ARRAY_CHUNK;
-    self->data[i] |= (tsk_bit_array_value_t) 1 << (bit - (TSK_BIT_ARRAY_NUM_BITS * i));
+    tsk_bitset_val_t i = (bit / TSK_BITSET_BITS);
+    *(BITSET_DATA_ROW(self, row) + i) |= (tsk_bitset_val_t) 1
+                                         << (bit - (TSK_BITSET_BITS * i));
 }
 
 bool
-tsk_bit_array_contains(const tsk_bit_array_t *self, const tsk_bit_array_value_t bit)
+tsk_bitset_contains(const tsk_bitset_t *self, tsk_size_t row, const tsk_bitset_val_t bit)
 {
-    tsk_bit_array_value_t i = bit >> TSK_BIT_ARRAY_CHUNK;
-    return self->data[i]
-           & ((tsk_bit_array_value_t) 1 << (bit - (TSK_BIT_ARRAY_NUM_BITS * i)));
+    tsk_bitset_val_t i = (bit / TSK_BITSET_BITS);
+    return *(BITSET_DATA_ROW(self, row) + i)
+           & ((tsk_bitset_val_t) 1 << (bit - (TSK_BITSET_BITS * i)));
 }
 
-tsk_size_t
-tsk_bit_array_count(const tsk_bit_array_t *self)
+static inline uint32_t
+popcount(tsk_bitset_val_t v)
 {
-    // Utilizes 12 operations per bit array. NB this only works on 32 bit integers.
+    // Utilizes 12 operations per chunk. NB this only works on 32 bit integers.
     // Taken from:
     //   https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
     // There's a nice breakdown of this algorithm here:
     //   https://stackoverflow.com/a/109025
-    // Could probably do better with explicit SIMD (instead of SWAR), but not as
-    // portable: https://arxiv.org/pdf/1611.07612.pdf
     //
-    // There is one solution to explore further, which uses __builtin_popcountll.
-    // This option is relatively simple, but requires a 64 bit bit array and also
-    // involves some compiler flag plumbing (-mpopcnt)
+    // The gcc/clang compiler flag will -mpopcnt will convert this code to a
+    // popcnt instruction (most if not all modern CPUs will support this). The
+    // popcnt instruction will yield some speed improvements, which depend on
+    // the tree sequence.
+    //
+    // NB: 32bit counting is typically faster than 64bit counting for this task.
+    //     (at least on x86-64)
+
+    v = v - ((v >> 1) & 0x55555555);
+    v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
+    return (((v + (v >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24;
+}
 
-    tsk_bit_array_value_t tmp;
-    tsk_size_t i, count = 0;
+tsk_size_t
+tsk_bitset_count(const tsk_bitset_t *self, tsk_size_t row)
+{
+    tsk_size_t i = 0;
+    tsk_size_t count = 0;
+    const tsk_bitset_val_t *restrict self_d = BITSET_DATA_ROW(self, row);
 
-    for (i = 0; i < self->size; i++) {
-        tmp = self->data[i] - ((self->data[i] >> 1) & 0x55555555);
-        tmp = (tmp & 0x33333333) + ((tmp >> 2) & 0x33333333);
-        count += (((tmp + (tmp >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24;
+    for (i = 0; i < self->row_len; i++) {
+        count += popcount(self_d[i]);
     }
     return count;
 }
 
 void
-tsk_bit_array_get_items(
-    const tsk_bit_array_t *self, tsk_id_t *items, tsk_size_t *n_items)
+tsk_bitset_get_items(
+    const tsk_bitset_t *self, tsk_size_t row, tsk_id_t *items, tsk_size_t *n_items)
 {
     // Get the items stored in the row of a bitset.
-    // Uses a de Bruijn sequence lookup table to determine the lowest bit set. See the
-    // wikipedia article for more info: https://w.wiki/BYiF
+    // Uses a de Bruijn sequence lookup table to determine the lowest bit set.
+    // See the wikipedia article for more info: https://w.wiki/BYiF
 
     tsk_size_t i, n, off;
-    tsk_bit_array_value_t v, lsb; // least significant bit
+    tsk_bitset_val_t v, lsb; // least significant bit
     static const tsk_id_t lookup[32] = { 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25,
         17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 };
+    const tsk_bitset_val_t *restrict self_d = BITSET_DATA_ROW(self, row);
 
     n = 0;
-    for (i = 0; i < self->size; i++) {
-        v = self->data[i];
-        off = i * ((tsk_size_t) TSK_BIT_ARRAY_NUM_BITS);
+    for (i = 0; i < self->row_len; i++) {
+        v = self_d[i];
+        off = i * TSK_BITSET_BITS;
         if (v == 0) {
             continue;
         }
@@ -1381,7 +1404,7 @@ tsk_bit_array_get_items(
 }
 
 void
-tsk_bit_array_free(tsk_bit_array_t *self)
+tsk_bitset_free(tsk_bitset_t *self)
 {
     tsk_safe_free(self->data);
 }

{tskit-1.0.0b3 → tskit-1.0.1}/lib/tskit/core.h

@@ -147,7 +147,7 @@ sizes and types of externally visible structs.
 The library minor version. Incremented when non-breaking backward-compatible changes
 to the API or ABI are introduced, i.e., the addition of a new function.
 */
-#define TSK_VERSION_MINOR   2
+#define TSK_VERSION_MINOR   3
 /**
 The library patch version. Incremented when any changes not relevant to the
 to the API or ABI are introduced, i.e., internal refactors of bugfixes.
@@ -511,7 +511,7 @@ disallowed (use compute_mutation_times?).
 */
 #define TSK_ERR_DISALLOWED_UNKNOWN_MUTATION_TIME                    -510
 
-/** 
+/**
 A mutation's parent was not consistent with the topology of the tree.
  */
 #define TSK_ERR_BAD_MUTATION_PARENT                                 -511
@@ -803,6 +803,14 @@ More than 2147483647 alleles were specified.
 A user-specified allele map was used, but it contained zero alleles.
 */
 #define TSK_ERR_ZERO_ALLELES                                       -1103
+/**
+An allele used when decoding alignments had length other than one.
+*/
+#define TSK_ERR_BAD_ALLELE_LENGTH                                  -1104
+/**
+An allele used when decoding alignments matched the missing data character.
+*/
+#define TSK_ERR_MISSING_CHAR_COLLISION                             -1105
 /** @} */
 
 /**
@@ -963,6 +971,12 @@ not be freed by client code.
 */
 const char *tsk_strerror(int err);
 
+/* Redefine this macro in downstream builds if stdout is not the
+ * approriate stream to emit debug information when the TSK_DEBUG
+ * flag is passed to supporting functions (e.g. in R).
+ */
+#define TSK_DEFAULT_DEBUG_STREAM stdout
+
 #ifdef TSK_TRACE_ERRORS
 
 static inline int
@@ -973,6 +987,11 @@ _tsk_trace_error(int err, int line, const char *file)
     return err;
 }
 
+/*
+Developer note: this macro may be redefined as part of compilation for
+an R package, and should be treated as part of the documented API
+(no changes).
+*/
 #define tsk_trace_error(err) (_tsk_trace_error(err, __LINE__, __FILE__))
 #else
 #define tsk_trace_error(err) (err)
@@ -993,6 +1012,11 @@ means compiling without NDEBUG. This macro still asserts when NDEBUG is defined.
 If you are using this macro in your own software then please set TSK_BUG_ASSERT_MESSAGE
 to point users to your issue tracker.
 */
+/*
+Developer note: this macro may redefined as part of compilation for
+an R package, and should be treated as part of the documented API
+(no changes).
+*/
 #define tsk_bug_assert(condition)                                                       \
     do {                                                                                \
         if (!(condition)) {                                                             \
@@ -1104,29 +1128,31 @@ FILE *tsk_get_debug_stream(void);
 
 /* Bit Array functionality */
 
-typedef uint32_t tsk_bit_array_value_t;
+// define a 32-bit chunk size for our bitsets.
+// this means we'll be able to hold 32 distinct items in each 32 bit uint
+#define TSK_BITSET_BITS ((tsk_size_t) 32)
+typedef uint32_t tsk_bitset_val_t;
+
 typedef struct {
-    tsk_size_t size;             // Number of chunks per row
-    tsk_bit_array_value_t *data; // Array data
-} tsk_bit_array_t;
-
-#define TSK_BIT_ARRAY_CHUNK 5U
-#define TSK_BIT_ARRAY_NUM_BITS (1U << TSK_BIT_ARRAY_CHUNK)
-
-int tsk_bit_array_init(tsk_bit_array_t *self, tsk_size_t num_bits, tsk_size_t length);
-void tsk_bit_array_free(tsk_bit_array_t *self);
-void tsk_bit_array_get_row(
-    const tsk_bit_array_t *self, tsk_size_t row, tsk_bit_array_t *out);
-void tsk_bit_array_intersect(
-    const tsk_bit_array_t *self, const tsk_bit_array_t *other, tsk_bit_array_t *out);
-void tsk_bit_array_subtract(tsk_bit_array_t *self, const tsk_bit_array_t *other);
-void tsk_bit_array_add(tsk_bit_array_t *self, const tsk_bit_array_t *other);
-void tsk_bit_array_add_bit(tsk_bit_array_t *self, const tsk_bit_array_value_t bit);
-bool tsk_bit_array_contains(
-    const tsk_bit_array_t *self, const tsk_bit_array_value_t bit);
-tsk_size_t tsk_bit_array_count(const tsk_bit_array_t *self);
-void tsk_bit_array_get_items(
-    const tsk_bit_array_t *self, tsk_id_t *items, tsk_size_t *n_items);
+    tsk_size_t row_len; // Number of size TSK_BITSET_BITS chunks per row
+    tsk_size_t len;     // Number of rows
+    tsk_bitset_val_t *data;
+} tsk_bitset_t;
+
+int tsk_bitset_init(tsk_bitset_t *self, tsk_size_t num_bits, tsk_size_t length);
+void tsk_bitset_free(tsk_bitset_t *self);
+void tsk_bitset_intersect(const tsk_bitset_t *self, tsk_size_t self_row,
+    const tsk_bitset_t *other, tsk_size_t other_row, tsk_bitset_t *out);
+void tsk_bitset_subtract(tsk_bitset_t *self, tsk_size_t self_row,
+    const tsk_bitset_t *other, tsk_size_t other_row);
+void tsk_bitset_union(tsk_bitset_t *self, tsk_size_t self_row, const tsk_bitset_t *other,
+    tsk_size_t other_row);
+void tsk_bitset_set_bit(tsk_bitset_t *self, tsk_size_t row, const tsk_bitset_val_t bit);
+bool tsk_bitset_contains(
+    const tsk_bitset_t *self, tsk_size_t row, const tsk_bitset_val_t bit);
+tsk_size_t tsk_bitset_count(const tsk_bitset_t *self, tsk_size_t row);
+void tsk_bitset_get_items(
+    const tsk_bitset_t *self, tsk_size_t row, tsk_id_t *items, tsk_size_t *n_items);
 
 #ifdef __cplusplus
 }