copairs 0.4.1__tar.gz

copairs-0.4.1/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Broad Institute
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

copairs-0.4.1/PKG-INFO

@@ -0,0 +1,139 @@
+Metadata-Version: 2.1
+Name: copairs
+Version: 0.4.1
+Summary: Find pairs and compute metrics between them
+License: LICENSE.txt
+Keywords: pairwise,replication
+Author: John Arevalo
+Author-email: johnarevalo@gmail.com
+Requires-Python: >=3.9,<3.12
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: pandas (>=2.0.3)
+Requires-Dist: statsmodels (>=0.14.2,<0.15.0)
+Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Description-Content-Type: text/markdown
+
+ # copairs
+
+Find pairs and compute metrics between them.
+
+## Installation
+
+```bash
+pip install git+https://github.com/cytomining/copairs.git@v0.4.1
+```
+
+## Usage
+
+### Data
+
+Say you have a dataset with 20 samples taken in 3 plates `p1, p2, p3`,
+each plate is composed of 5 wells `w1, w2, w3, w4, w5`, and each well 
+has one or more labels (`t1, t2, t3, t4`) assigned.
+
+```python
+import pandas as pd
+import random
+
+random.seed(0)
+n_samples = 20
+dframe = pd.DataFrame({
+    'plate': [random.choice(['p1', 'p2', 'p3']) for _ in range(n_samples)],
+    'well': [random.choice(['w1', 'w2', 'w3', 'w4', 'w5']) for _ in range(n_samples)],
+    'label': [random.choice(['t1', 't2', 't3', 't4']) for _ in range(n_samples)]
+})
+dframe = dframe.drop_duplicates()
+dframe = dframe.sort_values(by=['plate', 'well', 'label'])
+dframe = dframe.reset_index(drop=True)
+```
+
+|    | plate   | well   | label   |
+|---:|:--------|:-------|:--------|
+|  0 | p1      | w2     | t4      |
+|  1 | p1      | w3     | t2      |
+|  2 | p1      | w3     | t4      |
+|  3 | p1      | w4     | t1      |
+|  4 | p1      | w4     | t3      |
+|  5 | p2      | w1     | t1      |
+|  6 | p2      | w2     | t1      |
+|  7 | p2      | w3     | t1      |
+|  8 | p2      | w3     | t2      |
+|  9 | p2      | w3     | t3      |
+| 10 | p2      | w4     | t2      |
+| 11 | p2      | w5     | t1      |
+| 12 | p2      | w5     | t3      |
+| 13 | p3      | w1     | t3      |
+| 14 | p3      | w1     | t4      |
+| 15 | p3      | w4     | t2      |
+| 16 | p3      | w5     | t2      |
+| 17 | p3      | w5     | t4      |
+
+### Getting valid pairs
+
+To get pairs of samples that share the same `label` but comes from different
+`plate`s at different `well` positions: 
+
+```python
+from copairs import Matcher
+matcher = Matcher(dframe, ['plate', 'well', 'label'], seed=0)
+pairs_dict = matcher.get_all_pairs(sameby=['label'], diffby=['plate', 'well'])
+```
+
+`pairs_dict` is a `label_id: pairs` dictionary containing the list of valid
+pairs for every unique value of `labels`
+
+```
+{'t4': [(0, 17), (0, 14), (17, 2), (2, 14)],
+ 't2': [(1, 16), (1, 10), (1, 15), (8, 16), (8, 15), (10, 16)],
+ 't1': [(3, 11), (3, 5), (3, 6), (3, 7)],
+ 't3': [(9, 4), (9, 13), (13, 4), (13, 12), (4, 12)]}
+```
+
+### Getting valid pairs from a multilabel column
+
+For eficiency reasons, you may not want to have duplicated rows. You can
+group all the labels in a single row and use `MatcherMultilabel` to find the
+corresponding pairs:
+
+```python
+dframe_multi = dframe.groupby(['plate', 'well'])['label'].unique().reset_index()
+```
+
+|    | plate   | well   | label              |
+|---:|:--------|:-------|:-------------------|
+|  0 | p1      | w2     | ['t4']             |
+|  1 | p1      | w3     | ['t2', 't4']       |
+|  2 | p1      | w4     | ['t1', 't3']       |
+|  3 | p2      | w1     | ['t1']             |
+|  4 | p2      | w2     | ['t1']             |
+|  5 | p2      | w3     | ['t1', 't2', 't3'] |
+|  6 | p2      | w4     | ['t2']             |
+|  7 | p2      | w5     | ['t1', 't3']       |
+|  8 | p3      | w1     | ['t3', 't4']       |
+|  9 | p3      | w4     | ['t2']             |
+| 10 | p3      | w5     | ['t2', 't4']       |
+
+```python
+from copairs import MatcherMultilabel
+matcher_multi = MatcherMultilabel(dframe_multi,
+                                  columns=['plate', 'well', 'label'],
+                                  multilabel_col='label',
+                                  seed=0)
+pairs_multi = matcher_multi.get_all_pairs(sameby=['label'],
+                                          diffby=['plate', 'well'])
+```
+
+`pairs_multi` is also a `label_id: pairs` dictionary with the same
+structure discussed before:
+
+```
+{'t4': [(0, 10), (0, 8), (10, 1), (1, 8)],
+ 't2': [(1, 10), (1, 6), (1, 9), (5, 10), (5, 9), (6, 10)],
+ 't1': [(2, 7), (2, 3), (2, 4), (2, 5)],
+ 't3': [(5, 2), (5, 8), (8, 2), (8, 7), (2, 7)]}
+```
+

copairs-0.4.1/README.md

@@ -0,0 +1,119 @@
+ # copairs
+
+Find pairs and compute metrics between them.
+
+## Installation
+
+```bash
+pip install git+https://github.com/cytomining/copairs.git@v0.4.1
+```
+
+## Usage
+
+### Data
+
+Say you have a dataset with 20 samples taken in 3 plates `p1, p2, p3`,
+each plate is composed of 5 wells `w1, w2, w3, w4, w5`, and each well 
+has one or more labels (`t1, t2, t3, t4`) assigned.
+
+```python
+import pandas as pd
+import random
+
+random.seed(0)
+n_samples = 20
+dframe = pd.DataFrame({
+    'plate': [random.choice(['p1', 'p2', 'p3']) for _ in range(n_samples)],
+    'well': [random.choice(['w1', 'w2', 'w3', 'w4', 'w5']) for _ in range(n_samples)],
+    'label': [random.choice(['t1', 't2', 't3', 't4']) for _ in range(n_samples)]
+})
+dframe = dframe.drop_duplicates()
+dframe = dframe.sort_values(by=['plate', 'well', 'label'])
+dframe = dframe.reset_index(drop=True)
+```
+
+|    | plate   | well   | label   |
+|---:|:--------|:-------|:--------|
+|  0 | p1      | w2     | t4      |
+|  1 | p1      | w3     | t2      |
+|  2 | p1      | w3     | t4      |
+|  3 | p1      | w4     | t1      |
+|  4 | p1      | w4     | t3      |
+|  5 | p2      | w1     | t1      |
+|  6 | p2      | w2     | t1      |
+|  7 | p2      | w3     | t1      |
+|  8 | p2      | w3     | t2      |
+|  9 | p2      | w3     | t3      |
+| 10 | p2      | w4     | t2      |
+| 11 | p2      | w5     | t1      |
+| 12 | p2      | w5     | t3      |
+| 13 | p3      | w1     | t3      |
+| 14 | p3      | w1     | t4      |
+| 15 | p3      | w4     | t2      |
+| 16 | p3      | w5     | t2      |
+| 17 | p3      | w5     | t4      |
+
+### Getting valid pairs
+
+To get pairs of samples that share the same `label` but comes from different
+`plate`s at different `well` positions: 
+
+```python
+from copairs import Matcher
+matcher = Matcher(dframe, ['plate', 'well', 'label'], seed=0)
+pairs_dict = matcher.get_all_pairs(sameby=['label'], diffby=['plate', 'well'])
+```
+
+`pairs_dict` is a `label_id: pairs` dictionary containing the list of valid
+pairs for every unique value of `labels`
+
+```
+{'t4': [(0, 17), (0, 14), (17, 2), (2, 14)],
+ 't2': [(1, 16), (1, 10), (1, 15), (8, 16), (8, 15), (10, 16)],
+ 't1': [(3, 11), (3, 5), (3, 6), (3, 7)],
+ 't3': [(9, 4), (9, 13), (13, 4), (13, 12), (4, 12)]}
+```
+
+### Getting valid pairs from a multilabel column
+
+For eficiency reasons, you may not want to have duplicated rows. You can
+group all the labels in a single row and use `MatcherMultilabel` to find the
+corresponding pairs:
+
+```python
+dframe_multi = dframe.groupby(['plate', 'well'])['label'].unique().reset_index()
+```
+
+|    | plate   | well   | label              |
+|---:|:--------|:-------|:-------------------|
+|  0 | p1      | w2     | ['t4']             |
+|  1 | p1      | w3     | ['t2', 't4']       |
+|  2 | p1      | w4     | ['t1', 't3']       |
+|  3 | p2      | w1     | ['t1']             |
+|  4 | p2      | w2     | ['t1']             |
+|  5 | p2      | w3     | ['t1', 't2', 't3'] |
+|  6 | p2      | w4     | ['t2']             |
+|  7 | p2      | w5     | ['t1', 't3']       |
+|  8 | p3      | w1     | ['t3', 't4']       |
+|  9 | p3      | w4     | ['t2']             |
+| 10 | p3      | w5     | ['t2', 't4']       |
+
+```python
+from copairs import MatcherMultilabel
+matcher_multi = MatcherMultilabel(dframe_multi,
+                                  columns=['plate', 'well', 'label'],
+                                  multilabel_col='label',
+                                  seed=0)
+pairs_multi = matcher_multi.get_all_pairs(sameby=['label'],
+                                          diffby=['plate', 'well'])
+```
+
+`pairs_multi` is also a `label_id: pairs` dictionary with the same
+structure discussed before:
+
+```
+{'t4': [(0, 10), (0, 8), (10, 1), (1, 8)],
+ 't2': [(1, 10), (1, 6), (1, 9), (5, 10), (5, 9), (6, 10)],
+ 't1': [(2, 7), (2, 3), (2, 4), (2, 5)],
+ 't3': [(5, 2), (5, 8), (8, 2), (8, 7), (2, 7)]}
+```

copairs-0.4.1/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "copairs"
+version = "0.4.1"
+description = "Find pairs and compute metrics between them"
+readme = "README.md"
+requires-python = ">=3.9,<3.12"
+license = {file = "LICENSE.txt"}
+keywords = ["pairwise", "replication"]
+authors = [
+  {name = "John Arevalo", email = "johnarevalo@gmail.com" }
+]
+dependencies = [
+  "pandas",
+  "tqdm",
+  "statsmodels"
+]
+
+[project.optional-dependencies]
+plot = ["plotly"]
+
+[project.urls]
+"Homepage" = "https://github.com/cytomining/copairs"
+"Bug Reports" = "https://github.com/cytomining/copairs/issues"
+"Source" = "https://github.com/cytomining/copairs/"
+
+# [build-system]
+# requires = ["setuptools"]
+# build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"] 
+
+[tool.poetry]
+name = "copairs"
+version = "0.4.1"
+description = "Find pairs and compute metrics between them"
+readme = "README.md"
+license = "LICENSE.txt"
+keywords = ["pairwise", "replication"]
+authors = [
+  "John Arevalo <johnarevalo@gmail.com>",
+]
+packages = [{include = "copairs", from= "src"}]
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.12"
+tqdm = "^4.66.1"
+pandas = ">=2.0.3"
+statsmodels = "^0.14.2"
+
+[tool.poetry.group.dev.dependencies]
+jupytext = "^1.15.0"
+ipdb = "^0.13.13"
+jupyter = "^1.0.0"
+ruff = "<0.2.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+

copairs-0.4.1/src/copairs/__init__.py

@@ -0,0 +1,4 @@
+"""
+Package to create pairwise lists based on sameby and diffby criteria
+"""
+from .matching import Matcher, MatcherMultilabel

copairs-0.4.1/src/copairs/compute.py

@@ -0,0 +1,199 @@
+import itertools
+import os
+from multiprocessing.pool import ThreadPool
+from pathlib import Path
+from typing import Callable
+
+import numpy as np
+from tqdm.autonotebook import tqdm
+
+
+def parallel_map(par_func, items):
+    """Execute par_func(i) for every i in items using ThreadPool and tqdm."""
+    num_items = len(items)
+    pool_size = min(num_items, os.cpu_count())
+    chunksize = num_items // pool_size
+    with ThreadPool(pool_size) as pool:
+        tasks = pool.imap_unordered(par_func, items, chunksize=chunksize)
+        for _ in tqdm(tasks, total=len(items), leave=False):
+            pass
+
+
+def batch_processing(
+    pairwise_op: Callable[[np.ndarray, np.ndarray], np.ndarray],
+):
+    """Decorator adding the batch_size param to run the function with
+    multithreading using a list of paired indices"""
+
+    def batched_fn(feats: np.ndarray, pair_ix: np.ndarray, batch_size: int):
+        num_pairs = len(pair_ix)
+        result = np.empty(num_pairs, dtype=np.float32)
+
+        def par_func(i):
+            x_sample = feats[pair_ix[i : i + batch_size, 0]]
+            y_sample = feats[pair_ix[i : i + batch_size, 1]]
+            result[i : i + len(x_sample)] = pairwise_op(x_sample, y_sample)
+
+        parallel_map(par_func, np.arange(0, num_pairs, batch_size))
+
+        return result
+
+    return batched_fn
+
+
+@batch_processing
+def pairwise_corr(x_sample: np.ndarray, y_sample: np.ndarray) -> np.ndarray:
+    """
+    Compute pearson correlation between two matrices in a paired row-wise
+    fashion. `x_sample` and `y_sample` must be of the same shape.
+    """
+    x_mean = x_sample.mean(axis=1, keepdims=True)
+    y_mean = y_sample.mean(axis=1, keepdims=True)
+
+    x_center = x_sample - x_mean
+    y_center = y_sample - y_mean
+
+    numer = (x_center * y_center).sum(axis=1)
+
+    denom = (x_center**2).sum(axis=1) * (y_center**2).sum(axis=1)
+    denom = np.sqrt(denom)
+
+    corrs = numer / denom
+    return corrs
+
+
+@batch_processing
+def pairwise_cosine(x_sample: np.ndarray, y_sample: np.ndarray) -> np.ndarray:
+    x_norm = x_sample / np.linalg.norm(x_sample, axis=1)[:, np.newaxis]
+    y_norm = y_sample / np.linalg.norm(y_sample, axis=1)[:, np.newaxis]
+    c_sim = np.sum(x_norm * y_norm, axis=1)
+    return c_sim
+
+
+def random_binary_matrix(n, m, k, rng):
+    """Generate a random binary matrix of n*m with exactly k values in 1 per row.
+    Args:
+    n: Number of rows.
+    m: Number of columns.
+    k: Number of 1's per row.
+
+    Returns:
+    A: Random binary matrix of n*m with exactly k values in 1 per row.
+    """
+    matrix = np.zeros((n, m), dtype=int)
+    matrix[:, :k] = 1
+    rng.permuted(matrix, axis=1, out=matrix)
+    return matrix
+
+
+def average_precision(rel_k) -> np.ndarray:
+    """Compute average precision based on binary list sorted by relevance"""
+    tp = np.cumsum(rel_k, axis=1)
+    num_pos = tp[:, -1]
+    k = np.arange(1, rel_k.shape[1] + 1)
+    pr_k = tp / k
+    ap = (pr_k * rel_k).sum(axis=1) / num_pos
+    return ap
+
+
+def ap_contiguous(rel_k_list, counts):
+    """Compute average precision from a list of contiguous values"""
+    cutoffs = to_cutoffs(counts)
+
+    num_pos = np.add.reduceat(rel_k_list, cutoffs)
+    shift = np.empty_like(num_pos)
+    shift[0], shift[1:] = 0, num_pos[:-1]
+
+    tp = rel_k_list.cumsum() - np.repeat(shift.cumsum(), counts)
+    k = np.arange(1, len(rel_k_list) + 1) - np.repeat(cutoffs, counts)
+
+    pr_k = tp / k
+    ap_scores = np.add.reduceat(pr_k * rel_k_list, cutoffs) / num_pos
+    null_confs = np.stack([num_pos, counts], axis=1)
+    return ap_scores, null_confs
+
+
+def random_ap(num_perm: int, num_pos: int, total: int, seed) -> np.ndarray:
+    """Compute multiple average_precision scores generated at random"""
+    rng = np.random.default_rng(seed)
+    rel_k = random_binary_matrix(num_perm, total, num_pos, rng)
+    null_dist = average_precision(rel_k)
+    return null_dist
+
+
+def null_dist_cached(num_pos, total, seed, null_size, cache_dir):
+    if seed is not None:
+        cache_file = cache_dir / f"n{total}_k{num_pos}.npy"
+        if cache_file.is_file():
+            null_dist = np.load(cache_file)
+        else:
+            null_dist = random_ap(null_size, num_pos, total, seed)
+            np.save(cache_file, null_dist)
+    else:
+        null_dist = random_ap(null_size, num_pos, total, seed)
+    return null_dist
+
+
+def get_null_dists(confs, null_size, seed):
+    cache_dir = Path.home() / ".copairs" / f"seed{seed}" / f"ns{null_size}"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    num_confs = len(confs)
+    rng = np.random.default_rng(seed)
+    seeds = rng.integers(8096, size=num_confs)
+
+    null_dists = np.empty([len(confs), null_size], dtype=np.float32)
+
+    def par_func(i):
+        num_pos, total = confs[i]
+        null_dists[i] = null_dist_cached(num_pos, total, seeds[i], null_size, cache_dir)
+
+    parallel_map(par_func, np.arange(num_confs))
+    return null_dists
+
+
+def p_values(ap_scores: np.ndarray, null_confs: np.ndarray, null_size: int, seed: int):
+    """Calculate p values for an array of ap_scores and null configurations. It uses the path
+    folder to cache null calculations.
+
+    Parameters
+    ----------
+    ap_scores : np.ndarray
+        Ap scores for which to calculate p value.
+    null_confs : np.ndarray
+        Number of average precisions calculated. It serves as an indicator of
+        how relevant is the resultant score.
+    null_size : int
+    seed : int
+        Random initializing value.
+
+    Examples
+    --------
+    FIXME: Add docs.
+
+
+    """
+    confs, rev_ix = np.unique(null_confs, axis=0, return_inverse=True)
+    null_dists = get_null_dists(confs, null_size, seed)
+    null_dists.sort(axis=1)
+    pvals = np.empty(len(ap_scores), dtype=np.float32)
+    for i, (ap_score, ix) in enumerate(zip(ap_scores, rev_ix)):
+        # Reverse to get from hi to low
+        num = null_size - np.searchsorted(null_dists[ix], ap_score)
+        pvals[i] = (num + 1) / (null_size + 1)
+    return pvals
+
+
+def concat_ranges(start: np.ndarray, end: np.ndarray) -> np.ndarray:
+    """Create a 1-d array concatenating multiple ranges"""
+    slices = map(range, start, end)
+    slices = itertools.chain.from_iterable(slices)
+    count = (end - start).sum()
+    mask = np.fromiter(slices, dtype=np.int32, count=count)
+    return mask
+
+
+def to_cutoffs(counts: np.ndarray):
+    """Convert a list of counts into cutoff indices."""
+    cutoffs = np.empty_like(counts)
+    cutoffs[0], cutoffs[1:] = 0, counts.cumsum()[:-1]
+    return cutoffs

copairs-0.4.1/src/copairs/map/__init__.py

@@ -0,0 +1,3 @@
+from .map import mean_average_precision
+from . import multilabel
+from .average_precision import average_precision

copairs-0.4.1/src/copairs/map/average_precision.py

@@ -0,0 +1,94 @@
+import itertools
+import logging
+
+import numpy as np
+import pandas as pd
+
+from copairs import compute
+from copairs.matching import Matcher, UnpairedException
+
+from .filter import evaluate_and_filter, flatten_str_list, validate_pipeline_input
+
+logger = logging.getLogger("copairs")
+
+
+def build_rank_lists(pos_pairs, neg_pairs, pos_sims, neg_sims):
+    labels = np.concatenate(
+        [
+            np.ones(pos_pairs.size, dtype=np.int32),
+            np.zeros(neg_pairs.size, dtype=np.int32),
+        ]
+    )
+    ix = np.concatenate([pos_pairs.ravel(), neg_pairs.ravel()])
+    sim_all = np.concatenate([np.repeat(pos_sims, 2), np.repeat(neg_sims, 2)])
+    ix_sort = np.lexsort([1 - sim_all, ix])
+    rel_k_list = labels[ix_sort]
+    paired_ix, counts = np.unique(ix, return_counts=True)
+    return paired_ix, rel_k_list, counts
+
+
+def average_precision(
+    meta, feats, pos_sameby, pos_diffby, neg_sameby, neg_diffby, batch_size=20000
+) -> pd.DataFrame:
+    columns = flatten_str_list(pos_sameby, pos_diffby, neg_sameby, neg_diffby)
+    validate_pipeline_input(meta, feats, columns)
+
+    # Critical!, otherwise the indexing wont work
+    meta = meta.reset_index(drop=True).copy()
+    logger.info("Indexing metadata...")
+    matcher = Matcher(*evaluate_and_filter(meta, columns), seed=0)
+
+    logger.info("Finding positive pairs...")
+    pos_pairs = matcher.get_all_pairs(sameby=pos_sameby, diffby=pos_diffby)
+    pos_total = sum(len(p) for p in pos_pairs.values())
+    if pos_total == 0:
+        raise UnpairedException("Unable to find positive pairs.")
+    pos_pairs = np.fromiter(
+        itertools.chain.from_iterable(pos_pairs.values()),
+        dtype=np.dtype((np.int32, 2)),
+        count=pos_total,
+    )
+
+    logger.info("Finding negative pairs...")
+    neg_pairs = matcher.get_all_pairs(sameby=neg_sameby, diffby=neg_diffby)
+    neg_total = sum(len(p) for p in neg_pairs.values())
+    if neg_total == 0:
+        raise UnpairedException("Unable to find negative pairs.")
+    neg_pairs = np.fromiter(
+        itertools.chain.from_iterable(neg_pairs.values()),
+        dtype=np.dtype((np.int32, 2)),
+        count=neg_total,
+    )
+
+    logger.info("Computing positive similarities...")
+    pos_sims = compute.pairwise_cosine(feats, pos_pairs, batch_size)
+
+    logger.info("Computing negative similarities...")
+    neg_sims = compute.pairwise_cosine(feats, neg_pairs, batch_size)
+
+    logger.info("Building rank lists...")
+    paired_ix, rel_k_list, counts = build_rank_lists(
+        pos_pairs, neg_pairs, pos_sims, neg_sims
+    )
+
+    logger.info("Computing average precision...")
+    ap_scores, null_confs = compute.ap_contiguous(rel_k_list, counts)
+
+    logger.info("Creating result DataFrame...")
+    meta["n_pos_pairs"] = 0
+    meta["n_total_pairs"] = 0
+    meta.loc[paired_ix, "average_precision"] = ap_scores
+    meta.loc[paired_ix, "n_pos_pairs"] = null_confs[:, 0]
+    meta.loc[paired_ix, "n_total_pairs"] = null_confs[:, 1]
+    logger.info("Finished.")
+    return meta
+
+
+def p_values(dframe: pd.DataFrame, null_size: int, seed: int):
+    """Compute p-values"""
+    mask = dframe["n_pos_pairs"] > 0
+    pvals = np.full(len(dframe), np.nan, dtype=np.float32)
+    scores = dframe.loc[mask, "average_precision"].values
+    null_confs = dframe.loc[mask, ["n_pos_pairs", "n_total_pairs"]].values
+    pvals[mask] = compute.p_values(scores, null_confs, null_size, seed)
+    return pvals