redisbench-admin 0.11.1__py3-none-any.whl → 0.11.4__py3-none-any.whl

redisbench_admin/compare/compare.py

@@ -653,7 +653,7 @@ def get_by_strings(
         comparison_str = comparison_branch
 
     if baseline_tag is not None:
-        if comparison_covered:
+        if baseline_covered:
             logging.error(
                 "--baseline-branch and --baseline-tag are mutually exclusive. Pick one..."
             )
@@ -1043,9 +1043,9 @@ def get_v_pct_change_and_largest_var(
             if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n):
                 comparison_values.append(tuple[1])
         comparison_df = pd.DataFrame(comparison_values)
-        comparison_median = float(comparison_df.median())
+        comparison_median = float(comparison_df.median().iloc[0])
         comparison_v = comparison_median
-        comparison_std = float(comparison_df.std())
+        comparison_std = float(comparison_df.std().iloc[0])
         if verbose:
             logging.info(
                 "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(

redisbench_admin/deploy/deploy.py

@@ -73,15 +73,7 @@ def deploy_command_logic(args, project_name, project_version):
     tf_triggering_env = "redisbench-admin-deploy"
     logging.info("Setting an infra timeout of {} secs".format(infra_timeout_secs))
     if args.destroy is False:
-        (
-            tf_return_code,
-            _,
-            _,
-            _,
-            _,
-            _,
-            _,
-        ) = setup_remote_environment(
+        (tf_return_code, _, _, _, _, _, _,) = setup_remote_environment(
             tf,
             tf_github_sha,
             tf_github_actor,

redisbench_admin/export/export.py

@@ -42,13 +42,7 @@ def export_command_logic(args, project_name, project_version):
     deployment_name = args.deployment_name
     deployment_type = args.deployment_type
     results_format = args.results_format
-    (
-        _,
-        github_branch,
-        github_org,
-        github_repo,
-        _,
-    ) = git_vars_crosscheck(
+    (_, github_branch, github_org, github_repo, _,) = git_vars_crosscheck(
         None, args.github_branch, args.github_org, args.github_repo, None
     )
     exporter_timemetric_path = None

redisbench_admin/profilers/perf.py

@@ -400,9 +400,9 @@ class Perf:
             "Main THREAD Flame Graph: " + use_case, details
         )
         if artifact_result is True:
-            outputs["Main THREAD Flame Graph {}".format(identifier)] = (
-                flame_graph_output
-            )
+            outputs[
+                "Main THREAD Flame Graph {}".format(identifier)
+            ] = flame_graph_output
         result &= artifact_result
 
         tid = self.pid
@@ -440,9 +440,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report per dso,sym
@@ -460,9 +460,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym with callgraph {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym with callgraph {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report per dso,sym,srcline
@@ -487,9 +487,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym,srcline {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym,srcline {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         self.logger.info(
@@ -527,9 +527,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report top self-cpu {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report top self-cpu {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report --stdio report
@@ -546,9 +546,9 @@ class Perf:
             )
 
             if artifact_result is True:
-                outputs["perf report top self-cpu (dso={})".format(binary)] = (
-                    perf_report_artifact
-                )
+                outputs[
+                    "perf report top self-cpu (dso={})".format(binary)
+                ] = perf_report_artifact
         result &= artifact_result
 
         if self.callgraph_mode == "dwarf":
@@ -590,9 +590,9 @@ class Perf:
                 )
                 result &= artifact_result
                 if artifact_result is True:
-                    outputs["Top entries in text form by LOC"] = (
-                        pprof_artifact_text_output
-                    )
+                    outputs[
+                        "Top entries in text form by LOC"
+                    ] = pprof_artifact_text_output
                     tabular_data_map["text-lines"] = tabular_data
                 self.logger.info("Generating pprof png output")
                 pprof_png_output = self.output + ".pprof.png"
@@ -604,9 +604,9 @@ class Perf:
                     self.output,
                 )
                 if artifact_result is True:
-                    outputs["Output graph image in PNG format"] = (
-                        pprof_artifact_png_output
-                    )
+                    outputs[
+                        "Output graph image in PNG format"
+                    ] = pprof_artifact_png_output
                 result &= artifact_result
 
         # save stack collapsed

redisbench_admin/run/ann/pkg/.dockerignore

@@ -0,0 +1,2 @@
+data
+results

redisbench_admin/run/ann/pkg/.git

@@ -0,0 +1 @@
+gitdir: ../../../../.git/modules/redisbench_admin/run/ann/pkg

redisbench_admin/run/ann/pkg/.github/workflows/benchmarks.yml

@@ -0,0 +1,100 @@
+name: ANN benchmarks
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        include:
+          - library: annoy
+            dataset: random-xs-20-angular
+          - library: dolphinn
+            dataset: random-xs-20-angular
+          - library: faiss
+            dataset: random-xs-20-angular
+          - library: flann
+            dataset: random-xs-20-angular
+          - library: kgraph
+            dataset: random-xs-20-angular
+          - library: milvus
+            dataset: random-xs-20-angular
+          - library: mrpt
+            dataset: random-xs-20-angular
+          - library: n2
+            dataset: random-xs-20-angular
+          - library: nearpy
+            dataset: random-xs-20-angular
+          - library: ngt
+            dataset: random-xs-20-angular
+          - library: nmslib
+            dataset: random-xs-20-angular
+          - library: hnswlib
+            dataset: random-xs-20-angular
+          - library: puffinn
+            dataset: random-xs-20-angular
+          - library: pynndescent
+            dataset: random-xs-20-angular
+          - library: rpforest
+            dataset: random-xs-20-angular
+          - library: sklearn
+            dataset: random-xs-20-angular
+          - library: sptag
+            dataset: random-xs-20-angular
+          - library: mih
+            dataset: random-xs-16-hamming
+          - library: datasketch
+            dataset: random-s-jaccard
+          - library: scann
+            dataset: random-xs-20-angular
+          - library: elasticsearch
+            dataset: random-xs-20-angular
+          - library: elastiknn
+            dataset: random-xs-20-angular
+          - library: opensearchknn
+            dataset: random-xs-20-angular
+          - library: diskann
+            dataset: random-xs-20-angular
+          - library: puffinn
+            dataset: random-s-jaccard
+          - library: pynndescent
+            dataset: random-s-jaccard
+          - library: vespa
+            dataset: random-xs-20-angular
+          - library: scipy
+            dataset: random-xs-20-angular
+          - library: vald
+            dataset: random-xs-20-angular
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v2 # Pull the repository
+
+    - name: Install various apt packages
+      run: sudo apt-get install -y libhdf5-dev python3-numpy python3-scipy python3-matplotlib python3-sklearn
+
+    - name: Install dependencies
+      run: |
+        pip3 install -r requirements.txt
+        python3 install.py
+
+      env:
+        LIBRARY: ${{ matrix.library }}
+        DATASET: ${{ matrix.dataset }}
+    
+    - name: Run the benchmark
+      run: |
+        python3 run.py --docker-tag ann-benchmarks-${LIBRARY} --max-n-algorithms 5 --dataset $DATASET --run-disabled --timeout 300
+        python3 run.py --docker-tag ann-benchmarks-${LIBRARY} --max-n-algorithms 5 --dataset $DATASET --run-disabled --batch --timeout 300
+        sudo chmod -R 777 results/
+        python3 plot.py --dataset $DATASET --output plot.png
+        python3 plot.py --dataset $DATASET --output plot-batch.png --batch
+        python3 -m unittest test/test-metrics.py
+        python3 -m unittest test/test-jaccard.py
+        python3 create_website.py --outputdir . --scatter --latex
+
+      env:
+        LIBRARY: ${{ matrix.library }}
+        DATASET: ${{ matrix.dataset }}

redisbench_admin/run/ann/pkg/.gitignore

@@ -0,0 +1,21 @@
+.DS_Store
+*.pyc
+*.o
+protocol/c/fr-*
+
+install/*.txt
+install/*.yaml
+install/lib-*/
+data/*
+*.class
+
+*.log
+
+results/*
+!results/*.png
+website
+
+venv
+
+.idea
+.vscode

redisbench_admin/run/ann/pkg/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Erik Bernhardsson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

redisbench_admin/run/ann/pkg/README.md

@@ -0,0 +1,157 @@
+Benchmarking nearest neighbors
+==============================
+
+[![Build Status](https://img.shields.io/github/workflow/status/erikbern/ann-benchmarks/ANN%20benchmarks?style=flat-square)](https://github.com/erikbern/ann-benchmarks/actions?query=workflow:benchmarks)
+
+Doing fast searching of nearest neighbors in high dimensional spaces is an increasingly important problem, but so far there has not been a lot of empirical attempts at comparing approaches in an objective way.
+
+This project contains some tools to benchmark various implementations of approximate nearest neighbor (ANN) search for different metrics. We have pregenerated datasets (in HDF5) formats and we also have Docker containers for each algorithm. There's a [test suite](https://travis-ci.org/erikbern/ann-benchmarks) that makes sure every algorithm works.
+
+Evaluated
+=========
+
+* [Annoy](https://github.com/spotify/annoy)
+* [FLANN](http://www.cs.ubc.ca/research/flann/)
+* [scikit-learn](http://scikit-learn.org/stable/modules/neighbors.html): LSHForest, KDTree, BallTree
+* [PANNS](https://github.com/ryanrhymes/panns)
+* [NearPy](http://pixelogik.github.io/NearPy/)
+* [KGraph](https://github.com/aaalgo/kgraph)
+* [NMSLIB (Non-Metric Space Library)](https://github.com/nmslib/nmslib): SWGraph, HNSW, BallTree, MPLSH
+* [hnswlib (a part of nmslib project)](https://github.com/nmslib/hnsw)
+* [RPForest](https://github.com/lyst/rpforest)
+* [FAISS](https://github.com/facebookresearch/faiss.git)
+* [DolphinnPy](https://github.com/ipsarros/DolphinnPy)
+* [Datasketch](https://github.com/ekzhu/datasketch)
+* [PyNNDescent](https://github.com/lmcinnes/pynndescent)
+* [MRPT](https://github.com/teemupitkanen/mrpt)
+* [NGT](https://github.com/yahoojapan/NGT): ONNG, PANNG, QG
+* [SPTAG](https://github.com/microsoft/SPTAG)
+* [PUFFINN](https://github.com/puffinn/puffinn)
+* [N2](https://github.com/kakao/n2)
+* [ScaNN](https://github.com/google-research/google-research/tree/master/scann)
+* [Elastiknn](https://github.com/alexklibisz/elastiknn)
+* [OpenSearch KNN](https://github.com/opensearch-project/k-NN)
+* [DiskANN](https://github.com/microsoft/diskann): Vamana, Vamana-PQ
+* [Vespa](https://github.com/vespa-engine/vespa)
+* [scipy](https://docs.scipy.org/doc/scipy/reference/spatial.html): cKDTree
+* [vald](https://github.com/vdaas/vald)
+
+Data sets
+=========
+
+We have a number of precomputed data sets for this. All data sets are pre-split into train/test and come with ground truth data in the form of the top 100 neighbors. We store them in a HDF5 format:
+
+| Dataset                                                           | Dimensions | Train size | Test size | Neighbors | Distance  | Download                                                                   |
+| ----------------------------------------------------------------- | ---------: | ---------: | --------: | --------: | --------- | -------------------------------------------------------------------------- |
+| [DEEP1B](http://sites.skoltech.ru/compvision/noimi/)              |         96 |  9,990,000 |    10,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/deep-image-96-angular.hdf5) (3.6GB)
+| [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) |        784 |     60,000 |    10,000 |       100 | Euclidean | [HDF5](http://ann-benchmarks.com/fashion-mnist-784-euclidean.hdf5) (217MB) |
+| [GIST](http://corpus-texmex.irisa.fr/)                            |        960 |  1,000,000 |     1,000 |       100 | Euclidean | [HDF5](http://ann-benchmarks.com/gist-960-euclidean.hdf5) (3.6GB)          |
+| [GloVe](http://nlp.stanford.edu/projects/glove/)                  |         25 |  1,183,514 |    10,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/glove-25-angular.hdf5) (121MB)            |
+| GloVe                                                             |         50 |  1,183,514 |    10,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/glove-50-angular.hdf5) (235MB)            |
+| GloVe                                                             |        100 |  1,183,514 |    10,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/glove-100-angular.hdf5) (463MB)           |
+| GloVe                                                             |        200 |  1,183,514 |    10,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/glove-200-angular.hdf5) (918MB)           |
+| [Kosarak](http://fimi.uantwerpen.be/data/)                        |      27983 |     74,962 |       500 |       100 | Jaccard   | [HDF5](http://ann-benchmarks.com/kosarak-jaccard.hdf5) (2.0GB)             |
+| [MNIST](http://yann.lecun.com/exdb/mnist/)                        |        784 |     60,000 |    10,000 |       100 | Euclidean | [HDF5](http://ann-benchmarks.com/mnist-784-euclidean.hdf5) (217MB)         |
+| [NYTimes](https://archive.ics.uci.edu/ml/datasets/bag+of+words)   |        256 |    290,000 |    10,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/nytimes-256-angular.hdf5) (301MB)         |
+| [SIFT](http://corpus-texmex.irisa.fr/)                           |        128 |  1,000,000 |    10,000 |       100 | Euclidean | [HDF5](http://ann-benchmarks.com/sift-128-euclidean.hdf5) (501MB)          |
+| [Last.fm](https://github.com/erikbern/ann-benchmarks/pull/91)     |         65 |    292,385 |    50,000 |       100 | Angular   | [HDF5](http://ann-benchmarks.com/lastfm-64-dot.hdf5) (135MB)               |
+
+Results
+=======
+
+Interactive plots can be found at <http://ann-benchmarks.com>. These are all as of December 2021, running all benchmarks on a r5.4xlarge machine on AWS with `--parallelism 7`:
+
+glove-100-angular
+-----------------
+
+![glove-100-angular](https://raw.github.com/erikbern/ann-benchmarks/master/results/glove-100-angular.png)
+
+sift-128-euclidean
+------------------
+
+![glove-100-angular](https://raw.github.com/erikbern/ann-benchmarks/master/results/sift-128-euclidean.png)
+
+fashion-mnist-784-euclidean
+---------------------------
+
+![fashion-mnist-784-euclidean](https://raw.github.com/erikbern/ann-benchmarks/master/results/fashion-mnist-784-euclidean.png)
+
+lastfm-64-dot
+------------------
+
+![lastfm-64-dot](https://raw.github.com/erikbern/ann-benchmarks/master/results/lastfm-64-dot.png)
+
+nytimes-256-angular
+-------------------
+
+![nytimes-256-angular](https://raw.github.com/erikbern/ann-benchmarks/master/results/nytimes-256-angular.png)
+
+glove-25-angular
+----------------
+
+![glove-25-angular](https://raw.github.com/erikbern/ann-benchmarks/master/results/glove-25-angular.png)
+
+Install
+=======
+
+The only prerequisite is Python (tested with 3.6) and Docker.
+
+1. Clone the repo.
+2. Run `pip install -r requirements.txt`.
+3. Run `python install.py` to build all the libraries inside Docker containers (this can take a while, like 10-30 minutes).
+
+Running
+=======
+
+1. Run `python run.py` (this can take an extremely long time, potentially days)
+2. Run `python plot.py` or `python create_website.py` to plot results.
+
+You can customize the algorithms and datasets if you want to:
+
+* Check that `algos.yaml` contains the parameter settings that you want to test
+* To run experiments on SIFT, invoke `python run.py --dataset glove-100-angular`. See `python run.py --help` for more information on possible settings. Note that experiments can take a long time. 
+* To process the results, either use `python plot.py --dataset glove-100-angular` or `python create_website.py`. An example call: `python create_website.py --plottype recall/time --latex --scatter --outputdir website/`. 
+
+Including your algorithm
+========================
+
+1. Add your algorithm into `ann_benchmarks/algorithms` by providing a small Python wrapper.
+2. Add a Dockerfile in `install/` for it
+3. Add it to `algos.yaml`
+4. Add it to `.github/workflows/benchmarks.yml`
+
+Principles
+==========
+
+* Everyone is welcome to submit pull requests with tweaks and changes to how each library is being used.
+* In particular: if you are the author of any of these libraries, and you think the benchmark can be improved, consider making the improvement and submitting a pull request.
+* This is meant to be an ongoing project and represent the current state.
+* Make everything easy to replicate, including installing and preparing the datasets.
+* Try many different values of parameters for each library and ignore the points that are not on the precision-performance frontier.
+* High-dimensional datasets with approximately 100-1000 dimensions. This is challenging but also realistic. Not more than 1000 dimensions because those problems should probably be solved by doing dimensionality reduction separately.
+* Single queries are used by default. ANN-Benchmarks enforces that only one CPU is saturated during experimentation, i.e., no multi-threading. A batch mode is available that provides all queries to the implementations at once. Add the flag `--batch` to `run.py` and `plot.py` to enable batch mode. 
+* Avoid extremely costly index building (more than several hours).
+* Focus on datasets that fit in RAM. For billion-scale benchmarks, see the related [big-ann-benchmarks](https://github.com/harsha-simhadri/big-ann-benchmarks) project.
+* We mainly support CPU-based ANN algorithms. GPU support exists for FAISS, but it has to be compiled with GPU support locally and experiments must be run using the flags `--local --batch`. 
+* Do proper train/test set of index data and query points.
+* Note that we consider that set similarity datasets are sparse and thus we pass a **sorted** array of integers to algorithms to represent the set of each user.
+
+
+Authors
+=======
+
+Built by [Erik Bernhardsson](https://erikbern.com) with significant contributions from [Martin Aumüller](http://itu.dk/people/maau/) and [Alexander Faithfull](https://github.com/ale-f).
+
+Related Publication
+==================
+
+The following publication details design principles behind the benchmarking framework: 
+
+- M. Aumüller, E. Bernhardsson, A. Faithfull:
+[ANN-Benchmarks: A Benchmarking Tool for Approximate Nearest Neighbor Algorithms](https://arxiv.org/abs/1807.05614). Information Systems 2019. DOI: [10.1016/j.is.2019.02.006](https://doi.org/10.1016/j.is.2019.02.006)
+
+Related Projects
+================
+
+- [big-ann-benchmarks](https://github.com/harsha-simhadri/big-ann-benchmarks) is a benchmarking effort for billion-scale approximate nearest neighbor search as part of the [NeurIPS'21 Competition track](https://neurips.cc/Conferences/2021/CompetitionTrack).
+