redisbench-admin 0.11.64__py3-none-any.whl → 0.11.66__py3-none-any.whl

redisbench_admin/run/ann/pkg/create_website.py

@@ -0,0 +1,272 @@
+import matplotlib as mpl
+mpl.use('Agg')  # noqa
+import argparse
+import os
+import json
+import pickle
+import yaml
+import numpy
+import hashlib
+from jinja2 import Environment, FileSystemLoader
+
+from ann_benchmarks import results
+from ann_benchmarks.datasets import get_dataset
+from ann_benchmarks.plotting.plot_variants import (all_plot_variants
+                                                   as plot_variants)
+from ann_benchmarks.plotting.metrics import all_metrics as metrics
+from ann_benchmarks.plotting.utils import (get_plot_label, compute_metrics,
+                                           compute_all_metrics,
+                                           create_pointset,
+                                           create_linestyles)
+import plot
+
+colors = [
+    "rgba(166,206,227,1)",
+    "rgba(31,120,180,1)",
+    "rgba(178,223,138,1)",
+    "rgba(51,160,44,1)",
+    "rgba(251,154,153,1)",
+    "rgba(227,26,28,1)",
+    "rgba(253,191,111,1)",
+    "rgba(255,127,0,1)",
+    "rgba(202,178,214,1)"
+]
+
+point_styles = {
+    "o": "circle",
+    "<": "triangle",
+    "*": "star",
+    "x": "cross",
+    "+": "rect",
+}
+
+
+def convert_color(color):
+    r, g, b, a = color
+    return "rgba(%(r)d, %(g)d, %(b)d, %(a)d)" % {
+        "r": r * 255, "g": g * 255, "b": b * 255, "a": a}
+
+
+def convert_linestyle(ls):
+    new_ls = {}
+    for algo in ls.keys():
+        algostyle = ls[algo]
+        new_ls[algo] = (convert_color(algostyle[0]),
+                        convert_color(algostyle[1]),
+                        algostyle[2], point_styles[algostyle[3]])
+    return new_ls
+
+
+def get_run_desc(properties):
+    return "%(dataset)s_%(count)d_%(distance)s" % properties
+
+
+def get_dataset_from_desc(desc):
+    return desc.split("_")[0]
+
+
+def get_count_from_desc(desc):
+    return desc.split("_")[1]
+
+
+def get_distance_from_desc(desc):
+    return desc.split("_")[2]
+
+
+def get_dataset_label(desc):
+    return "{} (k = {})".format(get_dataset_from_desc(desc),
+                                get_count_from_desc(desc))
+
+
+def directory_path(s):
+    if not os.path.isdir(s):
+        raise argparse.ArgumentTypeError("'%s' is not a directory" % s)
+    return s + "/"
+
+
+def prepare_data(data, xn, yn):
+    """Change format from (algo, instance, dict) to (algo, instance, x, y)."""
+    res = []
+    for algo, algo_name, result in data:
+        res.append((algo, algo_name, result[xn], result[yn]))
+    return res
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '--plottype',
+    help='Generate only the plots specified',
+    nargs='*',
+    choices=plot_variants.keys(),
+    default=plot_variants.keys())
+parser.add_argument(
+    '--outputdir',
+    help='Select output directory',
+    default='.',
+    type=directory_path,
+    action='store')
+parser.add_argument(
+    '--latex',
+    help='generates latex code for each plot',
+    action='store_true')
+parser.add_argument(
+    '--scatter',
+    help='create scatterplot for data',
+    action='store_true')
+parser.add_argument(
+    '--recompute',
+    help='Clears the cache and recomputes the metrics',
+    action='store_true')
+args = parser.parse_args()
+
+
+def get_lines(all_data, xn, yn, render_all_points):
+    """ For each algorithm run on a dataset, obtain its performance
+    curve coords."""
+    plot_data = []
+    for algo in sorted(all_data.keys(), key=lambda x: x.lower()):
+        xs, ys, ls, axs, ays, als = \
+            create_pointset(prepare_data(all_data[algo], xn, yn), xn, yn)
+        if render_all_points:
+            xs, ys, ls = axs, ays, als
+        plot_data.append({"name": algo, "coords": zip(xs, ys), "labels": ls,
+                          "scatter": render_all_points})
+    return plot_data
+
+
+def create_plot(all_data, xn, yn, linestyle, j2_env, additional_label="",
+                plottype="line"):
+    xm, ym = (metrics[xn], metrics[yn])
+    render_all_points = plottype == "bubble"
+    plot_data = get_lines(all_data, xn, yn, render_all_points)
+    latex_code = j2_env.get_template("latex.template").\
+        render(plot_data=plot_data, caption=get_plot_label(xm, ym),
+               xlabel=xm["description"], ylabel=ym["description"])
+    plot_data = get_lines(all_data, xn, yn, render_all_points)
+    button_label = hashlib.sha224((get_plot_label(xm, ym) + additional_label)
+                                  .encode("utf-8")).hexdigest()
+    return j2_env.get_template("chartjs.template").\
+        render(args=args, latex_code=latex_code, button_label=button_label,
+               data_points=plot_data,
+               xlabel=xm["description"], ylabel=ym["description"],
+               plottype=plottype, plot_label=get_plot_label(xm, ym),
+               label=additional_label, linestyle=linestyle,
+               render_all_points=render_all_points)
+
+
+def build_detail_site(data, label_func, j2_env, linestyles, batch=False):
+    for (name, runs) in data.items():
+        print("Building '%s'" % name)
+        all_runs = runs.keys()
+        label = label_func(name)
+        data = {"normal": [], "scatter": []}
+
+        for plottype in args.plottype:
+            xn, yn = plot_variants[plottype]
+            data["normal"].append(create_plot(
+                runs, xn, yn, convert_linestyle(linestyles), j2_env))
+            if args.scatter:
+                data["scatter"].append(
+                    create_plot(runs, xn, yn, convert_linestyle(linestyles),
+                                j2_env, "Scatterplot ", "bubble"))
+
+        # create png plot for summary page
+        data_for_plot = {}
+        for k in runs.keys():
+            data_for_plot[k] = prepare_data(runs[k], 'k-nn', 'qps')
+        plot.create_plot(
+            data_for_plot, False,
+            'linear', 'log', 'k-nn', 'qps',
+            args.outputdir + name + '.png',
+            linestyles, batch)
+        output_path = \
+            args.outputdir + name + '.html'
+        with open(output_path, "w") as text_file:
+            text_file.write(j2_env.get_template("detail_page.html").
+                            render(title=label, plot_data=data,
+                                   args=args, batch=batch))
+
+
+def build_index_site(datasets, algorithms, j2_env, file_name):
+    dataset_data = {'batch': [], 'non-batch': []}
+    for mode in ['batch', 'non-batch']:
+        distance_measures = sorted(
+            set([get_distance_from_desc(e) for e in datasets[mode].keys()]))
+        sorted_datasets = sorted(
+            set([get_dataset_from_desc(e) for e in datasets[mode].keys()]))
+
+        for dm in distance_measures:
+            d = {"name": dm.capitalize(), "entries": []}
+            for ds in sorted_datasets:
+                matching_datasets = [e for e in datasets[mode].keys()
+                                     if get_dataset_from_desc(e) == ds and  # noqa
+                                     get_distance_from_desc(e) == dm]
+                sorted_matches = sorted(
+                    matching_datasets,
+                    key=lambda e: int(get_count_from_desc(e)))
+                for idd in sorted_matches:
+                    d["entries"].append(
+                        {"name": idd, "desc": get_dataset_label(idd)})
+            dataset_data[mode].append(d)
+
+    with open(args.outputdir + "index.html", "w") as text_file:
+        text_file.write(j2_env.get_template("summary.html").
+                        render(title="ANN-Benchmarks",
+                               dataset_with_distances=dataset_data,
+                               algorithms=algorithms))
+
+
+def load_all_results():
+    """Read all result files and compute all metrics"""
+    all_runs_by_dataset = {'batch': {}, 'non-batch': {}}
+    all_runs_by_algorithm = {'batch': {}, 'non-batch': {}}
+    cached_true_dist = []
+    old_sdn = None
+    for mode in ["non-batch", "batch"]:
+        for properties, f in results.load_all_results(batch_mode=(mode == "batch")):
+            sdn = get_run_desc(properties)
+            if sdn != old_sdn:
+                dataset, _ = get_dataset(properties["dataset"])
+                cached_true_dist = list(dataset["distances"])
+                old_sdn = sdn
+            algo_ds = get_dataset_label(sdn)
+            desc_suffix = ("-batch" if mode == "batch" else "")
+            algo = properties["algo"] + desc_suffix
+            sdn += desc_suffix
+            ms = compute_all_metrics(
+                cached_true_dist, f, properties, args.recompute)
+            all_runs_by_algorithm[mode].setdefault(
+                algo, {}).setdefault(algo_ds, []).append(ms)
+            all_runs_by_dataset[mode].setdefault(
+                sdn, {}).setdefault(algo, []).append(ms)
+
+    return (all_runs_by_dataset, all_runs_by_algorithm)
+
+
+j2_env = Environment(loader=FileSystemLoader("./templates/"), trim_blocks=True)
+j2_env.globals.update(zip=zip, len=len)
+runs_by_ds, runs_by_algo = load_all_results()
+dataset_names = [get_dataset_label(x) for x in list(
+    runs_by_ds['batch'].keys()) + list(runs_by_ds['non-batch'].keys())]
+algorithm_names = list(runs_by_algo['batch'].keys(
+)) + list(runs_by_algo['non-batch'].keys())
+
+linestyles = {**create_linestyles(dataset_names),
+              **create_linestyles(algorithm_names)}
+
+build_detail_site(
+    runs_by_ds['non-batch'],
+    lambda label: get_dataset_label(label), j2_env, linestyles, False)
+
+build_detail_site(
+    runs_by_ds['batch'],
+    lambda label: get_dataset_label(label), j2_env, linestyles, True)
+
+build_detail_site(
+    runs_by_algo['non-batch'],
+    lambda x: x, j2_env, linestyles, False)
+
+build_detail_site(
+    runs_by_algo['batch'], lambda x: x, j2_env, linestyles, True)
+
+build_index_site(runs_by_ds, runs_by_algo, j2_env, "index.html")

redisbench_admin/run/ann/pkg/install/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:18.04
+
+RUN apt-get update
+RUN apt-get install -y python3-numpy python3-scipy python3-pip build-essential git
+RUN pip3 install -U pip
+
+WORKDIR /home/app
+COPY requirements.txt run_algorithm.py ./
+RUN pip3 install -r requirements.txt
+
+ENTRYPOINT ["python3", "-u", "run_algorithm.py"]

redisbench_admin/run/ann/pkg/install/Dockerfile.annoy

@@ -0,0 +1,5 @@
+FROM ann-benchmarks
+
+RUN git clone https://github.com/spotify/annoy
+RUN cd annoy && python3 setup.py install
+RUN python3 -c 'import annoy'

redisbench_admin/run/ann/pkg/install/Dockerfile.datasketch

@@ -0,0 +1,4 @@
+FROM ann-benchmarks
+
+RUN pip3 install datasketch
+RUN python3 -c 'import datasketch'

redisbench_admin/run/ann/pkg/install/Dockerfile.diskann

@@ -0,0 +1,29 @@
+FROM ann-benchmarks
+
+RUN apt-get update
+RUN apt-get install -y wget git cmake g++ libaio-dev libgoogle-perftools-dev clang-format-4.0 libboost-dev python3 python3-setuptools python3-pip
+RUN pip3 install pybind11 numpy
+
+RUN cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN cd /tmp && apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN cd /tmp && rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN cd /tmp && sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
+RUN apt-get update
+RUN apt-get install -y intel-mkl-64bit-2020.0-088
+
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/libblas.so     libblas.so-x86_64-linux-gnu      /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/libblas.so.3   libblas.so.3-x86_64-linux-gnu    /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/liblapack.so   liblapack.so-x86_64-linux-gnu    /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/liblapack.so.3 liblapack.so.3-x86_64-linux-gnu  /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+
+RUN echo "/opt/intel/lib/intel64"     >  /etc/ld.so.conf.d/mkl.conf
+RUN echo "/opt/intel/mkl/lib/intel64" >> /etc/ld.so.conf.d/mkl.conf
+RUN ldconfig
+RUN echo "MKL_THREADING_LAYER=GNU" >> /etc/environment
+
+RUN git clone --single-branch --branch python_bindings https://github.com/microsoft/diskann
+RUN mkdir -p diskann/build
+RUN cd diskann/build && cmake -DCMAKE_BUILD_TYPE=Release ..
+RUN cd diskann/build && make -j
+RUN cd diskann/python && pip install -e .
+RUN python3 -c 'import vamanapy'

redisbench_admin/run/ann/pkg/install/Dockerfile.diskann_pq

@@ -0,0 +1,31 @@
+FROM ann-benchmarks
+
+RUN apt-get update
+RUN apt-get install -y wget git cmake g++ libaio-dev libgoogle-perftools-dev clang-format-4.0 libboost-dev python3 python3-setuptools python3-pip
+RUN pip3 install pybind11 numpy
+
+RUN cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN cd /tmp && apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN cd /tmp && rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN cd /tmp && sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
+RUN apt-get update
+RUN apt-get install -y intel-mkl-64bit-2020.0-088
+
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/libblas.so     libblas.so-x86_64-linux-gnu      /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/libblas.so.3   libblas.so.3-x86_64-linux-gnu    /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/liblapack.so   liblapack.so-x86_64-linux-gnu    /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+RUN update-alternatives --install /usr/lib/x86_64-linux-gnu/liblapack.so.3 liblapack.so.3-x86_64-linux-gnu  /opt/intel/mkl/lib/intel64/libmkl_rt.so 150
+
+RUN echo "/opt/intel/lib/intel64"     >  /etc/ld.so.conf.d/mkl.conf
+RUN echo "/opt/intel/mkl/lib/intel64" >> /etc/ld.so.conf.d/mkl.conf
+RUN ldconfig
+RUN echo "MKL_THREADING_LAYER=GNU" >> /etc/environment
+RUN export LD_LIBRARY_PATH="$PATH:/opt/intel/compilers_and_libraries/linux/lib/intel64"
+RUN export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/intel/compilers_and_libraries/linux/mkl/lib/intel64/"
+
+RUN git clone --single-branch --branch python_bindings_quantized https://github.com/microsoft/diskann
+RUN mkdir -p diskann/build
+RUN cd diskann/build && cmake -DCMAKE_BUILD_TYPE=Release ..
+RUN cd diskann/build && make -j
+RUN cd diskann/python && pip install -e .
+RUN python3 -c 'import vamanapy'

redisbench_admin/run/ann/pkg/install/Dockerfile.dolphinn

@@ -0,0 +1,5 @@
+FROM ann-benchmarks
+
+RUN git clone https://github.com/ipsarros/DolphinnPy lib-dolphinnpy
+ENV PYTHONPATH lib-dolphinnpy
+RUN python3 -c 'import dolphinn'

redisbench_admin/run/ann/pkg/install/Dockerfile.elasticsearch

@@ -0,0 +1,45 @@
+FROM ann-benchmarks
+
+WORKDIR /home/app
+
+# Install elasticsearch.
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt install -y wget curl htop
+RUN wget --quiet https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-amd64.deb \
+    && dpkg -i elasticsearch-7.9.2-amd64.deb \
+    && rm elasticsearch-7.9.2-amd64.deb
+
+# Install python client.
+RUN python3 -m pip install --upgrade elasticsearch==7.9.1
+
+# Configure elasticsearch and JVM for single-node, single-core.
+RUN echo '\
+discovery.type: single-node\n\
+network.host: 0.0.0.0\n\
+node.master: true\n\
+node.data: true\n\
+node.processors: 1\n\
+thread_pool.write.size: 1\n\
+thread_pool.search.size: 1\n\
+thread_pool.search.queue_size: 1\n\
+path.data: /var/lib/elasticsearch\n\
+path.logs: /var/log/elasticsearch\n\
+' > /etc/elasticsearch/elasticsearch.yml
+
+RUN echo '\
+-Xms3G\n\
+-Xmx3G\n\
+-XX:+UseG1GC\n\
+-XX:G1ReservePercent=25\n\
+-XX:InitiatingHeapOccupancyPercent=30\n\
+-XX:+HeapDumpOnOutOfMemoryError\n\
+-XX:HeapDumpPath=/var/lib/elasticsearch\n\
+-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log\n\
+-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m' > /etc/elasticsearch/jvm.options
+
+# Make sure you can start the service.
+RUN service elasticsearch start && service elasticsearch stop
+
+# Custom entrypoint that also starts the Elasticsearch server.
+RUN echo 'service elasticsearch start && python3 -u run_algorithm.py "$@"' > entrypoint.sh
+ENTRYPOINT ["/bin/bash", "/home/app/entrypoint.sh"]

redisbench_admin/run/ann/pkg/install/Dockerfile.elastiknn

@@ -0,0 +1,61 @@
+FROM ann-benchmarks
+
+WORKDIR /home/app
+
+# Install elasticsearch.
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt install -y wget curl htop
+RUN wget --quiet https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.9.2-amd64.deb \
+    && dpkg -i elasticsearch-oss-7.9.2-amd64.deb \
+    && rm elasticsearch-oss-7.9.2-amd64.deb
+
+# Install python client.
+RUN python3 -m pip install --upgrade elastiknn-client==0.1.0rc47
+
+# Install plugin.
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch \
+    https://github.com/alexklibisz/elastiknn/releases/download/0.1.0-PRE47/elastiknn-0.1.0-PRE47_es7.9.2.zip
+
+# Configure elasticsearch and JVM for single-node, single-core.
+RUN cp /etc/elasticsearch/jvm.options /etc/elasticsearch/jvm.options.bak
+RUN cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
+
+RUN echo '\
+discovery.type: single-node\n\
+network.host: 0.0.0.0\n\
+node.master: true\n\
+node.data: true\n\
+node.processors: 1\n\
+thread_pool.write.size: 1\n\
+thread_pool.search.size: 1\n\
+thread_pool.search.queue_size: 1\n\
+path.data: /var/lib/elasticsearch\n\
+path.logs: /var/log/elasticsearch\n\
+' > /etc/elasticsearch/elasticsearch.yml
+
+RUN echo '\
+-Xms3G\n\
+-Xmx3G\n\
+-XX:+UseG1GC\n\
+-XX:G1ReservePercent=25\n\
+-XX:InitiatingHeapOccupancyPercent=30\n\
+-XX:+HeapDumpOnOutOfMemoryError\n\
+-XX:HeapDumpPath=/var/lib/elasticsearch\n\
+-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log\n\
+-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m\n\
+-Dcom.sun.management.jmxremote.ssl=false\n\
+-Dcom.sun.management.jmxremote.authenticate=false\n\
+-Dcom.sun.management.jmxremote.local.only=false\n\
+-Dcom.sun.management.jmxremote.port=8097\n\
+-Dcom.sun.management.jmxremote.rmi.port=8097\n\
+-Djava.rmi.server.hostname=localhost' > /etc/elasticsearch/jvm.options
+
+# JMX port. Need to also map the port when running.
+EXPOSE 8097
+
+# Make sure you can start the service.
+RUN service elasticsearch start && service elasticsearch stop
+
+# Custom entrypoint that also starts the Elasticsearch server.\
+RUN echo 'service elasticsearch start && python3 -u run_algorithm.py "$@"' > entrypoint.sh
+ENTRYPOINT ["/bin/bash", "/home/app/entrypoint.sh"]

redisbench_admin/run/ann/pkg/install/Dockerfile.faiss

@@ -0,0 +1,18 @@
+FROM ann-benchmarks
+
+RUN apt update && apt install -y wget
+RUN wget https://repo.anaconda.com/archive/Anaconda3-2020.11-Linux-x86_64.sh
+RUN bash Anaconda3-2020.11-Linux-x86_64.sh -b
+
+ENV PATH /root/anaconda3/bin:$PATH
+
+RUN python3 -m pip install ansicolors==1.1.8 docker==5.0.2
+RUN conda install -c pytorch faiss-cpu h5py numpy 
+
+# https://developpaper.com/a-pit-of-mkl-library-in-linux-anaconda/
+ENV LD_PRELOAD /root/anaconda3/lib/libmkl_core.so:/root/anaconda3/lib/libmkl_sequential.so 
+
+RUN python3 -c 'import faiss; print(faiss.IndexFlatL2)'
+
+
+

redisbench_admin/run/ann/pkg/install/Dockerfile.flann

@@ -0,0 +1,10 @@
+FROM ann-benchmarks
+
+RUN apt-get update && apt-get install -y cmake pkg-config liblz4-dev
+RUN git clone https://github.com/mariusmuja/flann
+RUN mkdir flann/build
+RUN cd flann/build && cmake ..
+RUN cd flann/build && make -j4
+RUN cd flann/build && make install
+RUN pip3 install sklearn
+RUN python3 -c 'import pyflann'

redisbench_admin/run/ann/pkg/install/Dockerfile.hnswlib

@@ -0,0 +1,10 @@
+FROM ann-benchmarks
+
+RUN apt-get install -y python-setuptools python-pip
+RUN pip3 install pybind11 numpy setuptools
+RUN git clone https://github.com/nmslib/hnsw.git;cd hnsw; git checkout denorm
+
+RUN cd hnsw/python_bindings; python3 setup.py install
+
+RUN python3 -c 'import hnswlib'
+

redisbench_admin/run/ann/pkg/install/Dockerfile.kgraph

@@ -0,0 +1,6 @@
+FROM ann-benchmarks
+
+RUN apt-get update && apt-get install -y libboost-timer-dev libboost-chrono-dev libboost-program-options-dev libboost-system-dev libboost-python-dev
+RUN git clone https://github.com/aaalgo/kgraph
+RUN cd kgraph && python3 setup.py build && python3 setup.py install
+RUN python3 -c 'import pykgraph'

redisbench_admin/run/ann/pkg/install/Dockerfile.mih

@@ -0,0 +1,4 @@
+FROM ann-benchmarks
+RUN apt-get update && apt-get install -y cmake libhdf5-dev
+RUN git clone https://github.com/maumueller/mih
+RUN cd mih && mkdir bin && cd bin && cmake ../ && make -j4

redisbench_admin/run/ann/pkg/install/Dockerfile.milvus

@@ -0,0 +1,27 @@
+# Install Milvus
+FROM milvusdb/milvus:0.6.0-cpu-d120719-2b40dd as milvus
+RUN apt-get update && apt-get install -y wget
+RUN wget https://raw.githubusercontent.com/milvus-io/docs/master/v0.6.0/assets/server_config.yaml
+RUN sed -i 's/cpu_cache_capacity: 16/cpu_cache_capacity: 4/' server_config.yaml  # otherwise my Docker blows up
+RUN mv server_config.yaml /var/lib/milvus/conf/server_config.yaml
+
+# Switch back to ANN-benchmarks base image and copy all files
+FROM ann-benchmarks
+COPY --from=milvus /var/lib/milvus /var/lib/milvus
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/var/lib/milvus/lib"
+RUN apt-get update
+RUN apt-get install -y libmysqlclient-dev
+
+# Python client
+RUN pip3 install pymilvus==0.2.7
+
+# Fixing some version incompatibility thing
+RUN pip3 install numpy==1.18 scipy==1.1.0 scikit-learn==0.21
+
+# Dumb entrypoint thing that runs the daemon as well
+RUN echo '#!/bin/bash' >> entrypoint.sh
+RUN echo '/var/lib/milvus/bin/milvus_server -d -c /var/lib/milvus/conf/server_config.yaml -l /var/lib/milvus/conf/log_config.conf' >> entrypoint.sh
+RUN echo 'sleep 5' >> entrypoint.sh
+RUN echo 'python3 -u run_algorithm.py "$@"' >> entrypoint.sh
+RUN chmod u+x entrypoint.sh
+ENTRYPOINT ["/home/app/entrypoint.sh"]

redisbench_admin/run/ann/pkg/install/Dockerfile.mrpt

@@ -0,0 +1,4 @@
+FROM ann-benchmarks
+
+RUN pip3 install sklearn
+RUN pip3 install git+https://github.com/vioshyvo/mrpt

redisbench_admin/run/ann/pkg/install/Dockerfile.n2

@@ -0,0 +1,5 @@
+FROM ann-benchmarks
+
+RUN pip3 install cython
+RUN pip3 install n2
+RUN python3 -c 'import n2'

redisbench_admin/run/ann/pkg/install/Dockerfile.nearpy

@@ -0,0 +1,5 @@
+FROM ann-benchmarks
+
+RUN apt-get install -y libhdf5-openmpi-dev cython
+RUN pip3 install nearpy bitarray redis sklearn
+RUN python3 -c 'import nearpy'

redisbench_admin/run/ann/pkg/install/Dockerfile.ngt

@@ -0,0 +1,13 @@
+FROM ann-benchmarks
+
+RUN apt-get update
+RUN apt-get install -y git cmake g++ python3 python3-setuptools python3-pip
+RUN pip3 install wheel pybind11
+RUN git clone https://github.com/yahoojapan/ngt.git
+RUN mkdir -p ngt/build
+RUN cd ngt/build && cmake ..
+RUN cd ngt/build && make && make install
+RUN ldconfig
+RUN cd ngt/python && python3 setup.py bdist_wheel
+RUN pip3 install ngt/python/dist/ngt-*-linux_x86_64.whl
+

redisbench_admin/run/ann/pkg/install/Dockerfile.nmslib

@@ -0,0 +1,10 @@
+FROM ann-benchmarks
+
+RUN apt-get update && apt-get install -y cmake libboost-all-dev libeigen3-dev libgsl0-dev
+RUN git clone https://github.com/searchivarius/nmslib.git
+RUN cd nmslib/similarity_search && cmake . -DWITH_EXTRAS=1
+RUN cd nmslib/similarity_search && make -j4
+RUN pip3 install pybind11
+RUN cd nmslib/python_bindings && python3 setup.py build
+RUN cd nmslib/python_bindings && python3 setup.py install
+RUN python3 -c 'import nmslib'

redisbench_admin/run/ann/pkg/install/Dockerfile.opensearchknn

@@ -0,0 +1,43 @@
+# Warning! Do not use this config in production! 
+# This is only for testing and security has been turned off.
+
+FROM ann-benchmarks
+
+WORKDIR /home/app
+
+# Install Opensearch following instructions from https://opensearch.org/docs/opensearch/install/tar/
+# and https://opensearch.org/docs/opensearch/install/important-settings/
+RUN apt-get install tmux wget gosu -y
+RUN wget https://artifacts.opensearch.org/releases/bundle/opensearch/1.0.0/opensearch-1.0.0-linux-x64.tar.gz
+RUN tar -zxf opensearch-1.0.0-linux-x64.tar.gz
+RUN rm -r opensearch-1.0.0-linux-x64.tar.gz opensearch-1.0.0/plugins/opensearch-security
+RUN chmod -R 777 opensearch-1.0.0
+RUN sysctl -w vm.max_map_count=262144
+
+# Install python client.
+RUN python3 -m pip install --upgrade elasticsearch==7.13.4 tqdm
+
+# Configure opensearch for single-node, single-core.
+RUN echo '\
+discovery.type: single-node\n\
+network.host: 0.0.0.0\n\
+node.roles: [ data, master ]\n\
+node.processors: 1\n\
+thread_pool.write.size: 1\n\
+thread_pool.search.size: 1\n\
+thread_pool.search.queue_size: 1' > opensearch-1.0.0/config/opensearch.yml
+
+RUN echo '\
+-Xms3G\n\
+-Xmx3G\n\
+-XX:InitiatingHeapOccupancyPercent=30\n\
+-XX:+HeapDumpOnOutOfMemoryError\n\
+-XX:HeapDumpPath=data\n\
+-XX:ErrorFile=logs/hs_err_pid%p.log\n\
+-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m' > opensearch-1.0.0/config/jvm.options
+
+# Custom entrypoint that also starts the Opensearch server
+RUN useradd -m opensearch
+RUN echo 'tmux new-session -d -s opensearch """exec gosu opensearch "./opensearch-1.0.0/opensearch-tar-install.sh""""' > entrypoint.sh
+RUN echo 'python3 -u run_algorithm.py "$@"' >> entrypoint.sh
+ENTRYPOINT ["/bin/bash", "/home/app/entrypoint.sh"]

redisbench_admin/run/ann/pkg/install/Dockerfile.puffinn

@@ -0,0 +1,6 @@
+FROM ann-benchmarks
+
+RUN pip3 install pypandoc
+RUN git clone https://github.com/puffinn/puffinn
+RUN cd puffinn && python3 setup.py install
+RUN python3 -c 'import puffinn'

redisbench_admin/run/ann/pkg/install/Dockerfile.pynndescent

@@ -0,0 +1,4 @@
+FROM ann-benchmarks
+
+RUN pip3 install numpy==1.16.4 pynndescent>=0.5
+RUN python3 -c 'import pynndescent'