sublinear 0.1.0__tar.gz

sublinear-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Ivan Nikitovic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sublinear-0.1.0/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: sublinear
+Version: 0.1.0
+Summary: Python library implementing a subset of streaming algorithms. Includes variations of these algorithms (e.g. adversarially robust), as well as support for multiple data types.
+Home-page: https://github.com/ivannikitovic/sublinear
+Author: Ivan Nikitovic
+Author-email: ivan.bnikitovic@gmail.com
+License: MIT
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# sublinear
+
+Python library implementing a subset of streaming algorithms. Includes variations of these algorithms (e.g. adversarially robust), as well as support for multiple data types.
+
+## :zap: Algorithms
+
+Here is the list of the currently implemented streaming algorithms.
+
+### F0 Estimation (Count of Distinct Elements)
+
+- BJKST Sketch (basic, plus, adversarially robust) [1]
+
+- HyperLogLog [2]
+
+### F1 Estimation (Length of Stream)
+
+- Morris (basic, plus, plus plus) [3][4]
+
+### F2 Estimation (Estimate of Second Moment)
+
+- AMS Sketch (basic, plus, plus plus) [5]
+
+### Frequency Table Estimation
+
+- Count Min Sketch [6]
+
+### Heavy Hitters
+
+- Misra-Gries Sketch [7]
+
+### Other
+
+- K Independent Hash Function [8]
+
+## :book: Bibliography
+
+[1] Bar-Yossef, Ziv, et al. "Counting distinct elements in a data stream." Randomization and Approximation Techniques in Computer Science. Springer Berlin Heidelberg, 2002.
+
+[2] Flajolet, Philippe, et al. "HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm." Conference on Analysis of Algorithms. Springer Berlin Heidelberg, 2007.
+
+[3] Morris, R. "Counting large numbers of events in small registers". Communications of the ACM 21, 10, 1978.
+
+[4] Flajolet, P. "Approximate Counting: A Detailed Analysis". BIT 25, 1985.
+
+[5] Noga Alon, Yossi Matias, Mario Szegedy,
+"The Space Complexity of Approximating the Frequency Moments".
+Journal of Computer and System Sciences,
+Volume 58, Issue 1,
+1999.
+
+[6] Cormode, Graham; S. Muthukrishnan. "An Improved Data Stream Summary: The Count-Min Sketch and its Applications". 2005.
+
+[7] Misra, J.; Gries, David. "Finding repeated elements". Science of Computer Programming. 1982
+
+[8] Wegman, Mark N., et al. "New Hash Functions and Their Use in Authentication and Set Equality". Journal of Computer and System Sciences. 1981.

sublinear-0.1.0/README.md

@@ -0,0 +1,55 @@
+# sublinear
+
+Python library implementing a subset of streaming algorithms. Includes variations of these algorithms (e.g. adversarially robust), as well as support for multiple data types.
+
+## :zap: Algorithms
+
+Here is the list of the currently implemented streaming algorithms.
+
+### F0 Estimation (Count of Distinct Elements)
+
+- BJKST Sketch (basic, plus, adversarially robust) [1]
+
+- HyperLogLog [2]
+
+### F1 Estimation (Length of Stream)
+
+- Morris (basic, plus, plus plus) [3][4]
+
+### F2 Estimation (Estimate of Second Moment)
+
+- AMS Sketch (basic, plus, plus plus) [5]
+
+### Frequency Table Estimation
+
+- Count Min Sketch [6]
+
+### Heavy Hitters
+
+- Misra-Gries Sketch [7]
+
+### Other
+
+- K Independent Hash Function [8]
+
+## :book: Bibliography
+
+[1] Bar-Yossef, Ziv, et al. "Counting distinct elements in a data stream." Randomization and Approximation Techniques in Computer Science. Springer Berlin Heidelberg, 2002.
+
+[2] Flajolet, Philippe, et al. "HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm." Conference on Analysis of Algorithms. Springer Berlin Heidelberg, 2007.
+
+[3] Morris, R. "Counting large numbers of events in small registers". Communications of the ACM 21, 10, 1978.
+
+[4] Flajolet, P. "Approximate Counting: A Detailed Analysis". BIT 25, 1985.
+
+[5] Noga Alon, Yossi Matias, Mario Szegedy,
+"The Space Complexity of Approximating the Frequency Moments".
+Journal of Computer and System Sciences,
+Volume 58, Issue 1,
+1999.
+
+[6] Cormode, Graham; S. Muthukrishnan. "An Improved Data Stream Summary: The Count-Min Sketch and its Applications". 2005.
+
+[7] Misra, J.; Gries, David. "Finding repeated elements". Science of Computer Programming. 1982
+
+[8] Wegman, Mark N., et al. "New Hash Functions and Their Use in Authentication and Set Equality". Journal of Computer and System Sciences. 1981.

sublinear-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sublinear-0.1.0/setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup, find_packages
+
+# read the contents of your README file
+from pathlib import Path
+this_directory = Path(__file__).parent
+long_description = (this_directory / "README.md").read_text()
+
+setup(
+    name="sublinear",
+    version="0.1.0",
+    description="Python library implementing a subset of streaming algorithms. Includes variations of these algorithms (e.g. adversarially robust), as well as support for multiple data types.",
+    long_description_content_type='text/markdown',
+    long_description=long_description,
+    license="MIT",
+    author="Ivan Nikitovic",
+    author_email="ivan.bnikitovic@gmail.com",
+    url="https://github.com/ivannikitovic/sublinear",
+    packages=find_packages(),
+    python_requires=">=3.7",
+    include_package_data=True,
+    install_requires=[
+        'numpy',
+        'matplotlib',
+    ],
+)

sublinear-0.1.0/sublinear/count/morris/experiments/experiment1.py

@@ -0,0 +1,32 @@
+from ..morris_plus_plus import MorrisPlusPlusCounter
+from ..morris_plus import MorrisPlusCounter
+from ..morris_basic import MorrisCounter
+
+def test_morris():
+    morris = MorrisCounter()
+    test_stream = [i for i in range(100)]
+
+    morris.process_stream(test_stream)
+    estimate = morris.estimate_count()
+
+    print(f"Morris:   Estimated count of elements: {estimate}")
+
+    s = 10
+    morris_plus = MorrisPlusCounter(s)
+
+    morris_plus.process_stream(test_stream)
+    estimate = morris_plus.estimate_count()
+
+    print(f"Morris+:  Estimated count of elements: {estimate}")
+
+    epsilon = 0.1
+    delta = 0.1
+    morris_plus_plus = MorrisPlusPlusCounter(epsilon, delta)
+
+    morris_plus_plus.process_stream(test_stream)
+    estimate = morris_plus_plus.estimate_count()
+
+    print(f"Morris++: Estimated count of elements: {estimate}")
+
+    print(f"Actual count of elements: {len(test_stream)}")
+

sublinear-0.1.0/sublinear/count/morris/morris_basic.py

@@ -0,0 +1,49 @@
+import random
+from typing import List
+
+class MorrisCounter:
+    """
+    Morris Counter is a probabilistic counter for counting elements in a stream
+    using logarithmic space. The algorithm was proposed by Robert Morris in 1978.
+
+    Attributes
+    ----------
+    c : int
+        Counter value.
+    """
+
+    def __init__(self):
+        """
+        Initializes Morris Counter.
+        """
+        self.c = 0
+
+    def process_stream(self, stream: List[int]) -> None:
+        """
+        Processes the input data stream with the Morris Counter algorithm.
+
+        Parameters
+        ----------
+        stream : List[int]
+            Data stream represented as a list of integers.
+        """
+        for item in stream:
+            self.process()
+
+    def process(self) -> None:
+        """
+        Updates the counter value with probability 1/(2^c).
+        """
+        if random.random() < 1 / (2 ** self.c):
+            self.c += 1
+
+    def estimate_count(self) -> int:
+        """
+        Estimates the count of the elements in the data stream.
+
+        Returns
+        -------
+        int
+            Estimated count of elements in the data stream.
+        """
+        return (2 ** self.c) - 1

sublinear-0.1.0/sublinear/count/morris/morris_plus.py

@@ -0,0 +1,53 @@
+from typing import List
+from .morris_basic import MorrisCounter
+
+class MorrisPlusCounter:
+    """
+    Morris Plus Counter is an extension of the Morris Counter that runs multiple
+    Morris Counters in parallel and outputs the average of the estimates. This
+    improves the accuracy of the count estimate.
+
+    Attributes
+    ----------
+    s : int
+        Number of Morris Counters.
+    counters : List[MorrisCounter]
+        List of Morris Counters.
+    """
+
+    def __init__(self, s: int):
+        """
+        Initializes Morris Plus Counter.
+
+        Parameters
+        ----------
+        s : int
+            Number of Morris Counters.
+        """
+        self.s = s
+        self.counters = [MorrisCounter() for _ in range(s)]
+
+    def process_stream(self, stream: List[int]) -> None:
+        """
+        Processes the input data stream with the Morris Plus Counter algorithm.
+
+        Parameters
+        ----------
+        stream : List[int]
+            Data stream represented as a list of integers.
+        """
+        for item in stream:
+            for counter in self.counters:
+                counter.process()
+
+    def estimate_count(self) -> int:
+        """
+        Estimates the count of the elements in the data stream.
+
+        Returns
+        -------
+        int
+            Estimated count of elements in the data stream.
+        """
+        estimates = [counter.estimate_count() for counter in self.counters]
+        return int(sum(estimates) / self.s)

sublinear-0.1.0/sublinear/count/morris/morris_plus_plus.py

@@ -0,0 +1,61 @@
+import math
+import numpy as np
+from typing import List
+
+from .morris_plus import MorrisPlusCounter
+
+class MorrisPlusPlusCounter:
+    """
+    Morris Plus Plus Counter is an extension of the Morris Plus Counter that
+    runs multiple Morris Plus Counters in parallel and outputs the median of
+    the estimates. This improves the accuracy of the count estimate further.
+
+    Attributes
+    ----------
+    s : int
+        Number of Morris Counters in each Morris Plus Counter.
+    t : int
+        Number of Morris Plus Counters.
+    counters : List[MorrisPlusCounter]
+        List of Morris Plus Counters.
+    """
+
+    def __init__(self, epsilon: float, delta: float):
+        """
+        Initializes Morris Plus Plus Counter.
+
+        Parameters
+        ----------
+        epsilon : float
+            Error factor.
+        delta : float
+            Confidence factor.
+        """
+        self.s = int(math.ceil(1 / (epsilon ** 2)))
+        self.t = int(math.ceil(math.log(1 / delta)))
+        self.counters = [MorrisPlusCounter(self.s) for _ in range(self.t)]
+
+    def process_stream(self, stream: List[int]) -> None:
+        """
+        Processes the input data stream with the Morris Plus Plus Counter algorithm.
+
+        Parameters
+        ----------
+        stream : List[int]
+            Data stream represented as a list of integers.
+        """
+        for item in stream:
+            for counter in self.counters:
+                counter.process_stream([item])
+
+    def estimate_count(self) -> int:
+        """
+        Estimates the count of the elements in the data stream.
+
+        Returns
+        -------
+        int
+            Estimated count of elements in the data stream.
+        """
+        estimates = [counter.estimate_count() for counter in self.counters]
+        return int(np.median(estimates))

sublinear-0.1.0/sublinear/distinct_elements/bjkst/bjkst_plus.py

@@ -0,0 +1,71 @@
+from typing import List
+from .bjkst_sketch import BJKSTSketch
+from math import ceil, log
+import statistics
+
+class BJKSTSketchPlus:
+    """
+    The BJKST sketch with epsilon-delta approximation is a probabilistic data structure
+    that serves as a distinct element counter in a stream of data.
+
+    This implementation runs multiple instances of the BJKST algorithm and returns the
+    median of the estimates to improve the accuracy of the estimation.
+    """
+
+    def __init__(self, n: int, epsilon: float, delta: float, b: float = 10, max_input_length: int = 64, c: float = 576):
+        """
+        Initializes BJKSTSketchPlus class.
+
+        Parameters
+        ----------
+        Initializes BJKST Sketch class.
+
+        Parameters
+        ----------
+        n: int
+            Size of the input universe.
+
+        b: float
+            Constant factor for controlling the hash function range of g.
+
+        c: float
+            Constant factor for size of set B.
+
+        epsilon: float
+            Approximation error factor.
+
+        delta: float
+            Confidence level.
+
+        max_input_length: int, optional
+            Maximum length of the input elements, used for generating hash functions.
+        """
+        self.instances = []
+        self.num_instances = ceil(log(1 / delta))
+        for _ in range(self.num_instances):
+            self.instances.append(BJKSTSketch(n, epsilon, b, max_input_length=max_input_length, c=c))
+
+    def process_stream(self, stream: List[object]) -> None:
+        """
+        Executes the BJKST algorithm with epsilon-delta approximation on a stream (list) of elements.
+
+        Parameters
+        ----------
+        stream: List[object]
+            Stream of objects represented as a list.
+        """
+        for instance in self.instances:
+            instance.process_stream(stream)
+
+    def estimate_distinct_elements(self) -> int:
+        """
+        Returns the estimated number of distinct elements in the stream using the median of
+        the results from the multiple instances.
+
+        Returns
+        -------
+        int
+            Estimated number of distinct elements.
+        """
+        estimates = [instance.estimate_distinct_elements() for instance in self.instances]
+        return int(statistics.median(estimates))

sublinear-0.1.0/sublinear/distinct_elements/bjkst/bjkst_sketch.py

@@ -0,0 +1,82 @@
+import math
+from typing import List
+from sublinear.utils.hash_generator import HashGenerator
+from sublinear.utils.zeros import Zeros
+
+class BJKSTSketch:
+    """
+    The BJKST sketch is a probabilistic data structure
+    that serves as a distinct element counter in a stream of data.
+
+    It uses two hash functions and a set to estimate the number of distinct elements.
+    It operates in sub-linear space, at the expense of some approximation error.
+
+    The BJKST algorithm was invented by Bar-Yossef, Jayram, Kumar, Sivakumar, and Trevisan.
+    """
+
+    def __init__(self, n: int, epsilon: float, b: float = 10, max_input_length: int = 64, c: float = 576):
+        """
+        Initializes BJKST Sketch class.
+
+        Parameters
+        ----------
+        n: int
+            Size of the input universe.
+
+        b: float
+            Constant factor for controlling the hash function range of g.
+
+        c: float
+            Constant factor for size of set B.
+
+        epsilon: float
+            Approximation error factor.
+
+        max_input_length: int, optional
+            Maximum length of the input elements, used for generating hash functions.
+        """
+        self.n = n
+        self.b = b
+        self.c = c
+        self.epsilon = epsilon
+
+        self.z = 0
+        self.B = set()
+
+        self.h_generator = HashGenerator(self.n, max_input_length, m=n)
+        self.h = self.h_generator.generate_hash_function()
+
+        g_size = int(b * n * (epsilon ** (-4)) * (math.log(n) ** 2))
+        self.g_generator = HashGenerator(self.n, max_input_length, m=g_size)
+        self.g = self.g_generator.generate_hash_function()
+
+    def process_stream(self, stream: List[object]) -> None:
+        """
+        Executes the BJKST algorithm on a stream (list) of elements.
+
+        Parameters
+        ----------
+        stream: List[object]
+            Stream of objects represented as a list.
+        """
+        for token in stream:
+            h_j = self.h_generator.hash_string(token)
+            zeros_hj = Zeros.zeros(h_j)
+            if zeros_hj >= self.z:
+                g_j = self.g_generator.hash_string(token) 
+                self.B.add((g_j, zeros_hj))
+
+                while len(self.B) >= self.c / (self.epsilon ** 2):
+                    self.z += 1
+                    self.B = {(a, b) for (a, b) in self.B if b >= self.z}
+
+    def estimate_distinct_elements(self) -> int:
+        """
+        Returns the estimated number of distinct elements in the stream.
+
+        Returns
+        -------
+        int
+            Estimated number of distinct elements.
+        """
+        return len(self.B) * (2 ** self.z)

sublinear-0.1.0/sublinear/distinct_elements/bjkst/experiments/experiment1.py

@@ -0,0 +1,21 @@
+# Experiment 1 (Test)
+
+from ..bjkst_sketch import BJKSTSketch
+
+def test_bjkst():
+
+    n = 128
+    epsilon = 0.1
+
+    bjkst = BJKSTSketch(n, epsilon)
+
+    test1 = []
+
+    for i in range(100):
+        test1 += i * [f"word{i}"]
+
+    bjkst.process_stream(test1)
+
+    estimate = bjkst.estimate_distinct_elements()
+
+    print("Estimated number of distinct elements:", estimate)

sublinear-0.1.0/sublinear/distinct_elements/bjkst/experiments/experiment2.py

@@ -0,0 +1,22 @@
+# Experiment 1 (Test)
+
+from ..bjkst_plus import BJKSTSketchPlus
+
+def test_bjkst_plus():
+
+    n = 128
+    epsilon = 0.1
+    delta = 0.1
+
+    bjkst_plus = BJKSTSketchPlus(n, epsilon, delta)
+
+    test1 = []
+
+    for i in range(100):
+        test1 += i * [f"word{i}"]
+
+    bjkst_plus.process_stream(test1)
+
+    estimate = bjkst_plus.estimate_distinct_elements()
+
+    print("Estimated number of distinct elements:", estimate)

sublinear-0.1.0/sublinear/distinct_elements/hyperloglog/experiments/experiment1.py

@@ -0,0 +1,20 @@
+from ..hyper_log_log import HyperLogLog
+import sys
+
+def test_hll():
+    hll = HyperLogLog(16) # Initialize HyperLogLog with 16-bit bucket address
+
+    num_elements = 100000
+
+    test2 = []
+    for i in range(num_elements):
+        test2 += [f"word{i}"]
+
+    for item in test2:
+        hll.add(item)
+
+    cardinality = hll.estimate_cardinality()
+
+    print(f"Estimated number of distinct elements: {cardinality}")
+
+    print(f"Actual number of distinct elements: {num_elements}")