sketch-profiler 0.1.0__tar.gz

sketch_profiler-0.1.0/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: sketch-profiler
+Version: 0.1.0
+Summary: Lightweight streaming data profiler using HyperLogLog and Count-Min Sketch
+Author: Jainesh Bharti
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+\# DataProfiler
+
+
+
+Lightweight, framework-agnostic streaming data profiler that estimates dataset statistics without loading the whole dataset into memory.
+
+
+
+\## Why
+
+
+
+Most profiling tools either need a full in-memory pass (pandas `.nunique()`, `.value\_counts()`) or a heavyweight cluster (Spark, DataSketches). DataProfiler works in a single pass, with constant memory, using nothing but plain Python.
+
+
+
+\## How it works
+
+
+
+\- \*\*HyperLogLog\*\* — estimates the number of unique values using probabilistic counting (\~2-5% error, constant memory, regardless of dataset size)
+
+\- \*\*Count-Min Sketch\*\* — estimates how often any given value appears, without storing every value seen
+
+\- Both are combined behind a single `DataProfiler` class, along with null/empty tracking
+
+
+
+\## Usage
+
+
+
+```python
+
+from dataprofiler import DataProfiler
+
+
+
+profiler = DataProfiler()
+
+data = \["a", "b", "a", None, "c", "a", "", "b", "d", "a"]
+
+
+
+for value in data:
+
+    profiler.update(value)
+
+
+
+profiler.report()
+
+```
+
+
+
+Output:
+
+```
+
+Total rows: 10
+
+Null percent: 20.0
+
+Unique estimate: 4
+
+```
+
+
+
+\## Project structure
+
+
+
+```
+
+dataprofiler/
+
+├── hyperloglog.py      # cardinality estimation
+
+├── countminsketch.py   # frequency estimation
+
+├── profiler.py         # combines both into DataProfiler
+
+└── \_\_init\_\_.py
+
+```
+
+
+
+\## Running tests
+
+
+
+```bash
+
+pytest test\_profiler.py -v
+
+```
+

sketch_profiler-0.1.0/README.md

@@ -0,0 +1,100 @@
+\# DataProfiler
+
+
+
+Lightweight, framework-agnostic streaming data profiler that estimates dataset statistics without loading the whole dataset into memory.
+
+
+
+\## Why
+
+
+
+Most profiling tools either need a full in-memory pass (pandas `.nunique()`, `.value\_counts()`) or a heavyweight cluster (Spark, DataSketches). DataProfiler works in a single pass, with constant memory, using nothing but plain Python.
+
+
+
+\## How it works
+
+
+
+\- \*\*HyperLogLog\*\* — estimates the number of unique values using probabilistic counting (\~2-5% error, constant memory, regardless of dataset size)
+
+\- \*\*Count-Min Sketch\*\* — estimates how often any given value appears, without storing every value seen
+
+\- Both are combined behind a single `DataProfiler` class, along with null/empty tracking
+
+
+
+\## Usage
+
+
+
+```python
+
+from dataprofiler import DataProfiler
+
+
+
+profiler = DataProfiler()
+
+data = \["a", "b", "a", None, "c", "a", "", "b", "d", "a"]
+
+
+
+for value in data:
+
+    profiler.update(value)
+
+
+
+profiler.report()
+
+```
+
+
+
+Output:
+
+```
+
+Total rows: 10
+
+Null percent: 20.0
+
+Unique estimate: 4
+
+```
+
+
+
+\## Project structure
+
+
+
+```
+
+dataprofiler/
+
+├── hyperloglog.py      # cardinality estimation
+
+├── countminsketch.py   # frequency estimation
+
+├── profiler.py         # combines both into DataProfiler
+
+└── \_\_init\_\_.py
+
+```
+
+
+
+\## Running tests
+
+
+
+```bash
+
+pytest test\_profiler.py -v
+
+```
+

sketch_profiler-0.1.0/dataprofiler/__init__.py

@@ -0,0 +1 @@
+from .profiler import DataProfiler

sketch_profiler-0.1.0/dataprofiler/countminsketch.py

@@ -0,0 +1,26 @@
+import hashlib
+import random
+
+
+class CountMinSketch:
+    def __init__(self, width=2000, depth=5):
+        self.width = width
+        self.depth = depth
+        self.table = [[0] * width for _ in range(depth)]
+        self.seeds = [random.randint(0, 10**6) for _ in range(depth)]
+
+    def _hash(self, value, seed):
+        h = hashlib.md5(f"{seed}-{value}".encode()).hexdigest()
+        return int(h, 16) % self.width
+
+    def add(self, value):
+        for i in range(self.depth):
+            idx = self._hash(value, self.seeds[i])
+            self.table[i][idx] += 1
+
+    def estimate(self, value):
+        counts = []
+        for i in range(self.depth):
+            idx = self._hash(value, self.seeds[i])
+            counts.append(self.table[i][idx])
+        return min(counts)

sketch_profiler-0.1.0/dataprofiler/hyperloglog.py

@@ -0,0 +1,41 @@
+import hashlib
+import math
+
+
+class HyperLogLog:
+    def __init__(self, b=10):
+        self.b = b
+        self.m = 2 ** b
+        self.buckets = [0] * self.m
+
+    def _hash(self, value):
+        h = hashlib.md5(str(value).encode()).hexdigest()
+        return int(h, 16)
+
+    def _leading_zeros(self, x, bits=118):
+        binary = format(x, f'0{bits}b')
+        count = 0
+        for char in binary:
+            if char == '0':
+                count += 1
+            else:
+                break
+        return count
+
+    def add(self, value):
+        x = self._hash(value)
+        bucket_index = x & (self.m - 1)
+        remaining_bits = x >> self.b
+        rank = self._leading_zeros(remaining_bits, bits=128 - self.b) + 1
+        self.buckets[bucket_index] = max(self.buckets[bucket_index], rank)
+
+    def count(self):
+        alpha = 0.7213 / (1 + 1.079 / self.m)
+        raw_estimate = alpha * (self.m ** 2) / sum(2 ** -b for b in self.buckets)
+
+        if raw_estimate <= 2.5 * self.m:
+            zeros = self.buckets.count(0)
+            if zeros != 0:
+                return int(self.m * math.log(self.m / zeros))
+
+        return int(raw_estimate)

sketch_profiler-0.1.0/dataprofiler/profiler.py

@@ -0,0 +1,24 @@
+from .hyperloglog import HyperLogLog
+from .countminsketch import CountMinSketch
+
+
+class DataProfiler:
+    def __init__(self):
+        self.hll = HyperLogLog(b=10)
+        self.cms = CountMinSketch()
+        self.total_count = 0
+        self.null_count = 0
+
+    def update(self, value):
+        self.total_count += 1
+        if value is None or value == "":
+            self.null_count += 1
+            return
+        self.hll.add(value)
+        self.cms.add(value)
+
+    def report(self):
+        null_pct = (self.null_count / self.total_count) * 100
+        print("Total rows:", self.total_count)
+        print("Null percent:", round(null_pct, 2))
+        print("Unique estimate:", self.hll.count())

sketch_profiler-0.1.0/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sketch-profiler"
+version = "0.1.0"
+description = "Lightweight streaming data profiler using HyperLogLog and Count-Min Sketch"
+readme = "README.md"
+requires-python = ">=3.8"
+authors = [{name = "Jainesh Bharti"}]
+
+[tool.setuptools]
+packages = ["dataprofiler"]

sketch_profiler-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sketch_profiler-0.1.0/sketch_profiler.egg-info/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: sketch-profiler
+Version: 0.1.0
+Summary: Lightweight streaming data profiler using HyperLogLog and Count-Min Sketch
+Author: Jainesh Bharti
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+\# DataProfiler
+
+
+
+Lightweight, framework-agnostic streaming data profiler that estimates dataset statistics without loading the whole dataset into memory.
+
+
+
+\## Why
+
+
+
+Most profiling tools either need a full in-memory pass (pandas `.nunique()`, `.value\_counts()`) or a heavyweight cluster (Spark, DataSketches). DataProfiler works in a single pass, with constant memory, using nothing but plain Python.
+
+
+
+\## How it works
+
+
+
+\- \*\*HyperLogLog\*\* — estimates the number of unique values using probabilistic counting (\~2-5% error, constant memory, regardless of dataset size)
+
+\- \*\*Count-Min Sketch\*\* — estimates how often any given value appears, without storing every value seen
+
+\- Both are combined behind a single `DataProfiler` class, along with null/empty tracking
+
+
+
+\## Usage
+
+
+
+```python
+
+from dataprofiler import DataProfiler
+
+
+
+profiler = DataProfiler()
+
+data = \["a", "b", "a", None, "c", "a", "", "b", "d", "a"]
+
+
+
+for value in data:
+
+    profiler.update(value)
+
+
+
+profiler.report()
+
+```
+
+
+
+Output:
+
+```
+
+Total rows: 10
+
+Null percent: 20.0
+
+Unique estimate: 4
+
+```
+
+
+
+\## Project structure
+
+
+
+```
+
+dataprofiler/
+
+├── hyperloglog.py      # cardinality estimation
+
+├── countminsketch.py   # frequency estimation
+
+├── profiler.py         # combines both into DataProfiler
+
+└── \_\_init\_\_.py
+
+```
+
+
+
+\## Running tests
+
+
+
+```bash
+
+pytest test\_profiler.py -v
+
+```
+

sketch_profiler-0.1.0/sketch_profiler.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+pyproject.toml
+dataprofiler/__init__.py
+dataprofiler/countminsketch.py
+dataprofiler/hyperloglog.py
+dataprofiler/profiler.py
+sketch_profiler.egg-info/PKG-INFO
+sketch_profiler.egg-info/SOURCES.txt
+sketch_profiler.egg-info/dependency_links.txt
+sketch_profiler.egg-info/top_level.txt

sketch_profiler-0.1.0/sketch_profiler.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

sketch_profiler-0.1.0/sketch_profiler.egg-info/top_level.txt

@@ -0,0 +1 @@
+dataprofiler