data-hacks3 0.0.4__py3-none-any.whl

data_hacks/bar_chart.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Generate an ascii bar chart for input data
+
+https://github.com/bitly/data_hacks
+"""
+import sys
+import math
+from collections import defaultdict
+from optparse import OptionParser
+from decimal import Decimal
+
+def load_stream(input_stream):
+    for line in input_stream:
+        clean_line = line.strip()
+        if not clean_line:
+            # skip empty lines (ie: newlines)
+            continue
+        if clean_line[0] in ['"', "'"]:
+            clean_line = clean_line.strip('"').strip("'")
+        if clean_line:
+            yield clean_line
+
+def run(input_stream, options):
+    data = defaultdict(int)
+    total = 0
+    for row in input_stream:
+        if options.agg_key_value:
+            kv = row.rstrip().rsplit(None, 1)
+            value = int(kv[1])
+            data[kv[0]] += value
+            total += value
+        elif options.agg_value_key:
+            kv = row.lstrip().split(None, 1)
+            value = int(kv[0])
+            data[kv[1]] += value
+            total += value
+        else:
+            data[row] += 1
+            total += 1
+    
+    if not data:
+        print("Error: no data")
+        sys.exit(1)
+    
+    max_length = max([len(key) for key in list(data.keys())])
+    max_length = min(max_length, 50)
+    value_characters = 80 - max_length
+    max_value = max(data.values())
+    scale = int(math.ceil(float(max_value) / value_characters))
+    scale = max(1, scale)
+    
+    print(("# each " + options.dot + " represents a count of %d. total %d" % (scale, total)))
+    
+    if options.sort_values:
+        data = [[value, key] for key, value in list(data.items())]
+        data.sort(key=lambda x: x[0], reverse=options.reverse_sort)
+    else:
+        # sort by keys
+        data = [[value, key] for key, value in list(data.items())]
+        if options.numeric_sort:
+            # keys could be numeric too
+            data.sort(key=lambda x: (Decimal(x[1])), reverse=options.reverse_sort)
+        else:
+            data.sort(key=lambda x: x[1], reverse=options.reverse_sort)
+    
+    str_format = "%" + str(max_length) + "s [%6d] %s%s"
+    percentage = ""
+    for value, key in data:
+        if options.percentage:
+            percentage = " (%0.2f%%)" % (100 * Decimal(value) / Decimal(total))
+        print((str_format % (key[:max_length], value, int(value / scale) * options.dot, percentage)))
+
+if __name__ == "__main__":
+    parser = OptionParser()
+    parser.usage = "cat data | %prog [options]"
+    parser.add_option("-a", "--agg", dest="agg_value_key", default=False, action="store_true",
+                        help="Two column input format, space seperated with value<space>key")
+    parser.add_option("-A", "--agg-key-value", dest="agg_key_value", default=False, action="store_true",
+                        help="Two column input format, space seperated with key<space>value")
+    parser.add_option("-k", "--sort-keys", dest="sort_keys", default=True, action="store_true",
+                        help="sort by the key [default]")
+    parser.add_option("-v", "--sort-values", dest="sort_values", default=False, action="store_true",
+                        help="sort by the frequence")
+    parser.add_option("-r", "--reverse-sort", dest="reverse_sort", default=False, action="store_true",
+                        help="reverse the sort")
+    parser.add_option("-n", "--numeric-sort", dest="numeric_sort", default=False, action="store_true",
+                        help="sort keys by numeric sequencing")
+    parser.add_option("-p", "--percentage", dest="percentage", default=False, action="store_true",
+                        help="List percentage for each bar")
+    parser.add_option("--dot", dest="dot", default='∎', help="Dot representation")
+
+    (options, args) = parser.parse_args()
+    
+    if sys.stdin.isatty():
+        parser.print_usage()
+        print("for more help use --help")
+        sys.exit(1)
+    run(load_stream(sys.stdin), options)
+

data_hacks/histogram.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Generate a text format histogram
+
+This is a loose port to python of the Perl version at
+http://www.pandamatak.com/people/anand/xfer/histo
+
+https://github.com/bitly/data_hacks
+"""
+
+import sys
+from decimal import Decimal
+import logging
+import math
+from optparse import OptionParser
+from collections import namedtuple
+
+
+class MVSD(object):
+    "A class that calculates a running Mean / Variance / Standard Deviation"
+    def __init__(self):
+        self.is_started = False
+        self.ss = Decimal(0)  # (running) sum of square deviations from mean
+        self.m = Decimal(0)  # (running) mean
+        self.total_w = Decimal(0)  # weight of items seen
+
+    def add(self, x, w=1):
+        "add another datapoint to the Mean / Variance / Standard Deviation"
+        if not isinstance(x, Decimal):
+            x = Decimal(x)
+        if not self.is_started:
+            self.m = x
+            self.ss = Decimal(0)
+            self.total_w = w
+            self.is_started = True
+        else:
+            temp_w = self.total_w + w
+            self.ss += (self.total_w * w * (x - self.m) *
+                        (x - self.m)) / temp_w
+            self.m += (x - self.m) / temp_w
+            self.total_w = temp_w
+
+    def var(self):
+        return self.ss / self.total_w
+
+    def sd(self):
+        return math.sqrt(self.var())
+
+    def mean(self):
+        return self.m
+
+DataPoint = namedtuple('DataPoint', ['value', 'count'])
+
+
+def test_mvsd():
+    mvsd = MVSD()
+    for x in range(10):
+        mvsd.add(x)
+
+    assert '%.2f' % mvsd.mean() == "4.50"
+    assert '%.2f' % mvsd.var() == "8.25"
+    assert '%.14f' % mvsd.sd() == "2.87228132326901"
+
+
+def load_stream(input_stream, agg_value_key, agg_key_value):
+    for line in input_stream:
+        clean_line = line.strip()
+        if not clean_line:
+            # skip empty lines (ie: newlines)
+            continue
+        if clean_line[0] in ['"', "'"]:
+            clean_line = clean_line.strip("\"'")
+        try:
+            if agg_key_value:
+                key, value = clean_line.rstrip().rsplit(None, 1)
+                yield DataPoint(Decimal(key), int(value))
+            elif agg_value_key:
+                value, key = clean_line.lstrip().split(None, 1)
+                yield DataPoint(Decimal(key), int(value))
+            else:
+                yield DataPoint(Decimal(clean_line), 1)
+        except:
+            logging.exception('failed %r', line)
+            print("invalid line %r" % line, file=sys.stderr)
+
+
+def median(values, key=None):
+    if not key:
+        key = lambda x: x  # identity; py3 map() does not accept None
+    length = len(values)
+    if length % 2:
+        median_indeces = [int(length/2)]
+    else:
+        median_indeces = [int(length/2)-1, int(length/2)]
+
+    values = sorted(values, key=key)
+    return sum(map(key,
+                   [values[i] for i in median_indeces])) / len(median_indeces)
+
+
+def test_median():
+    assert 6 == median([8, 7, 9, 1, 2, 6, 3])  # odd-sized list
+    assert 4.5 == median([4, 5, 2, 1, 9, 10])  # even-sized int list. (4+5)/2 = 4.5
+    # even-sized float list. (4.0+5)/2 = 4.5
+    assert "4.50" == "%.2f" % median([4.0, 5, 2, 1, 9, 10])
+
+
+def histogram(stream, options):
+    """
+    Loop over the stream and add each entry to the dataset, printing out at the
+    end.
+
+    stream yields Decimal()
+    """
+    if not options.min or not options.max:
+        # glob the iterator here so we can do min/max on it
+        data = list(stream)
+    else:
+        data = stream
+    bucket_scale = 1
+
+    if options.min:
+        min_v = Decimal(options.min)
+    else:
+        min_v = min(data, key=lambda x: x.value)
+        min_v = min_v.value
+    if options.max:
+        max_v = Decimal(options.max)
+    else:
+        max_v = max(data, key=lambda x: x.value)
+        max_v = max_v.value
+
+    if not max_v > min_v:
+        raise ValueError('max must be > min. max:%s min:%s' % (max_v, min_v))
+    diff = max_v - min_v
+
+    boundaries = []
+    bucket_counts = []
+    buckets = 0
+
+    if options.custbuckets:
+        bound = options.custbuckets.split(',')
+        bound_sort = sorted(map(Decimal, bound))
+
+        # if the last value is smaller than the maximum, replace it
+        if bound_sort[-1] < max_v:
+            bound_sort[-1] = max_v
+
+        # iterate through the sorted list and append to boundaries
+        for x in bound_sort:
+            if x >= min_v and x <= max_v:
+                boundaries.append(x)
+            elif x >= max_v:
+                boundaries.append(max_v)
+                break
+
+        # beware: the min_v is not included in the boundaries,
+        # so no need to do a -1!
+        bucket_counts = [0 for x in range(len(boundaries))]
+        buckets = len(boundaries)
+    elif options.logscale:
+        buckets = options.buckets and int(options.buckets) or 10
+        if buckets <= 0:
+            raise ValueError('# of buckets must be > 0')
+
+        def first_bucket_size(k, n):
+            r"""Logarithmic buckets means, the size of bucket i+1 is twice
+            the size of bucket i.
+            For k+1 buckets whose sum is n, we have
+            (note, k+1 buckets, since 0 is counted as well):
+                \sum_{i=0}^{k} x*2^i   = n
+                x * \sum_{i=0}^{k} 2^i = n
+                x * (2^{k+1} - 1)      = n
+                x = n/(2^{k+1} - 1)
+            """
+            return n/(2**(k+1)-1)
+
+        def log_steps(k, n):
+            "k logarithmic steps whose sum is n"
+            x = first_bucket_size(k-1, n)
+            sum = 0
+            for i in range(k):
+                sum += 2**i * x
+                yield sum
+        bucket_counts = [0 for x in range(buckets)]
+        for step in log_steps(buckets, diff):
+            boundaries.append(min_v + step)
+    else:
+        buckets = options.buckets and int(options.buckets) or 10
+        if buckets <= 0:
+            raise ValueError('# of buckets must be > 0')
+        step = diff / buckets
+        bucket_counts = [0 for x in range(buckets)]
+        for x in range(buckets):
+            boundaries.append(min_v + (step * (x + 1)))
+
+    skipped = 0
+    samples = 0
+    mvsd = MVSD()
+    accepted_data = []
+    for record in data:
+        samples += record.count
+        if options.mvsd:
+            mvsd.add(record.value, record.count)
+            accepted_data.append(record)
+        # find the bucket this goes in
+        if record.value < min_v or record.value > max_v:
+            skipped += record.count
+            continue
+        for bucket_postion, boundary in enumerate(boundaries):
+            if record.value <= boundary:
+                bucket_counts[bucket_postion] += record.count
+                break
+
+    # auto-pick the hash scale
+    if max(bucket_counts) > 75:
+        bucket_scale = int(max(bucket_counts) / 75)
+
+    print(("# NumSamples = %d; Min = %0.2f; Max = %0.2f" %
+          (samples, min_v, max_v)))
+    if skipped:
+        print(("# %d value%s outside of min/max" %
+              (skipped, skipped > 1 and 's' or '')))
+    if options.mvsd:
+        print(("# Mean = %f; Variance = %f; SD = %f; Median %f" %
+              (mvsd.mean(), mvsd.var(), mvsd.sd(),
+               median(accepted_data, key=lambda x: x.value))))
+    print(("# each " + options.dot + " represents a count of %d" % bucket_scale))
+    
+    bucket_min = min_v
+    bucket_max = min_v
+    percentage = ""
+    format_string = options.format + ' - ' + options.format + ' [%6d]: %s%s'
+    for bucket in range(buckets):
+        bucket_min = bucket_max
+        bucket_max = boundaries[bucket]
+        bucket_count = bucket_counts[bucket]
+        star_count = 0
+        if bucket_count:
+            star_count = int(bucket_count / bucket_scale)
+        if options.percentage:
+            percentage = " (%0.2f%%)" % (100 * Decimal(bucket_count) /
+                                         Decimal(samples))
+        print((format_string % (bucket_min, bucket_max, bucket_count, options.dot * star_count, percentage)))
+
+
+if __name__ == "__main__":
+    parser = OptionParser()
+    parser.usage = "cat data | %prog [options]"
+    parser.add_option("-a", "--agg", dest="agg_value_key", default=False,
+                      action="store_true", help="Two column input format, " +
+                      "space seperated with value<space>key")
+    parser.add_option("-A", "--agg-key-value", dest="agg_key_value",
+                      default=False, action="store_true", help="Two column " +
+                      "input format, space seperated with key<space>value")
+    parser.add_option("-m", "--min", dest="min",
+                      help="minimum value for graph")
+    parser.add_option("-x", "--max", dest="max",
+                      help="maximum value for graph")
+    parser.add_option("-b", "--buckets", dest="buckets",
+                      help="Number of buckets to use for the histogram")
+    parser.add_option("-l", "--logscale", dest="logscale", default=False,
+                      action="store_true",
+                      help="Buckets grow in logarithmic scale")
+    parser.add_option("-B", "--custom-buckets", dest="custbuckets",
+                      help="Comma seperated list of bucket " +
+                      "edges for the histogram")
+    parser.add_option("--no-mvsd", dest="mvsd", action="store_false",
+                      default=True, help="Disable the calculation of Mean, " +
+                      "Variance and SD (improves performance)")
+    parser.add_option("-f", "--bucket-format", dest="format", default="%10.4f",
+                      help="format for bucket numbers")
+    parser.add_option("-p", "--percentage", dest="percentage", default=False,
+                      action="store_true", help="List percentage for each bar")
+    parser.add_option("--dot", dest="dot", default='∎', help="Dot representation")
+
+    (options, args) = parser.parse_args()
+    if sys.stdin.isatty():
+        # if isatty() that means it's run without anything piped into it
+        parser.print_usage()
+        print("for more help use --help")
+        sys.exit(1)
+    histogram(load_stream(sys.stdin, options.agg_value_key,
+                          options.agg_key_value), options)

data_hacks/ninety_five_percent.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+#
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Calculate the 95% time from a list of times given on stdin
+
+https://github.com/bitly/data_hacks
+"""
+
+import sys
+import os
+from decimal import Decimal
+
+def run():
+    count = 0
+    data = {}
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            # skip empty lines (ie: newlines)
+            continue
+        try:
+            t = Decimal(line)
+            count +=1
+            data[t] = data.get(t, 0) + 1
+        except:
+            print("invalid line %r" % line, file=sys.stderr)
+    print(calc_95(data, count))
+        
+def calc_95(data, count):
+    # find the time it took for x entry, where x is the threshold
+    threshold = Decimal(count) * Decimal('.95')
+    start = Decimal(0)
+    times = list(data.keys())
+    times.sort()
+    for t in times:
+        # increment our count by the # of items in this time bucket
+        start += data[t]
+        if start > threshold:
+            return t
+
+if __name__ == "__main__":
+    if sys.stdin.isatty() or '--help' in sys.argv or '-h' in sys.argv:
+        print("Usage: cat data | %s" % os.path.basename(sys.argv[0]))
+        sys.exit(1)
+    run()

data_hacks/run_for.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# 
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Pass through data for a specified amount of time
+
+https://github.com/bitly/data_hacks
+"""
+
+import time
+import sys
+import os
+
+def getruntime(arg):
+    if not arg:
+        return
+    suffix = arg[-1]
+    base = int(arg[:-1])
+    if suffix == "s":
+        return base
+    elif suffix == "m":
+        return base * 60
+    elif suffix == "h":
+        return base * 60 * 60
+    elif suffix == "d":
+        return base * 60 * 60 * 24
+    else:
+        print("invalid time suffix %r. must be one of s,m,h,d" % arg, file=sys.stderr)
+
+def run(runtime):
+    end = time.time() + runtime
+    for line in sys.stdin:
+        sys.stdout.write(line)
+        if time.time() > end:
+            return
+
+if __name__ == "__main__":
+    usage = "Usage: tail -f access.log | %s [time] | ..." % os.path.basename(sys.argv[0])
+    help = "time can be in the format 10s, 10m, 10h, etc"
+    if sys.stdin.isatty():
+        print(usage)
+        print(help)
+        sys.exit(1)
+
+    runtime = getruntime(sys.argv[-1])
+    if not runtime:
+        print(usage)
+        sys.exit(1)
+    run(runtime)

data_hacks/sample.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# 
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Pass through a sampled percentage of data
+
+https://github.com/bitly/data_hacks
+"""
+
+import sys
+import random
+from optparse import OptionParser
+from decimal import Decimal
+
+def run(sample_rate):
+    input_stream = sys.stdin
+    for line in input_stream:
+        if random.randint(1,100) <= sample_rate:
+            sys.stdout.write(line)
+
+def get_sample_rate(rate_string):
+    """ return a rate as a percentage"""
+    if rate_string.endswith("%"):
+        rate = int(rate_string[:-1])
+    elif '/' in rate_string:
+        x, y  = rate_string.split('/')
+        rate = Decimal(x) / (Decimal(y) * Decimal('1.0'))
+        rate = int(rate * 100)
+    else:
+        raise ValueError("rate %r is invalid rate format must be '10%%' or '1/10'" % rate_string)
+    if rate < 1 or rate > 100:
+        raise ValueError('rate %r must be 1%% <= rate <= 100%% ' % rate_string)
+    return rate
+
+if __name__ == "__main__":
+    parser = OptionParser(usage="cat data | %prog [options] [sample_rate]")
+    parser.add_option("--verbose", dest="verbose", default=False, action="store_true")
+    (options, args) = parser.parse_args()
+    
+    if not args or sys.stdin.isatty():
+        parser.print_usage()
+        sys.exit(1)
+    
+    try:
+        sample_rate = get_sample_rate(sys.argv[-1])
+    except ValueError as e:
+        print(e, file=sys.stderr)
+        parser.print_usage()
+        sys.exit(1)
+    if options.verbose:
+        print("Sample rate is %d%%" % sample_rate, file=sys.stderr) 
+    run(sample_rate)

data_hacks3-0.0.4.data/scripts/bar_chart.py

@@ -0,0 +1,116 @@
+#!python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Generate an ascii bar chart for input data
+
+https://github.com/bitly/data_hacks
+"""
+import sys
+import math
+from collections import defaultdict
+from optparse import OptionParser
+from decimal import Decimal
+
+def load_stream(input_stream):
+    for line in input_stream:
+        clean_line = line.strip()
+        if not clean_line:
+            # skip empty lines (ie: newlines)
+            continue
+        if clean_line[0] in ['"', "'"]:
+            clean_line = clean_line.strip('"').strip("'")
+        if clean_line:
+            yield clean_line
+
+def run(input_stream, options):
+    data = defaultdict(int)
+    total = 0
+    for row in input_stream:
+        if options.agg_key_value:
+            kv = row.rstrip().rsplit(None, 1)
+            value = int(kv[1])
+            data[kv[0]] += value
+            total += value
+        elif options.agg_value_key:
+            kv = row.lstrip().split(None, 1)
+            value = int(kv[0])
+            data[kv[1]] += value
+            total += value
+        else:
+            data[row] += 1
+            total += 1
+    
+    if not data:
+        print("Error: no data")
+        sys.exit(1)
+    
+    max_length = max([len(key) for key in list(data.keys())])
+    max_length = min(max_length, 50)
+    value_characters = 80 - max_length
+    max_value = max(data.values())
+    scale = int(math.ceil(float(max_value) / value_characters))
+    scale = max(1, scale)
+    
+    print(("# each " + options.dot + " represents a count of %d. total %d" % (scale, total)))
+    
+    if options.sort_values:
+        data = [[value, key] for key, value in list(data.items())]
+        data.sort(key=lambda x: x[0], reverse=options.reverse_sort)
+    else:
+        # sort by keys
+        data = [[value, key] for key, value in list(data.items())]
+        if options.numeric_sort:
+            # keys could be numeric too
+            data.sort(key=lambda x: (Decimal(x[1])), reverse=options.reverse_sort)
+        else:
+            data.sort(key=lambda x: x[1], reverse=options.reverse_sort)
+    
+    str_format = "%" + str(max_length) + "s [%6d] %s%s"
+    percentage = ""
+    for value, key in data:
+        if options.percentage:
+            percentage = " (%0.2f%%)" % (100 * Decimal(value) / Decimal(total))
+        print((str_format % (key[:max_length], value, int(value / scale) * options.dot, percentage)))
+
+if __name__ == "__main__":
+    parser = OptionParser()
+    parser.usage = "cat data | %prog [options]"
+    parser.add_option("-a", "--agg", dest="agg_value_key", default=False, action="store_true",
+                        help="Two column input format, space seperated with value<space>key")
+    parser.add_option("-A", "--agg-key-value", dest="agg_key_value", default=False, action="store_true",
+                        help="Two column input format, space seperated with key<space>value")
+    parser.add_option("-k", "--sort-keys", dest="sort_keys", default=True, action="store_true",
+                        help="sort by the key [default]")
+    parser.add_option("-v", "--sort-values", dest="sort_values", default=False, action="store_true",
+                        help="sort by the frequence")
+    parser.add_option("-r", "--reverse-sort", dest="reverse_sort", default=False, action="store_true",
+                        help="reverse the sort")
+    parser.add_option("-n", "--numeric-sort", dest="numeric_sort", default=False, action="store_true",
+                        help="sort keys by numeric sequencing")
+    parser.add_option("-p", "--percentage", dest="percentage", default=False, action="store_true",
+                        help="List percentage for each bar")
+    parser.add_option("--dot", dest="dot", default='∎', help="Dot representation")
+
+    (options, args) = parser.parse_args()
+    
+    if sys.stdin.isatty():
+        parser.print_usage()
+        print("for more help use --help")
+        sys.exit(1)
+    run(load_stream(sys.stdin), options)
+

data_hacks3-0.0.4.data/scripts/histogram.py

@@ -0,0 +1,300 @@
+#!python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Generate a text format histogram
+
+This is a loose port to python of the Perl version at
+http://www.pandamatak.com/people/anand/xfer/histo
+
+https://github.com/bitly/data_hacks
+"""
+
+import sys
+from decimal import Decimal
+import logging
+import math
+from optparse import OptionParser
+from collections import namedtuple
+
+
+class MVSD(object):
+    "A class that calculates a running Mean / Variance / Standard Deviation"
+    def __init__(self):
+        self.is_started = False
+        self.ss = Decimal(0)  # (running) sum of square deviations from mean
+        self.m = Decimal(0)  # (running) mean
+        self.total_w = Decimal(0)  # weight of items seen
+
+    def add(self, x, w=1):
+        "add another datapoint to the Mean / Variance / Standard Deviation"
+        if not isinstance(x, Decimal):
+            x = Decimal(x)
+        if not self.is_started:
+            self.m = x
+            self.ss = Decimal(0)
+            self.total_w = w
+            self.is_started = True
+        else:
+            temp_w = self.total_w + w
+            self.ss += (self.total_w * w * (x - self.m) *
+                        (x - self.m)) / temp_w
+            self.m += (x - self.m) / temp_w
+            self.total_w = temp_w
+
+    def var(self):
+        return self.ss / self.total_w
+
+    def sd(self):
+        return math.sqrt(self.var())
+
+    def mean(self):
+        return self.m
+
+DataPoint = namedtuple('DataPoint', ['value', 'count'])
+
+
+def test_mvsd():
+    mvsd = MVSD()
+    for x in range(10):
+        mvsd.add(x)
+
+    assert '%.2f' % mvsd.mean() == "4.50"
+    assert '%.2f' % mvsd.var() == "8.25"
+    assert '%.14f' % mvsd.sd() == "2.87228132326901"
+
+
+def load_stream(input_stream, agg_value_key, agg_key_value):
+    for line in input_stream:
+        clean_line = line.strip()
+        if not clean_line:
+            # skip empty lines (ie: newlines)
+            continue
+        if clean_line[0] in ['"', "'"]:
+            clean_line = clean_line.strip("\"'")
+        try:
+            if agg_key_value:
+                key, value = clean_line.rstrip().rsplit(None, 1)
+                yield DataPoint(Decimal(key), int(value))
+            elif agg_value_key:
+                value, key = clean_line.lstrip().split(None, 1)
+                yield DataPoint(Decimal(key), int(value))
+            else:
+                yield DataPoint(Decimal(clean_line), 1)
+        except:
+            logging.exception('failed %r', line)
+            print("invalid line %r" % line, file=sys.stderr)
+
+
+def median(values, key=None):
+    if not key:
+        key = lambda x: x  # identity; py3 map() does not accept None
+    length = len(values)
+    if length % 2:
+        median_indeces = [int(length/2)]
+    else:
+        median_indeces = [int(length/2)-1, int(length/2)]
+
+    values = sorted(values, key=key)
+    return sum(map(key,
+                   [values[i] for i in median_indeces])) / len(median_indeces)
+
+
+def test_median():
+    assert 6 == median([8, 7, 9, 1, 2, 6, 3])  # odd-sized list
+    assert 4.5 == median([4, 5, 2, 1, 9, 10])  # even-sized int list. (4+5)/2 = 4.5
+    # even-sized float list. (4.0+5)/2 = 4.5
+    assert "4.50" == "%.2f" % median([4.0, 5, 2, 1, 9, 10])
+
+
+def histogram(stream, options):
+    """
+    Loop over the stream and add each entry to the dataset, printing out at the
+    end.
+
+    stream yields Decimal()
+    """
+    if not options.min or not options.max:
+        # glob the iterator here so we can do min/max on it
+        data = list(stream)
+    else:
+        data = stream
+    bucket_scale = 1
+
+    if options.min:
+        min_v = Decimal(options.min)
+    else:
+        min_v = min(data, key=lambda x: x.value)
+        min_v = min_v.value
+    if options.max:
+        max_v = Decimal(options.max)
+    else:
+        max_v = max(data, key=lambda x: x.value)
+        max_v = max_v.value
+
+    if not max_v > min_v:
+        raise ValueError('max must be > min. max:%s min:%s' % (max_v, min_v))
+    diff = max_v - min_v
+
+    boundaries = []
+    bucket_counts = []
+    buckets = 0
+
+    if options.custbuckets:
+        bound = options.custbuckets.split(',')
+        bound_sort = sorted(map(Decimal, bound))
+
+        # if the last value is smaller than the maximum, replace it
+        if bound_sort[-1] < max_v:
+            bound_sort[-1] = max_v
+
+        # iterate through the sorted list and append to boundaries
+        for x in bound_sort:
+            if x >= min_v and x <= max_v:
+                boundaries.append(x)
+            elif x >= max_v:
+                boundaries.append(max_v)
+                break
+
+        # beware: the min_v is not included in the boundaries,
+        # so no need to do a -1!
+        bucket_counts = [0 for x in range(len(boundaries))]
+        buckets = len(boundaries)
+    elif options.logscale:
+        buckets = options.buckets and int(options.buckets) or 10
+        if buckets <= 0:
+            raise ValueError('# of buckets must be > 0')
+
+        def first_bucket_size(k, n):
+            r"""Logarithmic buckets means, the size of bucket i+1 is twice
+            the size of bucket i.
+            For k+1 buckets whose sum is n, we have
+            (note, k+1 buckets, since 0 is counted as well):
+                \sum_{i=0}^{k} x*2^i   = n
+                x * \sum_{i=0}^{k} 2^i = n
+                x * (2^{k+1} - 1)      = n
+                x = n/(2^{k+1} - 1)
+            """
+            return n/(2**(k+1)-1)
+
+        def log_steps(k, n):
+            "k logarithmic steps whose sum is n"
+            x = first_bucket_size(k-1, n)
+            sum = 0
+            for i in range(k):
+                sum += 2**i * x
+                yield sum
+        bucket_counts = [0 for x in range(buckets)]
+        for step in log_steps(buckets, diff):
+            boundaries.append(min_v + step)
+    else:
+        buckets = options.buckets and int(options.buckets) or 10
+        if buckets <= 0:
+            raise ValueError('# of buckets must be > 0')
+        step = diff / buckets
+        bucket_counts = [0 for x in range(buckets)]
+        for x in range(buckets):
+            boundaries.append(min_v + (step * (x + 1)))
+
+    skipped = 0
+    samples = 0
+    mvsd = MVSD()
+    accepted_data = []
+    for record in data:
+        samples += record.count
+        if options.mvsd:
+            mvsd.add(record.value, record.count)
+            accepted_data.append(record)
+        # find the bucket this goes in
+        if record.value < min_v or record.value > max_v:
+            skipped += record.count
+            continue
+        for bucket_postion, boundary in enumerate(boundaries):
+            if record.value <= boundary:
+                bucket_counts[bucket_postion] += record.count
+                break
+
+    # auto-pick the hash scale
+    if max(bucket_counts) > 75:
+        bucket_scale = int(max(bucket_counts) / 75)
+
+    print(("# NumSamples = %d; Min = %0.2f; Max = %0.2f" %
+          (samples, min_v, max_v)))
+    if skipped:
+        print(("# %d value%s outside of min/max" %
+              (skipped, skipped > 1 and 's' or '')))
+    if options.mvsd:
+        print(("# Mean = %f; Variance = %f; SD = %f; Median %f" %
+              (mvsd.mean(), mvsd.var(), mvsd.sd(),
+               median(accepted_data, key=lambda x: x.value))))
+    print(("# each " + options.dot + " represents a count of %d" % bucket_scale))
+    
+    bucket_min = min_v
+    bucket_max = min_v
+    percentage = ""
+    format_string = options.format + ' - ' + options.format + ' [%6d]: %s%s'
+    for bucket in range(buckets):
+        bucket_min = bucket_max
+        bucket_max = boundaries[bucket]
+        bucket_count = bucket_counts[bucket]
+        star_count = 0
+        if bucket_count:
+            star_count = int(bucket_count / bucket_scale)
+        if options.percentage:
+            percentage = " (%0.2f%%)" % (100 * Decimal(bucket_count) /
+                                         Decimal(samples))
+        print((format_string % (bucket_min, bucket_max, bucket_count, options.dot * star_count, percentage)))
+
+
+if __name__ == "__main__":
+    parser = OptionParser()
+    parser.usage = "cat data | %prog [options]"
+    parser.add_option("-a", "--agg", dest="agg_value_key", default=False,
+                      action="store_true", help="Two column input format, " +
+                      "space seperated with value<space>key")
+    parser.add_option("-A", "--agg-key-value", dest="agg_key_value",
+                      default=False, action="store_true", help="Two column " +
+                      "input format, space seperated with key<space>value")
+    parser.add_option("-m", "--min", dest="min",
+                      help="minimum value for graph")
+    parser.add_option("-x", "--max", dest="max",
+                      help="maximum value for graph")
+    parser.add_option("-b", "--buckets", dest="buckets",
+                      help="Number of buckets to use for the histogram")
+    parser.add_option("-l", "--logscale", dest="logscale", default=False,
+                      action="store_true",
+                      help="Buckets grow in logarithmic scale")
+    parser.add_option("-B", "--custom-buckets", dest="custbuckets",
+                      help="Comma seperated list of bucket " +
+                      "edges for the histogram")
+    parser.add_option("--no-mvsd", dest="mvsd", action="store_false",
+                      default=True, help="Disable the calculation of Mean, " +
+                      "Variance and SD (improves performance)")
+    parser.add_option("-f", "--bucket-format", dest="format", default="%10.4f",
+                      help="format for bucket numbers")
+    parser.add_option("-p", "--percentage", dest="percentage", default=False,
+                      action="store_true", help="List percentage for each bar")
+    parser.add_option("--dot", dest="dot", default='∎', help="Dot representation")
+
+    (options, args) = parser.parse_args()
+    if sys.stdin.isatty():
+        # if isatty() that means it's run without anything piped into it
+        parser.print_usage()
+        print("for more help use --help")
+        sys.exit(1)
+    histogram(load_stream(sys.stdin, options.agg_value_key,
+                          options.agg_key_value), options)

data_hacks3-0.0.4.data/scripts/ninety_five_percent.py

@@ -0,0 +1,59 @@
+#!python
+#
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Calculate the 95% time from a list of times given on stdin
+
+https://github.com/bitly/data_hacks
+"""
+
+import sys
+import os
+from decimal import Decimal
+
+def run():
+    count = 0
+    data = {}
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            # skip empty lines (ie: newlines)
+            continue
+        try:
+            t = Decimal(line)
+            count +=1
+            data[t] = data.get(t, 0) + 1
+        except:
+            print("invalid line %r" % line, file=sys.stderr)
+    print(calc_95(data, count))
+        
+def calc_95(data, count):
+    # find the time it took for x entry, where x is the threshold
+    threshold = Decimal(count) * Decimal('.95')
+    start = Decimal(0)
+    times = list(data.keys())
+    times.sort()
+    for t in times:
+        # increment our count by the # of items in this time bucket
+        start += data[t]
+        if start > threshold:
+            return t
+
+if __name__ == "__main__":
+    if sys.stdin.isatty() or '--help' in sys.argv or '-h' in sys.argv:
+        print("Usage: cat data | %s" % os.path.basename(sys.argv[0]))
+        sys.exit(1)
+    run()

data_hacks3-0.0.4.data/scripts/run_for.py

@@ -0,0 +1,62 @@
+#!python
+# 
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Pass through data for a specified amount of time
+
+https://github.com/bitly/data_hacks
+"""
+
+import time
+import sys
+import os
+
+def getruntime(arg):
+    if not arg:
+        return
+    suffix = arg[-1]
+    base = int(arg[:-1])
+    if suffix == "s":
+        return base
+    elif suffix == "m":
+        return base * 60
+    elif suffix == "h":
+        return base * 60 * 60
+    elif suffix == "d":
+        return base * 60 * 60 * 24
+    else:
+        print("invalid time suffix %r. must be one of s,m,h,d" % arg, file=sys.stderr)
+
+def run(runtime):
+    end = time.time() + runtime
+    for line in sys.stdin:
+        sys.stdout.write(line)
+        if time.time() > end:
+            return
+
+if __name__ == "__main__":
+    usage = "Usage: tail -f access.log | %s [time] | ..." % os.path.basename(sys.argv[0])
+    help = "time can be in the format 10s, 10m, 10h, etc"
+    if sys.stdin.isatty():
+        print(usage)
+        print(help)
+        sys.exit(1)
+
+    runtime = getruntime(sys.argv[-1])
+    if not runtime:
+        print(usage)
+        sys.exit(1)
+    run(runtime)

data_hacks3-0.0.4.data/scripts/sample.py

@@ -0,0 +1,65 @@
+#!python
+# 
+# Copyright 2010 Bitly
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Pass through a sampled percentage of data
+
+https://github.com/bitly/data_hacks
+"""
+
+import sys
+import random
+from optparse import OptionParser
+from decimal import Decimal
+
+def run(sample_rate):
+    input_stream = sys.stdin
+    for line in input_stream:
+        if random.randint(1,100) <= sample_rate:
+            sys.stdout.write(line)
+
+def get_sample_rate(rate_string):
+    """ return a rate as a percentage"""
+    if rate_string.endswith("%"):
+        rate = int(rate_string[:-1])
+    elif '/' in rate_string:
+        x, y  = rate_string.split('/')
+        rate = Decimal(x) / (Decimal(y) * Decimal('1.0'))
+        rate = int(rate * 100)
+    else:
+        raise ValueError("rate %r is invalid rate format must be '10%%' or '1/10'" % rate_string)
+    if rate < 1 or rate > 100:
+        raise ValueError('rate %r must be 1%% <= rate <= 100%% ' % rate_string)
+    return rate
+
+if __name__ == "__main__":
+    parser = OptionParser(usage="cat data | %prog [options] [sample_rate]")
+    parser.add_option("--verbose", dest="verbose", default=False, action="store_true")
+    (options, args) = parser.parse_args()
+    
+    if not args or sys.stdin.isatty():
+        parser.print_usage()
+        sys.exit(1)
+    
+    try:
+        sample_rate = get_sample_rate(sys.argv[-1])
+    except ValueError as e:
+        print(e, file=sys.stderr)
+        parser.print_usage()
+        sys.exit(1)
+    if options.verbose:
+        print("Sample rate is %d%%" % sample_rate, file=sys.stderr) 
+    run(sample_rate)

data_hacks3-0.0.4.dist-info/METADATA

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: data_hacks3
+Version: 0.0.4
+Summary: Command line utilities for data analysis
+Home-page: https://github.com/nicholasren/data_hacks3
+Download-URL: http://github.com/downloads/nicholasren/data_hacks3/data_hacks3-0.0.4.tar.gz
+Author: ['Xiaojun Ren']
+Author-email: nicholas.x.ren@gmail.com
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Intended Audience :: System Administrators
+Classifier: Topic :: Terminals
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: download-url
+Dynamic: home-page
+Dynamic: summary

data_hacks3-0.0.4.dist-info/RECORD

@@ -0,0 +1,14 @@
+data_hacks/bar_chart.py,sha256=2gW0Ln2l4XG73RgCbPCD0pb0AD5MyH3RCkvSsBb89TA,4524
+data_hacks/histogram.py,sha256=gO8gngVZBOL1_ai_0X7MEMQFNHdSaANV70XOslvWm1A,10743
+data_hacks/ninety_five_percent.py,sha256=gXP6lob_CDKFhe5o56Tbhjoo2I4-M7X7uAQdUHGnHwU,1728
+data_hacks/run_for.py,sha256=emvHWo3BXsAlRZbOsVhq_i_Zx5xMHRJtu6QAD4apiw0,1663
+data_hacks/sample.py,sha256=mV3q3xIGkwUrWmTsS5QzJ_Cw7j_Vd4pQ2rcXLUKYLZc,2092
+data_hacks3-0.0.4.data/scripts/bar_chart.py,sha256=BXk65pdgmhjLHVZzAze9YP3lbUJ4xCjfRyAMImzYsLQ,4511
+data_hacks3-0.0.4.data/scripts/histogram.py,sha256=MxaLuX04KFRGaQ-ou9mAaqIabkwtDgBRSxaD2h94KPk,10730
+data_hacks3-0.0.4.data/scripts/ninety_five_percent.py,sha256=L-WuCWqxyeN2earIqetS5aOSV9hx94TKXNakOm1B2TQ,1715
+data_hacks3-0.0.4.data/scripts/run_for.py,sha256=0hWIlec0J6LQ4v1LeI0WN_XjeB2uyEzAKxTNdTx7Qno,1650
+data_hacks3-0.0.4.data/scripts/sample.py,sha256=qFhRU2r2FqUB-KlnqmaTHXY3_Gk5vSc2gLtM1HYS-4Y,2079
+data_hacks3-0.0.4.dist-info/METADATA,sha256=yd5zpEZtBqG_Qrm0OXReGCe_MDI_xANaKO3wVV60axg,601
+data_hacks3-0.0.4.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+data_hacks3-0.0.4.dist-info/top_level.txt,sha256=80xgV1LClXEWXrY9e3yPfstyNGj-hXFhCQsi50HCQrU,11
+data_hacks3-0.0.4.dist-info/RECORD,,

data_hacks3-0.0.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

data_hacks3-0.0.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+data_hacks