GJDutils 0.2.2__py3-none-any.whl

gjdutils/dicts.py

@@ -0,0 +1,216 @@
+import json
+from typing import Iterable, Optional
+
+from gjdutils.strings import is_string, jinja_render
+
+
+def compare_dict(d1, d2, ignore_underscores=True):
+    """
+    Returns None if they're the same, else returns the first
+    key that differs.
+    """
+    for k, v in d1.items():
+        if ignore_underscores and k.startswith("_"):
+            continue
+        if v != d2[k]:
+            return k
+    return None
+
+
+def dict_from_module(mod):
+    """
+    e.g.
+      import config
+      config.BLAH
+      cfg = dict_from_module(config)
+      cfg['BLAH']
+      # OR
+      cfg = dict_from_module('config')
+    """
+    if is_string(mod):
+        mod = __import__(mod)
+    else:
+        raise ValueError("Expected a string, got %s" % type(mod))
+    return {k: v for k, v in mod.__dict__.items() if not k.startswith("__")}
+
+
+def pop_copy(d, k):
+    """
+    A non-destructive version of POP that returns both the
+    updated copy of the dictionary and the popped value,
+    e.g.
+
+      pop_copy({'a': 100, 'b': 200}, 'a') -> {'b': 200}, 100
+    """
+    out = d.copy()
+    v = out.pop(k)
+    return out, v
+
+
+def pop_safe(d, k, default=None):
+    """
+    Destructive, like the default pop, but returns DEFAULT
+    if K is not a key in D.
+    """
+    return d.pop(k) if k in d else default
+
+
+def whittle_dict(d, keys):
+    d2 = {}
+    for k in keys:
+        d2[k] = d[k]
+    return d2
+
+
+def these_fields_only(d: dict, fields: Iterable[str], required=True):
+    """
+    Returns D2, containing just the KEYS from dictionary D, e.g.
+
+      these_fields_only({'a': 100, 'b': 200}, ['a']) -> {'a': 100}
+
+    If REQUIRED, will raise an exception if any of KEYS aren't keys in D.
+    """
+    # used to be called WHITTLE_DICT
+    if required:
+        # will fail if any of the fields in FIELDS are missing from D
+        d2 = {k: d[k] for k in fields}
+    else:
+        d2 = {k: d[k] for k in fields if k in d}
+    return d2
+
+
+def print_dict(d: dict):
+    print("\n".join(["%s: %s" % (k, d[k]) for k in sorted(d.keys())]))
+
+
+def truncate_dict(d: dict, n: Optional[int], reverse=False) -> dict:
+    l = list(d.items())
+    if reverse:
+        l = list(reversed(l))
+    l_trunc = l[:n]
+    d_trunc = dict(l_trunc)
+    return d_trunc
+
+
+# this is no longer needed. if you want to update a dict in an expression:
+#   d_both = {**d1, **d2}
+# or
+#   d_both = d1 | d2
+# def update_d(d1, d2): ...
+
+
+def combine_dicts(d1: dict, d2: dict, require_unique: bool = True) -> dict:
+    """
+    UPDATE: you can now do the simple version of this with `d1 | d2`
+    """
+    if require_unique:
+        d1_keys = set(d1.keys())
+        d2_keys = set(d2.keys())
+        overlapping_k = set.intersection(d1_keys, d2_keys)
+        assert not overlapping_k, "Overlapping keys: %s" % overlapping_k
+    # d = d1.copy()
+    # d.update(d2)
+    d = d1 | d2
+    # check we've maintained ordering. only makes sense if REQUIRE_UNIQUE is False
+    # assert list(d1.keys()) + list(d2.keys()) == list(d.keys())
+    return d
+
+
+def update_params_from_defaults(defaults: dict, params_in: dict):
+    for param_k in params_in.keys():
+        assert param_k in defaults.keys(), "Unexpected param '%s'" % param_k
+    params_out = combine_dicts(defaults, params_in, require_unique=False)
+    return params_out
+
+
+def reverse_dict(d):
+    """
+    Returns a dictionary with values as keys and vice versa.
+
+    Will fail if the values aren't unique or aren't
+    hashable.
+    """
+    vals = d.values()
+    # confirm that all the values are unique
+    assert len(vals) == len(set(vals))
+    d2 = {}
+    for k, v in d.items():
+        d2[v] = k
+    return d2
+
+
+class HashableDict(dict):
+    # http://code.activestate.com/recipes/414283-frozen-dictionaries/
+    def __hash__(self):
+        return hash(tuple(sorted(self.items())))
+
+
+def rename_fields(d: dict, renames: dict):
+    """
+    Renames multiple fields in a dictionary at the same time,
+    returning a new one, and preserving the order, e.g.
+
+    D = {'a': 100, 'b': 200}
+    RENAMES = {'a': 'c'}
+    ->
+    D2 = {'c': 100, 'b': 200}
+    """
+    d2 = {}
+    for k, v in d.items():
+        if k in renames.keys():
+            k2 = renames[k]
+            d2[k2] = v
+        else:
+            d2[k] = v
+    return d2
+
+
+def dict_from_list(lst: list[dict], key, skip_duplicates: bool = False):
+    """
+    Given a list of dicts LST, return a dict DIC where D[v] = an item IT in LST where lst[key]=v.
+
+    For example, given a list of EVENTS, return a dict where D[event_id] = event:
+
+        events_d = dict_from_list(events_l, key="id")
+    """
+    dic = {}
+    for it in lst:
+        assert key in it, f"Key {key} not found in {it}"
+        v = it[key]
+        if skip_duplicates and (v in dic):
+            continue
+        assert v not in dic, f"Found duplicate key {v}"
+        dic[v] = it
+    return dic
+
+
+def pprint_dict(d: dict):
+    return print(json.dumps(d, indent=4))
+
+
+def dict_as_html(d: dict) -> str:
+    return jinja_render(
+        """
+<div class="dict-view">
+    <ul>
+    {% for key, value in d.items() %}
+        <li>
+            <strong>{{ key }}</strong>: 
+            {% if value is mapping %}
+                {{ dict_as_html(value) | safe }}
+            {% elif value is sequence and value is not string %}
+                <ul>
+                {% for item in value %}
+                    <li>{{ item }}</li>
+                {% endfor %}
+                </ul>
+            {% else %}
+                {{ value }}
+            {% endif %}
+        </li>
+    {% endfor %}
+    </ul>
+</div>
+""",
+        context={"d": d, "dict_as_html": dict_as_html},
+    )

gjdutils/dsci.py

@@ -0,0 +1,202 @@
+from collections import Counter
+import itertools
+import numpy as np
+import pandas as pd
+import random
+from scipy import spatial, sparse
+import sys
+from typing import Any, Iterable, Sequence
+
+from .rand import set_seeds
+
+
+def init_display_options():
+    # so that it's easier to see things in the terminal
+    pd.set_option("display.max_rows", 10000)
+    pd.set_option("display.max_columns", 1000)
+    pd.set_option("display.max_colwidth", 100)
+    np.set_printoptions(precision=2)
+    np.set_printoptions(suppress=True)
+
+
+def jaccard_similarity(list1, list2) -> float:
+    """
+    For comparing how much overlap there is between two sets.
+
+    Returns a normalised 0-1 similarity score, where higher = more similar.
+
+    USAGE:
+        a = ['hello', 'foo', 'foo', 'tux']
+        b = ['blah', 'hello', 'foo']
+        jaccard_similarity(a, b)
+
+    from https://stackoverflow.com/a/56774335
+    """
+    intersection = len(set(list1).intersection(list2))
+    union = len(set(list1)) + len(set(list2)) - intersection
+    return intersection / union
+
+
+def calc_proportion_identical(lst: Any) -> float:
+    """
+    Returns a value between 0 and 1 for the uniformity of the values
+    in LST, i.e. higher if they're all the same.
+    """
+
+    def count_most_common(lst):
+        """
+        Find the most common item in LST, and count how many times it occurs.
+        """
+        # Counter(['a', 'b', 'a']).most_common(2) -> [
+        #   ('a', 2),
+        #   ('b', 1),
+        # ]
+        # so this gives the count of the most common (in this case 2 occurrences of 'a')
+        return Counter(lst).most_common(1)[0][1]
+
+    most_common = count_most_common(lst)
+    if most_common == 1:
+        return 0
+    else:
+        return most_common / len(lst)
+
+
+def calc_normalised_std_tightness(vals: Sequence[float]) -> float:
+    """
+    The standard deviation STD is in the same units as VALS, i.e.
+    it's unnormalised. We normalise by the (absolute) mean,
+    subtract from 1, and truncate.
+
+    This gives us a unbounded 'tightness' score,
+    i.e. 1 means no variability, 0 means a lot of variability, e.g.
+
+    [19, 21, 20, 20] -> 0.96
+    [19,  1, 40, 20] -> 0.31
+    [ 9,  1, 70,  0] -> 0
+    """
+    n = len(vals)
+    if n == 0:
+        raise Exception("Empty")
+    elif n == 1:
+        return 1.0
+    if n == 2:
+        deviation = abs(vals[0] - vals[1])
+    else:
+        deviation = float(np.std(vals))
+
+    average = abs(sum(vals) / n)
+    if average < 0.01:
+        # e.g. mean([-50, 50]) -> 0
+        # risking a divide-by-zero, which could produce unstable results.
+        # better to default to treating as not part of the cluster?
+        return 0
+
+    normalised_std = deviation / average
+    tightness = 1 - min(1, normalised_std)
+    assert 0 <= tightness <= 1
+    return tightness
+
+
+def calc_pair_amounts_closeness(amounts: Sequence[float]) -> float:
+    """
+    Returns higher the closer the two numbers.
+
+    Returns 0 if one number is zero but the other isn't,
+    or if they're of different signs.
+    """
+    assert len(amounts) == 2
+    amount1, amount2 = max(amounts), min(amounts)
+    if amount1 == 0.0 and amount2 == 0.0:
+        # avoid divide-by-zero
+        return 1.0
+    if amount1 > 0 and amount2 < 0:
+        # because a debit and a credit are never similar, no matter what their values
+        return 0.0
+    if amount1 < 0:
+        # if it's negative, they're both negative, and this only works
+        # for positive numbers, so swap sign (and therefore max/min
+        # will be swapped too)
+        amount1, amount2 = abs(amount2), abs(amount1)
+    val = 1 - (amount1 - amount2) / (amount1 + amount2)
+    assert 0 <= val <= 1
+    return val
+
+
+def convert_sim_dist_reciprocal(val: float) -> float:
+    """
+    Convert from similarity to distance with 1/x, dealing with divide-by-zero.
+    """
+    assert 0 <= val <= 1
+    out = sys.maxsize if val == 0 else (1 / val)
+    assert 0 <= out <= 1
+    return out
+
+
+def convert_sim_dist_oneminus(val: float) -> float:
+    """
+    Convert from similarity to distance with 1 - x.
+    """
+    assert 0 <= val <= 1
+    out = 1 - val
+    assert 0 <= out <= 1
+    return out
+
+
+def square_df_from_square(sq, features):
+    df = pd.DataFrame(sq)
+    # create a 'Feature' column
+    df["Feature"] = features
+    rename_dict = dict(zip(range(len(features)), features))
+    df.rename(columns=rename_dict, inplace=True)
+    # df = df.reindex_axis(['Feature'] + features, axis=1)
+    df = df.set_index("Feature")
+    return df
+
+
+def long_df_from_flat(dists_flat, features):
+    combos = [(f1, f2) for f1, f2 in itertools.permutations(features, 2)]
+    assert len(combos) == len(dists_flat)
+    dists_triplet = [
+        (combo[0], combo[1], dist) for combo, dist in zip(combos, dists_flat)
+    ]
+    dists_long_df = pd.DataFrame(dists_triplet, columns=["Feature", "Brand", "Score"])
+    return dists_long_df
+
+
+def square_df_from_flat(dists_flat, features):
+    nFeatures = len(features)
+    dists_sq = spatial.distance.squareform(np.array(dists_flat))
+    assert dists_sq.shape == (nFeatures, nFeatures)
+    return square_df_from_square(dists_sq, features)
+
+
+def pairwise_local(data, distance_func, format="long"):
+    """
+    Returns squareform pairwise distances DataFrame (run symmetrically).
+
+    DATA should be a iterable of arrays (e.g. a list of bitarrays).
+    Pairs of rows from DATA will be passed into DISTANCE_FUNC, which
+    should return a float.
+
+    If format 'square' (default), returns an (nFeatures x nFeatures)
+    square distances matrix.
+
+    If format 'flat', returns a vector of distances (that could be fed
+    into scipy squareform to produce the 'square' version).
+
+    If format 'long', returns recs weighted-sum model format.
+    """
+    features = sorted(data.keys())
+    dists_flat = [
+        distance_func(data[f1], data[f2])
+        for f1, f2 in itertools.permutations(features, 2)
+    ]
+    if format == "flat":
+        return dists_flat
+    if format == "long":
+        return long_df_from_flat(dists_flat, features)
+    elif format == "square":
+        dists_sq_df = square_df_from_flat(dists_flat, features)
+        return dists_sq_df
+    else:
+        raise Exception("Unknown FORMAT %s" % format)

gjdutils/dt.py

@@ -0,0 +1,296 @@
+from calendar import monthrange
+from datetime import datetime, date, timedelta
+import pendulum
+from typing import Optional, Union
+
+
+def dt_str(
+    dt: Optional[datetime] = None, seconds: bool = True, tz: Optional[str] = None
+) -> str:
+    """
+    e.g. 2020-Nov-18 at 7:39:20pm -> '201118_1939_20'
+
+    If TZ is None, defaults to UTC. Or set e.g. 'Europe/London'.
+    """
+    if dt is None:
+        dt = pendulum.now(tz=tz)
+    else:
+        dt = pendulum.instance(dt, tz=tz)
+    format = "YYMMDD_HHmm_ss" if seconds else "YYMMDD_HHmm"
+    return dt.format(format)
+
+
+# def dt_str(dt=None, hoursmins=True, seconds=True):
+#     """
+#     Returns the current date/time as a yymmdd_HHMM_S string,
+#     e.g. 091016_1916_21 for 16th Oct, 2009, at 7.16pm in the
+#     evening.
+
+#     By default, returns for NOW, unless you feed in DT.
+#     """
+#     if dt is None:
+#         dt = datetime.datetime.now()
+#     fmt = "%y%m%d"
+#     if hoursmins:
+#         fmt += "_%H%M"
+#     if seconds:
+#         fmt += "_%S"
+#     return dt.strftime(fmt)
+
+
+def str_dt(s):
+    """
+    Returns the current date/time as a DATETIME object, when
+    fed in a YYYYmmdd_HHMM_S string. See DT_STR.
+    """
+    now = datetime.now()
+    try:
+        # TODO this was a django function. need to find a replacement
+        return now.strptime(s, "%Y%m%d_%H%M_%S")
+    except ValueError:
+        # without seconds
+        try:
+            return now.strptime(s, "%Y%m%d_%H%M")
+        except ValueError:
+            # without time at all
+            return now.strptime(s, "%Y%m%d")
+
+
+def pendulum_from_date(date: Union[datetime, date]) -> date:
+    """
+    It's easier to always work with Pendulum objects, so
+    convert to that (works from Datetime, Date, or Pendulum).
+    """
+    return pendulum.date(date.year, date.month, date.day)
+
+
+def timedelta_float(td, units="days"):
+    """
+    Returns a float of timedelta TD in UNITS
+    (either 'days' or 'seconds').
+
+    Can be negative for things in the past.
+
+    timedelta returns the number of
+    days and the number of seconds, but you have to combine
+    them to get a float timedelta.
+
+    e.g. timedelta_float(now() - dt_last_week) == c. 7.0
+    """
+    # 86400 = number of seconds in a day
+    if units == "days":
+        return td.days + td.seconds / 86400.0
+    elif units == "seconds":
+        return td.days * 86400.0 + td.seconds
+    else:
+        raise Exception("Unknown units %s" % units)
+
+
+def serialize_datetimes(d, level=0):
+    """
+    Recursively walks through a dictionary, serializing datetime objects
+    into ISO 8601 formatted strings.
+
+    Set a (somewhat arbitrary) maximum of 20 levels of dictionaries. We
+    should never get to more than that, but if we do, it will just stop
+    serializing datetimes.
+    """
+    if level > 20:
+        raise ValueError("Too many levels trying to serialize datetimes")
+
+    for k in d.keys():
+        if type(d[k]) == datetime:
+            d[k] = d[k].isoformat()
+        elif type(d[k]) == dict:
+            d[k] = serialize_datetimes(d[k], level + 1)
+        else:
+            pass
+    return d
+
+
+def datetime_to_date(dt):
+    return date(year=dt.year, month=dt.month, day=dt.day)
+
+
+def date_to_datetime(dt):
+    if isinstance(dt, datetime):
+        return dt
+    return datetime(year=dt.year, month=dt.month, day=dt.day)
+
+
+def month_name(dt):
+    return datetime.strftime(dt, "%B")
+
+
+def near_in_time(dt1, dt2=None):
+    """
+    Compares two datetime and ensures that they're within 1s
+    of each other. Doesn't care which came first. Useful for unit tests.
+    """
+    if dt2 is None:
+        dt2 = datetime.now()
+    dt_diff = abs(dt1 - dt2)
+    return dt_diff.days == 0 and dt_diff.seconds < 1
+
+
+def pp_date(dt):
+    """
+    Human-readable (i.e. pretty-print) dates, e.g. for spreadsheets:
+
+    See http://docs.python.org/tutorial/stdlib.html
+
+    e.g. 31-Oct-2011
+    """
+    d = date_to_datetime(dt)
+    return d.strftime("%d-%b-%Y")
+
+
+def humanize_minutes(minutes: int):
+    """
+    e.g.
+        humanize_minutes(5) -> '5 minutes'
+        humanize_minutes(61) -> '1 hour'
+        humanize_minutes(1500) -> 'a day'
+    from https://github.com/python-humanize/humanize
+    """
+    from humanize import naturalday, naturaldelta
+
+    delta = timedelta(minutes=minutes)
+    # minimum_unit="minutes" is not supported
+    ndelta = naturaldelta(delta)
+    return ndelta
+
+
+def alltime():
+    # return YourModel.happened.order_by('dt')[0].dt
+    #
+    # hardcode to avoid the query
+    return datetime(year=2009, month=10, day=31, hour=22, minute=34, second=6)
+
+
+def first_last_day_of_month(dt):
+    """
+    Returns two DATETIMES, one for the first and one for the
+    last day of the month of DT.
+    """
+    first_day = datetime(year=dt.year, month=dt.month, day=1)
+    nDays = monthrange(dt.year, dt.month)[1]
+    last_day = datetime(year=dt.year, month=dt.month, day=nDays)
+    return first_day, last_day
+
+
+def recent_hour(nHours=1, dt=None):
+    """
+    Returns the DT for 1 hour (i.e. 3600 seconds) ago.
+    """
+    seconds = nHours * 3600
+    dt = dt or datetime.now()
+    return dt - timedelta(seconds=seconds)
+
+
+def recent_day(nDays=1, dt=None):
+    """
+    Returns the DT for 1 day (i.e. 24 hours) ago.
+
+    If NDAYS == 24 hours * NDAYS.
+    """
+    dt = dt or datetime.now()
+    return dt - timedelta(days=nDays)
+
+
+def start_of_day(dt=None):
+    """
+    Returns the Datetime for DT at midnight, i.e. the start of the day.
+    """
+    dt = dt or datetime.now()
+    return datetime(year=dt.year, month=dt.month, day=dt.day)
+
+
+def end_of_day(dt=None):
+    dt = dt or datetime.now()
+    return datetime(
+        year=dt.year, month=dt.month, day=dt.day, hour=23, minute=59, second=59
+    )
+
+
+def start_of_week(dt=None):
+    """
+    Returns the DT for the beginning of the week (i.e. the most recent Monday at 00:01.
+    """
+    # weekday(): Monday = 0. http://docs.python.org/library/datetime.html
+    dt = dt or datetime.now()
+    # subtract however many days since Monday from today to get to Monday
+    return start_of_day(dt - timedelta(days=dt.weekday()))
+
+
+def end_of_week(dt=None):
+    dt = dt or datetime.now()
+    return end_of_day(dt + timedelta(days=(6 - dt.weekday())))
+
+
+def start_of_month(dt=None):
+    dt = dt or datetime.now()
+    # xxx - we could have also used:
+    # start_of_day(now - timedelta(days=now.day))
+    return first_last_day_of_month(dt)[0]
+
+
+def end_of_month(dt=None):
+    dt = dt or datetime.now()
+    return first_last_day_of_month(dt)[1] + timedelta(hours=23, minutes=59, seconds=59)
+
+
+def day_containing(dt=None):
+    """Return the half-open day interval containing dt.
+
+    i.e. if dt is Today 12:26, return (Today 00:00, Tomorrow 00:00).
+    This can be used for a half-open comparison:
+
+        p, n = day_containing()
+        if x >= p and x < n:
+            # Do something because x is today.
+    """
+
+    p = start_of_day(dt)
+    n = p + timedelta(days=1)
+
+    return p, n
+
+
+def daily_iter(start, end):
+    """Iterate over half-open day intervals pairwise until the end of the range falls after end."""
+
+    p = start
+    n = start + timedelta(days=1)
+
+    while n < end:
+        yield p, n
+        p = n
+        n = n + timedelta(days=1)
+
+
+def week_containing(dt=None):
+    """Returns a half-open interval of the week containing dt, starting on Sunday."""
+
+    p = start_of_week(dt)
+    n = p + timedelta(days=7)
+    return p, n
+
+
+def weekly_iter(start, end):
+    """Iterate over weeks pairwise until the end of the range falls after end."""
+
+    p = start
+    n = start + timedelta(days=7)
+
+    while n < end:
+        yield p, n
+        p = n
+        n = n + timedelta(days=7)
+
+
+def date_from_datetime(d: date, as_pendulum: bool = False):
+    if as_pendulum:
+        return pendulum.date(d.year, d.month, d.day)
+    else:
+        return datetime(d.year, d.month, d.day, hour=0, minute=0)