GJDutils 0.2.2__py3-none-any.whl

gjdutils/env.py

@@ -0,0 +1,64 @@
+import os
+from pathlib import Path
+from typing import Any, TypeVar, cast
+from pydantic import StrictStr, TypeAdapter
+
+T = TypeVar("T")
+_processed_vars = set()
+
+
+def get_env_var(name: str, typ: Any = StrictStr) -> T:
+    """Get environment variable with type validation, e.g.
+
+    OPENAI_API_KEY = get_env_var("OPENAI_API_KEY")
+    NUM_WORKERS = get_env_var("NUM_WORKERS", typ=int)
+
+    Args:
+        name: Name of environment variable
+        type_: Pydantic type to validate against (default: StrictStr for non-empty string)
+
+    Returns:
+        The validated value with the specified type
+
+    Raises:
+        ValueError: If variable is missing or fails validation
+    """
+    try:
+        value = os.environ[name]
+        _processed_vars.add(name)
+
+        # Use TypeAdapter for validation
+        adapter = TypeAdapter(typ)
+        validated = adapter.validate_python(value)
+
+        # Return validated value directly
+        return cast(T, validated)
+    except KeyError:
+        raise ValueError(f"Missing required environment variable: {name}")
+    except Exception as e:
+        raise ValueError(f"Invalid value for {name}: {e}")
+
+
+def list_env_example_vars(env_example_filen: Path) -> set[str]:
+    """Get set of required variables from .env.example.
+
+    Args:
+        env_example_filen: Path to the .env.example file
+
+    Returns:
+        Set of environment variable names found in the file
+    """
+    assert env_example_filen.exists(), f"Missing env example file: {env_example_filen}"
+
+    required_vars = set()
+    with env_example_filen.open() as f:
+        for line in f:
+            line = line.strip()
+            # Skip comments and empty lines
+            if not line or line.startswith("#"):
+                continue
+            # Get variable name (everything before =)
+            var_name = line.split("=")[0].strip()
+            required_vars.add(var_name)
+
+    return required_vars

gjdutils/errors.py

@@ -0,0 +1,12 @@
+import inspect
+import sys
+import traceback
+
+# see also functions.func_name()
+
+
+def str_from_exception(name=None):
+    return {
+        "name": name,
+        "msg": "".join(traceback.format_exception(*sys.exc_info())),
+    }

gjdutils/files.py

@@ -0,0 +1,140 @@
+import os
+from pathlib import Path
+from typing import Sequence
+
+from .cmd import run_cmd
+from .strings import is_string, PathOrStr
+
+
+def split_filen(filen: Path | str):
+    """
+    Splits a filename into its path, stem, and extension (without dot), e.g.
+
+        split_filen('data/blah.mp4') -> ('data', 'blah', 'mp4')
+    """
+    filen = Path(filen)
+    return filen.parent, filen.stem, filen.suffix[1:] if filen.suffix else ""
+
+
+def create_dir_if_not_exists(dirn: str):
+    if not os.path.exists(dirn):
+        os.makedirs(dirn)
+
+
+def validate_ext(ext):
+    assert is_string(ext)
+    assert ext.lower() == ext
+    assert ext
+    assert ext[0] != "."
+
+
+def validate_dir(dirn):
+    dirn_path = Path(dirn)
+    assert dirn_path.exists() and dirn_path.is_dir()
+    return dirn_path
+
+
+def fulltext(
+    filens: Sequence[str],
+    patterns: list[str],
+    dirn: str,
+    file_ext: str,
+    case_sensitive=False,
+):
+    """
+    Returns: FOUND_FILES (list of filename strings)
+
+    Feed in a list of filenames (complete with extensions),
+    which will be fed to agrep for full-text
+    searching. Returns a list of files.
+
+    FILENS is a list of strings. If its non-empty, then
+    these will be fed in to agrep. If it's empty, then we'll
+    just feed in a '*.[freex_extension]'. Spaces in
+    filenames are escaped with backslashes, but this is the
+    only thing we're escaping.
+
+    PATTERNS is a list of strings, which will be ANDed
+    together in the agrep regex. Currently, this doesn't
+    escape the pattern regex at all, though it does surround it in
+    quotes, so the usual agrep rules apply.
+
+    Unless case_sensitive==True, will append a -i flag.
+    """
+    # from freex_sqlalchemy.py
+
+    # xxx this should check that all the files have extensions
+
+    if case_sensitive:
+        case_flag = ""
+    else:
+        case_flag = "-i"
+
+    # xxx should check that all the items in the pattern
+    # list are strings...
+    #
+    # first strip each of the pattern strings of whitespace,
+    # and remove the surrounding quotes - we'll add them
+    # back to the whole pattern_str when we create the CMD
+    #
+    # then AND together multiple patterns with agrep,
+    # using semicolons
+    for pat in patterns:
+        if pat[0] == '"':
+            pat = pat[1:]
+        if pat[-1] == '"':
+            pat = pat[0:-1]
+
+    pattern_str = ";".join([x.strip() for x in patterns])
+
+    if len(filens) > 0:
+        # escape all the spaces with back-slashes
+        filens = [x.replace(" ", "\\ ") for x in filens]
+
+        # convert to a space-delimited string (with spaces
+        # escaped by backslashes), and each file prepended by the
+        # database_dir, e.g.
+        # /blah/test0.freex /blah/hello\ world.freex
+        fnames_str = " ".join([os.path.join(dirn, filen) for filen in filens])
+
+        # the -l says to just return filenames only (no text
+        # context)
+        #
+        # put the pattern in quotes
+        #
+        # and then just list the files at the end
+        cmd = 'agrep -l %s "%s" %s' % (case_flag, pattern_str, fnames_str)
+
+    else:
+        # if we're not restricting the files we're looking
+        # through, then there could be too many files to run
+        # agrep on directly, so we have to pipe it from a
+        # find
+        #
+        # this is to avoid the '/usr/local/bin/agrep:
+        # Argument list too long' error
+        cmd = 'find %s -name "*.%s" -print0 | xargs -0 agrep -l %s "%s"' % (
+            dirn,
+            file_ext,
+            case_flag,
+            pattern_str,
+        )
+
+    # Run command with minimal output unless there's an error
+    retcode, out_str, _ = run_cmd(
+        cmd,
+        verbose=0,
+        check=False,  # Don't raise exception if no matches found (agrep returns 1)
+    )
+
+    if len(out_str) > 0:
+        # strip away the path to yield just the filename for
+        # each of the files in out_str
+        found_files = [os.path.basename(x) for x in out_str.strip().split("\n")]
+    else:
+        # if you run the above on an empty string, you get
+        # [''], whereas we really want to return an empty
+        # list if we didn't find anything
+        found_files = []
+
+    return found_files

gjdutils/functions.py

@@ -0,0 +1,6 @@
+import inspect
+
+
+def func_name():
+    # https://stackoverflow.com/a/13514318/230523
+    return inspect.currentframe().f_back.f_code.co_name

gjdutils/google_translate.py

@@ -0,0 +1,80 @@
+from cachetools import cached, LRUCache, TTLCache
+from google.cloud import translate_v2 as translate
+import html
+from typing import Optional
+
+
+def translate_text(
+    text: str,
+    lang_src_code: Optional[str],
+    lang_tgt_code: str,
+    verbose: int = 0,
+):
+    """Translates text into the target language.
+
+    Target must be an ISO 639-1 language code.
+    See https://g.co/cloud/translate/v2/translate-reference#supported_languages
+    """
+    translate_client = translate.Client()
+
+    lang_src_code = (
+        lang_src_code[:2].lower() if isinstance(lang_src_code, str) else None
+    )
+    lang_tgt_code = lang_tgt_code[:2].lower()
+    if lang_src_code == lang_tgt_code:
+        return text, None
+
+    # assert lang_src_code != lang_tgt_code, (
+    #     "Identical src and tgt language codes: %s" % lang_src_code
+    # )
+
+    # Text can also be a sequence of strings, in which case this method
+    # will return a sequence of results for each text.
+    if lang_src_code is None:
+        result = translate_client.translate(text, target_language=lang_tgt_code)
+    else:
+        result = translate_client.translate(
+            text,
+            target_language=lang_tgt_code,
+            source_language=lang_src_code,
+        )
+
+    translated_text = result["translatedText"]
+
+    # fix escaping, e.g.
+    #   I've done it a week with no improvement
+    #   ->
+    #   I've done it a week with no improvement
+    translated_text = html.unescape(translated_text)
+
+    if verbose > 0:
+        print(f"{lang_src_code} -> {lang_tgt_code}")
+        print(f"\t\"{result['input']}\" -> \"{translated_text}\"")
+        if lang_src_code is None:
+            print(f"\t\tDetected source language: {result['detectedSourceLanguage']}")
+
+    return translated_text, result
+
+
+# translated_text, result = translate_text(
+#     text="Hello, world",
+#     lang_src_code="en",
+#     lang_tgt_code="el",
+#     verbose=0,
+# )
+# translated_text
+
+
+@cached(cache={})
+def detect_language(text: str, verbose: int = 0) -> tuple[str, dict]:
+    """
+    Detects the text's language.
+    """
+    translate_client = translate.Client()
+
+    # Text can also be a sequence of strings, in which case this method
+    # will return a sequence of results for each text.
+    result = translate_client.detect_language(text)
+    language, confidence = result["language"], result["confidence"]
+    print(f"Ran detect_language for {text} -> {language} at confidence {confidence}")
+    return language, confidence

gjdutils/hashing.py

@@ -0,0 +1,32 @@
+import base64
+import hashlib
+
+
+def hash_readable(s, n=10):
+    """
+    Returns a string hash that contains base32 characters instead of a number,
+    to make it more readable (and still low risk of collisions if you truncate it).
+
+    e.g. hash_readable('hello') => 'vl2mmho4yx'
+
+    Unlike Python's default hash function, this should be deterministic
+    across sessions (because we're using 'hashlib').
+
+    I'm using this for anonymising email addresses if I don't have a user UUID.
+    """
+    if isinstance(s, str):
+        s = bytes(s, "utf-8")
+    hasher = hashlib.sha1(s)
+    b32 = base64.b32encode(hasher.digest())[:n]
+    return b32.decode("utf-8").lower()
+
+
+def hash_consistent(obj):
+    """
+    Supposedly gives the same response every time you call it, even after restarting the kernel.
+
+    N.B. This is based on output from GitHub Copilot, and I haven't tried it.
+    """
+    obj_str = str(obj)
+    hash_obj = hashlib.sha256(obj_str.encode()).hexdigest()
+    return hash_obj

gjdutils/html.py

@@ -0,0 +1,87 @@
+from bs4 import BeautifulSoup
+from lxml import html as lxml_html
+from lxml.html import tostring, fromstring
+from lxml.etree import Element, _Element as ElementType
+from typing import Optional, Iterable, Union
+
+
+def remove_html_tags(html: str):
+    soup = BeautifulSoup(html, "html.parser")
+    return soup.get_text()
+
+
+def contents_of_body(soup):
+    """
+    e.g.
+      BeautifulSoup('<p>hello</p><p>world</world>', features='lxml')
+        =>
+          <p>hello</p>
+          <p>world</p>
+
+    N.B. for html.parser, you might just be able to do: str(soup)
+    """
+    # it might be better to prettify with body hidden=True???
+    return "\n".join([str(t) for t in soup.body.contents])
+
+
+def compare_html(h1, h2):
+    h1p = BeautifulSoup(h1, features="html.parser").prettify().strip()
+    h2p = BeautifulSoup(h2, features="html.parser").prettify().strip()
+    assert h1p == h2p
+
+
+def remove_attrs_from_html(h):
+    """
+    Gets rid of all the attrs in the html.
+    """
+    soup = BeautifulSoup(h, features="lxml")
+    for t in soup.recursiveChildGenerator():
+        t.attrs = {}  # type: ignore
+    # whitespace_from_linebreaks(
+    # contents_of_body(soup)
+    # )
+    return contents_of_body(soup)
+
+
+def adjust_indentation(pretty_html, indent: int):
+    # from https://www.perplexity.ai/search/can-you-customise-the-beautifu-225tf.pISaiggsL5tNL.gA
+    lines = pretty_html.split("\n")
+    adjusted_lines = []
+    for line in lines:
+        line_lstrip = line.lstrip(" ")
+        leading_spaces = len(line) - len(line_lstrip)
+        indent_level = leading_spaces // 1  # default indent is 1 space
+        adjusted_lines.append(" " * (indent_level * indent) + line_lstrip)
+    return "\n".join(adjusted_lines)
+
+
+def prettify_html(
+    html: Union[str, ElementType, list[ElementType]],  # BeautifulSoup
+    indent: int = 2,
+    n: Optional[int] = None,  # number of chars to show
+):
+    # if isinstance(html, pq):
+    #     html = html.outer_html()  # type: ignore
+    if isinstance(html, list):
+        # then we'll handle it as a string in a moment
+        html = "".join(
+            [tostring(e, method="html").decode() for e in html]
+        )  #  type: ignore
+    if isinstance(html, ElementType):
+        # this will do some cleaning and fixing. but
+        # you need document_fromstring() if you want to make sure
+        # that it's a full html doc, e.g. with html, body
+        html = tostring(html, method="html").decode()
+
+    soup = BeautifulSoup(html, "html.parser")
+    # html2 = tostring(html, pretty_print=True, method="html").decode()  # type: ignore
+    # the lxml pretty_print just isn't as good as BS4, e.g. with a list of elements
+    # it wraps things in a div fragment, but the pretty-print of that isn't right
+    html2 = soup.prettify()
+    prettified = adjust_indentation(html2, indent=indent)[:n]  # type: ignore
+    return prettified
+
+
+def pprettify_html(*args, **kwargs) -> None:
+    html = prettify_html(*args, **kwargs)
+    print(html)

gjdutils/indexing.py

@@ -0,0 +1,97 @@
+from decimal import Decimal, getcontext
+from typing import Optional
+
+"""
+For manual ordering and reording in a database:
+- every item gets a Decimal location (LOC) between 0 and 1
+- the LOCs of all items are sorted
+- the LOC of a new item is calculated as the average of the LOCs of the items before and after it
+- the LOC won't ever be 0 or 1, so there will always be a gap for you to insert afterwards
+- if you insert at the beginning or end, the LOC will be half of the first or last item's LOC
+
+I think Figma used this.
+"""
+
+# Set precision high enough to handle many divisions
+getcontext().prec = 28
+
+
+def locs_for(n: int) -> list[Decimal]:
+    # Assign initial idx values with buffers at both ends
+    locs = []
+    for i in range(n):
+        loc = Decimal(i + 1) / Decimal(n + 1)
+        locs.append(loc)
+    assert len(locs) == n
+    return locs
+
+
+def loc_for_insert_at(locs: list[Decimal], position: int, do_insert: bool = True):
+    """
+    TODO: rewrite in terms of LOC_BETWEEN
+    """
+    assert locs == sorted(
+        locs
+    ), f"Input LOCS are unsorted, so things are already broken - {locs}"
+    list_length = len(locs)
+    if position == 0:  # Insert at the beginning
+        newloc = locs[0] / 2 if list_length > 0 else Decimal("0.5")
+    elif position >= list_length:  # Insert at the end
+        newloc = locs[-1] + (1 - locs[-1]) / 2 if list_length > 0 else Decimal("0.5")
+    elif position < 0:
+        raise Exception(f"Position must be non-negative, but got {position}")
+    else:  # Insert between two items
+        newloc = (locs[position - 1] + locs[position]) / 2
+    if do_insert:
+        locs.insert(position, newloc)
+        assert locs == sorted(locs), f"Somehow we've broken the LOCS sorting: {locs}"
+    return newloc
+
+
+def loc_for_insert_at2(locs: list[Decimal], position: int, do_insert: bool = True):
+    """
+    Functional version of LOC_FOR_INSERT_AT that returns a new list instead of
+    modifying the input list.
+
+    Uses LOC_BETWEEN.
+    """
+    assert locs == sorted(
+        locs
+    ), f"Input LOCS are unsorted, so things are already broken - {locs}"
+    list_length = len(locs)
+    if position < 0:
+        raise Exception(f"Position must be non-negative, but got {position}")
+    elif position == 0:  # Insert at the beginning
+        loc1 = None
+        loc2 = locs[0] if list_length > 0 else None
+    elif position >= list_length:  # Insert at the end
+        loc1 = locs[-1] if list_length > 0 else None
+        loc2 = None
+    else:  # Insert between two items
+        loc1 = locs[position - 1]
+        loc2 = locs[position]
+    newloc = loc_between(loc1, loc2)
+    if do_insert:
+        locs.insert(position, newloc)
+        assert locs == sorted(locs), f"Somehow we've broken the LOCS sorting: {locs}"
+    return newloc
+
+
+def loc_between(loc1: Optional[Decimal], loc2: Optional[Decimal]) -> Decimal:
+    if loc1 is not None and loc2 is not None:
+        assert (
+            loc1 >= 0 and loc2 <= 1
+        ), f"LOCs must be between 0 and 1, but got {loc1} and {loc2}"
+        return (loc1 + loc2) / 2
+    elif loc1 is None and loc2 is not None:
+        return loc2 / 2
+    elif loc1 is not None and loc2 is None:
+        return loc1 + (1 - loc1) / 2
+    elif loc1 is None and loc2 is None:
+        return Decimal("0.5")
+    else:
+        raise Exception(f"This should never happen: {loc1}, {loc2}")
+
+
+def disp(locs: list[Decimal]):
+    print(", ".join(["%.3f" % loc for loc in locs]))

gjdutils/iterfunc.py

@@ -0,0 +1,99 @@
+import itertools
+from typing import Sequence
+
+
+def contiguous_pairs(lst: Sequence):
+    """
+    Given a list LST, return the contiguous pairs, e.g.
+
+    [10, 20, 30, 40, 50]
+    ->
+    [(10, 20), (20, 30), (30, 40), (40, 50)]
+
+    (from GitHub Copilot)
+    """
+    pairs = [(lst[i], lst[i + 1]) for i in range(len(lst) - 1)]
+    return pairs
+
+
+def flatten(lol):
+    """
+    See http://stackoverflow.com/questions/406121/flattening-a-shallow-list-in-python
+
+    e.g. [['image00', 'image01'], ['image10'], []] -> ['image00', 'image01', 'image10']
+    """
+
+    chain = list(itertools.chain(*lol))
+    return chain
+
+
+# def flatten(list_of_lists):
+#     """
+#     Flatten one level of nesting
+
+#     from https://docs.python.org/3/library/itertools.html#itertools-recipes
+#     """
+#     return list(chain.from_iterable(list_of_lists))
+
+
+def unique(items):
+    """
+    Returns KEEP, a list based on ITEMS, but with duplicates
+    removed (preserving order, based on first new example).
+
+    http://stackoverflow.com/questions/89178/in-python-what-is-the-fastest-algorithm-for-removing-duplicates-from-a-list-so-t
+
+    unique([1, 1, 2, 'a', 'a', 3]) -> [1, 2, 'a', 3]
+    """
+    found = set([])
+    keep = []
+    for item in items:
+        if item not in found:
+            found.add(item)
+            keep.append(item)
+    return keep
+
+
+def uniquify_list(lst):
+    """Return a list of the elements in s, but without duplicates, preserving order.
+
+    from comment in http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560
+
+      Lightweight and fast ..., Raymond Hettinger, 2002/03/17
+
+    """
+
+    set = {}
+    return [set.setdefault(e, e) for e in lst if e not in set]
+
+
+def grouper(iterable, n):
+    """
+    Collect data into fixed-length chunks or blocks. If
+    the last block is too small, returns a truncated block.
+
+    e.g. grouper('ABCDEFG', 3) --> ABC DEF G
+
+    From http://stackoverflow.com/a/8991553/230523
+    """
+    it = iter(iterable)
+    while True:
+        chunk = tuple(itertools.islice(it, n))
+        if not chunk:
+            return
+        yield chunk
+
+
+def grouper_ragged(iterable, n):
+    """
+    Collect data into non-overlapping chunks - the last one might be shorter than the others
+
+    >>> print(list(grouper('ABCDEFG', 3)))  # [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)]
+
+    from https://stackoverflow.com/a/41333827/230523
+    """
+    it = iter(iterable)
+    group = tuple(itertools.islice(it, n))
+    while group:
+        yield group
+        group = tuple(itertools.islice(it, n))

gjdutils/jsons.py

@@ -0,0 +1,70 @@
+import json
+from typing import Optional
+
+
+def jsonify(x):
+    def json_dumper_robust(obj):
+        try:
+            return obj.toJSON()
+        except:
+            try:
+                return str(obj)
+            except:
+                return None
+
+    return json.dumps(x, sort_keys=True, indent=4, default=json_dumper_robust)
+
+
+# from o-1
+# class RobustJSONEncoder(json.JSONEncoder):
+#     def __init__(self, *args, **kwargs):
+#         self.seen = set()
+#         super().__init__(*args, **kwargs)
+
+#     def default(self, obj):
+#         if id(obj) in self.seen:
+#             return None  # Replace circular references with None or a placeholder
+#         self.seen.add(id(obj))
+#         try:
+#             return obj.toJSON()
+#         except:
+#             try:
+#                 return str(obj)
+#             except:
+#                 return None
+
+
+# def jsonify(x):
+#     return json.dumps(x, cls=RobustJSONEncoder, sort_keys=True, indent=4)
+
+
+def to_json(
+    inps: list,
+    fields: Optional[list] = None,
+    skip_if_missing: bool = False,
+    skip_empties: bool = True,
+    max_str_len: Optional[int] = 1000,
+) -> str:
+    """
+    Convert a list of dicts to a JSON string, with only the fields we want,
+    and in the same order as FIELDS.
+    """
+    outs = []
+    for inp in inps:
+        if fields is None:
+            fields = inp.keys()
+        # we want to make sure to return a dict with only the fields we want,
+        # and in the same order as FIELDS
+        out = {}
+        for k in fields:
+            if skip_if_missing and (k not in inp):
+                continue
+            v = inp[k]  # will error if missing and !SKIP_IF_MISSING
+            if skip_empties and (v is None or v == ""):
+                continue
+            if max_str_len and isinstance(v, str) and len(v) > max_str_len:
+                v = v[:max_str_len] + "..."
+            out[k] = v
+        outs.append(out)
+    outs_j = json.dumps(outs, indent=2)
+    return outs_j