GJDutils 0.2.2__py3-none-any.whl

gjdutils/__init__.py

@@ -0,0 +1,12 @@
+import os
+import sys
+
+# Add the root directory to Python path to import __VERSION__
+root_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+
+from __VERSION__ import __version__
+
+# Export version at package level
+__version__ = __version__

gjdutils/audios.py

@@ -0,0 +1,39 @@
+import os
+from typing import Optional
+
+
+def play_mp3(mp3_filen: str, prog: str = "cli", speed: Optional[float] = None):
+    prog = prog.lower().strip()
+    full_mp3_filen = os.path.abspath(os.path.expanduser(mp3_filen))
+    if prog == "vlc":
+        # pip install python-vlc
+        import vlc
+
+        vlc_mp3_filen = os.path.join("file://", full_mp3_filen)
+        p = vlc.MediaPlayer(vlc_mp3_filen)
+        if speed is not None:
+            p.set_rate(speed)  # type: ignore
+        p.play()  # type: ignore
+    elif prog == "pygame":
+        assert speed is None, "Not implemented speed for pygame"
+        import pygame
+
+        pygame.init()
+        pygame.mixer.init()
+        pygame.mixer.music.load(full_mp3_filen)
+        pygame.mixer.music.play()
+        pygame.event.wait()
+    elif prog == "playsound":
+        # maybe set to 1.2.2 if you're having trouble installing
+        from playsound import playsound
+
+        assert speed is None, "Playsound doesn't support changing speed"
+        # https://stackoverflow.com/a/63147250/230523
+        playsound(mp3_filen)
+    elif prog == "cli":
+        cmd = f"afplay -r {speed} '{full_mp3_filen}'"
+        # print(cmd)
+        os.system(cmd)
+    else:
+        raise Exception(f"Unknown PROG '{prog}'")
+    return full_mp3_filen

gjdutils/cacheing.py

@@ -0,0 +1,237 @@
+from functools import update_wrapper
+import inspect
+import re
+
+
+# from abracadjabra.utils
+def generate_mckey(prefix, d):
+    """
+    Should generate a legal, human-readable, unique and
+    predictable string Memcached key from dictionary
+    D. N.B. we're using 'mckey' to distinguish Memcached
+    'keys' from dictionary 'keys'.
+
+    Designed to be called at the top of your function with
+    locals(), so it ignores any REQUEST and SELF keys - but
+    you can always add {'user': request.user} if needed.
+
+    Notes:
+
+    - Prepends PREFIX + '__' to the MCKEY generated from D.
+
+    - For dictionaries, concatenates the keys+values, sorted
+      by key.
+
+    - Converts lists and querysets to lists of IDs, hashing
+      the result if too long.
+
+    - Hashes the MCKEY if it's non-ascii or too long. Removes spaces etc.
+
+    Keys are separated from their value by '::', and the
+    key/value pairs are separated from one another by '__'.
+
+    e.g.
+
+       {'a': 100, 'b': 'blah'} -> u'a::100__b::blah'
+
+    xxx - should be moved to utils.caching, along with Evan's cmcd
+    """
+
+    def sorted_dict_by_keys(d):
+        """
+        Returns a SortedDict, with the keys sorted alphabetically.
+
+        This might not be necessary, since I think the order
+        of a python dict's keys() is deterministic, but by
+        sorting by dictionary keys, it's easier to know in
+        advance what the generated MCKEY should look like.
+        The idea is to ensure that no matter how D was
+        created, you'll know what the key should be.
+        """
+        sorted_d = SortedDict()
+        for k in sorted(d.keys()):
+            sorted_d[k] = d[k]
+        return sorted_d
+
+    def to_str_or_hash(s):
+        """
+        Tries to convert S to a STR. If it doesn't work,
+        just return the hash.
+        """
+        try:
+            s = str(s)
+        except UnicodeEncodeError:
+            s = str(hash(s))
+        return s
+
+    def hash_if_too_long(s):
+        """
+        Return the HASH of S rather than S if it's too long
+        (since the hash is only 10 characters).
+
+        We call this on each component and then once more at
+        the end because we want to keep the overall result
+        as human-readable as possible, while still being
+        unique.
+        """
+        if len(s) > MAX_MEMCACHED_KEY_LEN:
+            s = str(hash(s))
+        return s
+
+    def iterable_to_string(seq):
+        """
+        If the items in SEQ are Django Models,
+        store a comma-separated list of ids.
+
+        Otherwise, just join the items in SEQ.
+
+        e.g. [Thing.objects.get(id=1), Thing.objects.get(id=2)] -> 'Thing:1,2'
+        """
+        if not seq:
+            return ""
+        pieces = [to_str_or_hash(x) for x in seq]
+        model_prefix = ""
+        s = model_prefix + ",".join(pieces)
+        s = hash_if_too_long(s)
+        return s
+
+    def sanitize_val(v):
+        """
+        If V is an iterable, turn it into a comma-separated
+        string (of IDs, if Models).
+
+        Even though (empirically) it appears that Django's
+        cache.set and cache.get use Memcached's binary
+        protocol (so they can deal with non-ascii keys), it
+        seems safer to require the key to be ascii.
+        """
+        if isinstance(v, str):
+            pass
+        elif isinstance(v, unicode):
+            v = to_str_or_hash(v)
+        elif hasattr(v, "pk"):
+            # for instances, i decided not to separate the
+            # modelname from the id with a colon to
+            # distinguish them from querysets
+            v = v._meta.object_name + str(v.pk)
+        elif isinstance(v, dict):
+            # we might decide that even if we *can* deal
+            # with dicts like this, it's too crazy to be worth it...
+            v = generate_mckey("", v)
+        elif isiterable(v):
+            v = iterable_to_string(v)
+        else:
+            v = to_str_or_hash(v)
+        v = v.strip()
+        return hash_if_too_long(v)
+
+    # 250 bytes, minus global KEY_PREFIX, plus leave extra room in case
+    MAX_MEMCACHED_KEY_LEN = 200
+
+    prefix = to_str_or_hash(prefix).upper()
+
+    assert isinstance(d, dict)
+    # ignore REQUEST and SELF, so you can easily pass in locals() for D
+    if "request" in d:
+        del d["request"]
+    if "self" in d:
+        del d["self"]
+    d = sorted_dict_by_keys(d)
+
+    # require everything to be a nice ascii string
+    pieces = "__".join(
+        [
+            "%s::%s"
+            % (
+                sanitize_val(k),
+                sanitize_val(v),
+            )
+            for k, v in d.items()
+        ]
+    )
+
+    prefix_pieces = prefix + "__" + pieces
+    # replace all whitespace with underscores
+    prefix_pieces = re.sub("[ \t\r\n]+", "_", prefix_pieces)
+    # not too long
+    prefix_pieces = hash_if_too_long(prefix_pieces)
+    # make sure it's ascii-friendly
+    prefix_pieces = str(prefix_pieces)
+    return prefix_pieces
+
+
+def cmcd(prefix=None, arg_names=(), expiry=None):
+    """Caches the return value of func based on the cache key generated by
+    generate_mckey. The prefix argument to the `generate_mckey` is
+    determined from the module and the name of the function if `prefix` is
+    `None`. `arg_names` should be a sequence of strings that will be
+    pulled from the kwargs dict and passed to `generate_mckey` to generate
+    a key.
+
+    Unfortunately we don't have access to the same locals() as the
+    function itself, so the functions we're wrapping with this need to
+    take keyword arguments, and the arguments we're generating the cache
+    from must be specified.
+
+    NOTE: prefix must be defined in settings.CACHE_EXPIRY, OR set expiry=EXPIRY_TIME, e.g.
+
+    See utils.tests for usage.
+    """
+
+    def dec(func, prefix=prefix, arg_names=arg_names, expiry=expiry):
+        if expiry is None:
+            if prefix == None:
+                prefix = ".".join((func.__module__, func.__name__))
+
+            prefix = prefix.upper()
+            if prefix not in sett.CACHE_EXPIRY:
+                raise Exception(
+                    "Prefix %s must be defined in settings.CACHE_EXPIRY if expiry is not specified"
+                    % prefix
+                )
+
+            expiry = sett.CACHE_EXPIRY[prefix]
+
+        fspec = inspect.getargspec(func)
+        pos_args = fspec.args
+        defaults = fspec.defaults
+        defaults = defaults if defaults else ()
+
+        if pos_args or defaults:
+            if not arg_names:
+                raise Exception(
+                    "arg_names must be specified for functions that take arguments."
+                )
+
+            default_args = dict(
+                (k, v) for k, v in zip(reversed(pos_args), reversed(defaults))
+            )
+            noargs = False
+        else:
+            noargs = True
+
+        def f(*args, **kwargs):
+            if not noargs:
+                all_args = dict(default_args)
+                all_args.update(dict((n, v) for n, v in zip(pos_args, args)))
+                all_args.update(kwargs)
+                d = dict((k, all_args.get(k, None)) for k in arg_names)
+            else:
+                all_args = {}
+                d = {}
+
+            mckey = generate_mckey(prefix, d)
+            cached = cache.get(mckey)
+
+            if cached:
+                return cached
+
+            val = func(*args, **kwargs)
+
+            cache.set(mckey, val, expiry)
+
+            return val
+
+        return update_wrapper(f, func)
+
+    return dec

gjdutils/cmd.py

@@ -0,0 +1,149 @@
+from rich.console import Console
+import subprocess
+import sys
+import time
+from typing import Union, Optional, Dict
+from pathlib import Path
+
+from gjdutils.shell import fatal_error_msg
+
+
+def run_cmd(
+    cmd: Union[str, list[str]],
+    before_msg: Optional[str] = None,
+    fatal_msg: Optional[str] = None,
+    verbose: int = 2,
+    replace_sys_python_executable: bool = True,
+    dry_run: bool = False,
+    **subprocess_kwargs,
+) -> tuple[int, str, Dict]:
+    """Run a shell command with enhanced output and error handling.
+
+    Args:
+        cmd: Command to run as string (shell=True) or list of strings (shell=False)
+        before_msg: Optional message to display before running command (green)
+        fatal_msg: Optional message to use if command fails (calls fatal_error_msg)
+        verbose: Output verbosity level:
+            0 = silent
+            1 = show before_msg if provided
+            2 = also show command being run (default)
+            3 = also show working directory and duration
+            4 = also show command stdout output
+        replace_sys_python_executable: Replace 'python ' with sys.executable
+        dry_run: If True, only print what would be run
+        **subprocess_kwargs: Additional arguments passed to subprocess.run
+
+    Returns:
+        Tuple of (returncode, stdout, extra) where extra is a dict containing:
+        - stderr: Standard error output
+        - duration: Time taken to run command
+        - cmd_str: Final command string that was run
+        - cwd: Working directory
+        - input_args: Original function arguments
+        - subprocess_result: Full subprocess.CompletedProcess object
+
+    Examples:
+        Simple usage with string command:
+        >>> retcode, out, _ = run_cmd4("ls -l", before_msg="Listing files...")
+        Listing files...
+        $ ls -l
+        >>> print(out)
+        total 8
+        -rw-r--r-- 1 user user 2048 Mar 15 10:00 example.txt
+
+        Complex usage with list command and error handling:
+        >>> cmd = ["pytest", "tests/", "-v", "--cov"]
+        >>> retcode, out, extra = run_cmd4(
+        ...     cmd,
+        ...     before_msg="Running tests with coverage...",
+        ...     fatal_msg="Tests failed!",
+        ...     verbose=2,
+        ...     timeout=300,
+        ...     check=True
+        ... )
+        Running tests with coverage...
+        $ pytest tests/ -v --cov
+        === test session starts ===
+        ...
+    """
+    input_args = locals()
+
+    console = Console()
+
+    start_time = time.time()
+
+    # Convert list command to string if needed
+    cmd_str = " ".join(cmd) if isinstance(cmd, list) else cmd
+
+    # Replace python executable if requested
+    if replace_sys_python_executable and cmd_str.startswith("python "):
+        cmd_str = f"{sys.executable} {cmd_str[7:]}"
+
+    # Handle verbosity
+    if verbose >= 1 and before_msg:
+        console.print(f"[green]{before_msg}[/green]")
+    if verbose >= 2:
+        console.print(f"[white]$ {cmd_str}[/white]")
+
+    # Handle dry run
+    if dry_run:
+        return (
+            0,
+            "",
+            {
+                "stderr": "",
+                "duration": 0,
+                "cmd_str": cmd_str,
+                "cwd": str(Path.cwd()),
+                "input_args": input_args,
+                "subprocess_result": None,
+            },
+        )
+
+    # Set defaults for subprocess
+    subprocess_kwargs.setdefault("shell", isinstance(cmd, str))
+    subprocess_kwargs.setdefault("capture_output", True)
+    subprocess_kwargs.setdefault("text", True)
+
+    try:
+        result = subprocess.run(
+            cmd if isinstance(cmd, list) else cmd_str,
+            **subprocess_kwargs,
+        )
+    except subprocess.TimeoutExpired as e:
+        if fatal_msg:
+            fatal_error_msg(
+                fatal_msg,
+                f"Command timed out after {subprocess_kwargs.get('timeout', '?')}s",
+            )
+        raise
+
+    duration = time.time() - start_time
+
+    # Show additional info at verbose level 3
+    if verbose >= 3:
+        console.print(f"[blue]Working directory: {Path.cwd()}[/blue]")
+        console.print(f"[blue]Duration: {duration:.2f}s[/blue]")
+    if verbose >= 4:
+        console.print(f"[blue]Command output:[/blue]\n{result.stdout}")
+
+    # Handle errors
+    if result.returncode != 0:
+        # Show both stdout and stderr for failed commands
+        if result.stdout:
+            console.print(f"[red]Command output:[/red]\n{result.stdout}")
+        if result.stderr:
+            console.print(f"[red]Command error output:[/red]\n{result.stderr}")
+        if fatal_msg:
+            fatal_error_msg(fatal_msg)
+
+    extra = {
+        "stderr": result.stderr,
+        "duration": duration,
+        "cmd_str": cmd_str,
+        "cwd": str(Path.cwd()),
+        "input_args": input_args,
+        "subprocess_result": result,
+    }
+
+    return result.returncode, result.stdout.strip(), extra

gjdutils/colab.py

@@ -0,0 +1,39 @@
+import os
+
+from .runtime import in_colab
+
+# https://stackoverflow.com/a/53586419/230523
+IN_COLAB = in_colab()
+# also specified in authortools_demo.ipynb
+GOOGLE_DRIVE_MOUNT_PATH = "/content/drive"
+GOOGLE_DRIVE_OUTPUT_PATH = os.path.join(
+    GOOGLE_DRIVE_MOUNT_PATH,
+    "Shareddrives",
+    "Blah",  # TODO
+)
+
+
+def colab_path_if_needed(filen: str):
+    """
+    Prepend the Google Drive mount path for Colab if IN_COLAB is True.
+    """
+    if IN_COLAB:
+        filen = os.path.join(GOOGLE_DRIVE_OUTPUT_PATH, filen)
+    return filen
+
+
+def set_css_for_colab():
+    from IPython.display import HTML, display
+
+    # from https://stackoverflow.com/a/61401455/230523
+    display(
+        HTML(
+            """
+        <style>
+            pre { 
+                white-space: pre-wrap;
+            }
+        </style>
+        """
+        )
+    )

gjdutils/collections.py

@@ -0,0 +1,36 @@
+from collections import OrderedDict
+from typing import Any, Callable, Literal, Sequence, TypeVar
+
+T = TypeVar("T")
+
+
+def found_one(lst: Sequence[T]) -> T | Literal[False]:
+    if len(lst) == 0:
+        return False
+    elif len(lst) == 1:
+        found = lst[0]
+        assert found is not False, "Too confusing - we found something, but it's False"
+        return found
+    else:
+        return False
+
+
+def find_duplicates(lst: Sequence[T]) -> list[T]:
+    return [item for item in lst if lst.count(item) > 1]
+
+    from collections import OrderedDict
+
+
+# def uniquify(items: Sequence[T], key: Callable[[T], Any] | None = None) -> list[T]:
+# this would be useful if you wanted to uniquify something non-hashable, but I couldn't get it to work
+# https://www.perplexity.ai/search/in-python-unique-version-of-a-5r0iCRlBSjm2Dv6HGLu_6g
+# return list(OrderedDict.fromkeys(map(key, items) if key else items))
+
+
+def uniquify(items: Sequence[T]) -> list[T]:
+    # https://www.perplexity.ai/search/unique-version-of-a-list-prese-qYpae.JBRDedvHdmEyOqfA
+    # seen = set()
+    # return [x for x in lst if not (x in seen or seen.add(x))]
+
+    # https://www.w3resource.com/python-exercises/list-advanced/python-list-advanced-exercise-8.php
+    return list(dict.fromkeys(items))

gjdutils/decorators.py

@@ -0,0 +1,34 @@
+from functools import wraps
+from rich.console import Console
+from typing import Callable, TypeVar, Any, cast
+
+console = Console()
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def console_print_doc(color: str = "blue") -> Callable[[F], F]:
+    """
+    A decorator that prints the docstring of a function when it starts running.
+    The entire docstring will be printed in the specified color.
+
+    Args:
+        color (str): Color for the docstring text. Defaults to "blue".
+
+    Example:
+        @console_print_doc(color="green")
+        def my_function():
+            "This entire docstring will be green"
+            pass
+    """
+
+    def decorator(func: F) -> F:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            if func.__doc__:
+                console.print(func.__doc__.strip(), style=color)
+            return func(*args, **kwargs)
+
+        return cast(F, wrapper)
+
+    return decorator