GJDutils 0.2.2__py3-none-any.whl

gjdutils/pypi_build.py

@@ -0,0 +1,112 @@
+"""Shared utilities for PyPI package building and testing."""
+
+from pathlib import Path
+import urllib.request
+import urllib.error
+import shutil
+from typing import Literal
+from rich.console import Console
+from rich.progress import track
+from packaging.version import Version
+
+from gjdutils.cmd import run_cmd
+from gjdutils import __version__
+
+console = Console()
+
+# Type for PyPI environment
+PyPIEnv = Literal["test", "prod"]
+
+
+def verify_installation(python_path: Path):
+    # Command: python -c "import gjdutils; print(gjdutils.__version__)"
+    retcode, installed_version, extra = run_cmd(
+        f'{python_path} -c "import gjdutils; print(gjdutils.__version__)"',
+        before_msg="Verify package installation by importing and checking version...",
+        fatal_msg="Failed to import gjdutils",
+    )
+    assert (
+        installed_version == __version__
+    ), f"Installed version {installed_version} does not match expected version {__version__}"
+    console.print(f"gjdutils version: {installed_version}")
+    return installed_version
+
+
+def check_install_optional_features(python_path: Path, *, from_test_pypi: bool = False):
+    """Test installation of optional feature sets."""
+    features = [
+        "audio_lang",
+        "dt",
+        "llm",
+        "html_web",
+    ]
+
+    for feature in track(features, description="Installing features"):
+        console.print(f"\nTesting feature set: {feature}", style="yellow")
+        if from_test_pypi:
+            cmd = f"{python_path} -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ gjdutils"
+        else:
+            cmd = f"{python_path} -m pip install '.[{feature}]'"
+        run_cmd(
+            cmd,
+            before_msg=f"Installing feature set: {feature}...",
+            fatal_msg=f"Failed to install {feature} feature",
+        )
+        console.print(f"[green]Successfully installed {feature} feature[/green]")
+
+
+def check_version_exists(version: Version, pypi_env: PyPIEnv) -> bool:
+    """Check if version already exists on specified PyPI environment.
+
+    Args:
+        version: Version string to check (must be valid semantic version)
+        pypi_env: PyPI environment to check ("test" or "prod")
+    """
+    base_url = {
+        "test": "https://test.pypi.org",
+        "prod": "https://pypi.org",
+    }[pypi_env]
+    try:
+        url = f"{base_url}/pypi/gjdutils/{str(version)}/json"
+        urllib.request.urlopen(url)
+        return True
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return False
+        raise  # Re-raise other HTTP errors
+
+
+def clean_build_dirs():
+    """Clean build directories (dist/ and build/)."""
+    # Command: rm -rf dist/ build/
+    shutil.rmtree("dist", ignore_errors=True)
+    shutil.rmtree("build", ignore_errors=True)
+
+
+def build_package():
+    """Build package with python -m build."""
+    return run_cmd(
+        "python -m build",
+        before_msg="Building package...",
+        fatal_msg="Failed to build package",
+    )
+
+
+def upload_to_pypi(pypi_env: PyPIEnv):
+    """Upload package to specified PyPI environment.
+
+    Args:
+        pypi_env: PyPI environment to upload to ("test" or "prod")
+    """
+    if pypi_env == "test":
+        cmd = "twine upload -r testpypi dist/*"
+    elif pypi_env == "prod":
+        cmd = "twine upload dist/*"
+    else:
+        raise ValueError(f"Invalid PyPI environment: {pypi_env}")
+
+    return run_cmd(
+        cmd,
+        before_msg=f"Uploading package to {pypi_env} PyPI...",
+        fatal_msg=f"Failed to upload to {pypi_env} PyPI",
+    )

gjdutils/pytest_utils.py

@@ -0,0 +1,24 @@
+import os
+import sys
+
+
+def is_pytest():
+    # https://stackoverflow.com/a/44595269/230523
+    #
+    # "Of course, this solution only works if the code you're trying to test does not use pytest itself.
+    mod_bool = "pytest" in sys.modules
+
+    # # from https://stackoverflow.com/a/58866220/230523
+    # #
+    # # "This method works only when an actual test is being run.
+    # # "This detection will not work when modules are imported during pytest collection.
+    # env_bool = "PYTEST_CURRENT_TEST" in os.environ
+    # if mod_bool and env_bool:
+    #     return True
+    # elif not mod_bool and not env_bool:
+    #     return False
+    # else:
+    #     raise RuntimeError(
+    #         "It's unclear whether we're in a unit test - it might be part of the pytest setup, or you might have imported pytest as part of your main codebase."
+    #     )
+    return mod_bool

gjdutils/rand.py

@@ -0,0 +1,65 @@
+import random
+from typing import Optional
+import uuid
+
+DEFAULT_RANDOM_SEED = 42
+READABLE_RAND_CHARS = "23456789abcdefghjkmnprstuvwxyz"
+
+
+def gen_uuid(n: int = 8):
+    # e.g. '161b58a4d8'
+    return str(uuid.uuid4()).replace("-", "")[:n]
+
+
+def gen_readable_uuid(n: int = 7, valid: Optional[list | str] = None):
+    # build az N digit random string with letters or numbers
+    # excluding o,0,i,1,l,q (too similar to g) for easy reading
+    #
+    # so this isn't officially a UUID
+    if valid is None:
+        valid = READABLE_RAND_CHARS
+    return "".join([random.choice(valid) for _ in range(n)])
+
+
+def assert_valid_readable_rand_id(
+    id_: str, nchars: Optional[int] = None, valid_chars: Optional[str] = None
+):
+    """
+    Check if ID_ is a valid readable random id, as generated by GEN_READABLE_UUID.
+    """
+    assert id_, "id_ cannot be empty"
+    if nchars is not None:
+        assert len(id_) == nchars, f"len({id_}) != {nchars}"
+    if valid_chars is None:
+        valid_chars = READABLE_RAND_CHARS
+    assert all([c in valid_chars for c in id_]), f"Invalid character in {id_}"
+
+
+def shuffle_copy(lst):
+    lst2 = lst.copy()
+    random.shuffle(lst2)
+    return lst2
+
+
+def set_seeds(seed: int = DEFAULT_RANDOM_SEED):
+    # TODO if SEED is None, make it dynamic
+    random.seed(seed)
+
+    try:
+        import numpy as np
+
+        np.random.seed(seed)
+    except ImportError:
+        pass
+
+    try:
+        import torch  # type: ignore
+
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed(seed)
+        torch.cuda.manual_seed_all(seed)
+        # If you are using cudnn (a GPU-accelerated library for deep neural networks)
+        torch.backends.cudnn.deterministic = True
+
+    except ImportError:
+        pass

gjdutils/regex.py

@@ -0,0 +1,78 @@
+import re
+from typing import Iterable
+
+# Based on:
+#   https://github.com/gregdetre/emacs-freex/blob/63523bf3b9032cc75b55ee28929dcdaf7714a419/freex_sqlalchemy.py#L653
+#   https://github.com/gregdetre/emacs-freex/blob/63523bf3b9032cc75b55ee28929dcdaf7714a419/freex_sqlalchemy.py#L713
+# That code has faster versions for much bigger lists of aliases, but this is fine for now.
+
+
+def compile_regex_for_matching_aliases(aliases: Iterable[str]):
+    """
+    Builds and compiles the implicit link regular expression.
+    """
+    # add .lower() here and elsewhere if you want case-insensitive
+    aliases = [re.escape(a) for a in aliases]
+
+    # ensure that 'jubba wubba' comes before 'jubba'
+    aliases.sort(reverse=True)
+
+    # build the regexp string to be single words
+    #
+    # it used to look like this, which worked nicely, unless
+    # there was a carriage return
+    alias_regex_str = "\\b" + "\\b|\\b".join(aliases) + "\\b"
+
+    # we want to deal with the possibility that there are 0
+    # or 1 spaces after a word, followed by 0 or 1 carriage
+    # returns, followed by zero or more spaces, which is
+    # what might happen if an implicit link was to span two
+    # lines in an indented paragraph. that's what ' ?\n? *' does
+    #
+    # the \\b that gets
+    # added later will ensure that there's a word boundary
+    # of *some* kind.
+    alias_regex_str = alias_regex_str.replace("\\ ", " ?\\\n? *")
+
+    # for ages, it wasn't matching things like 'Smith &
+    # Jones (2006)', because there was some problem with the
+    # parentheses. i eventually realized that it matched the
+    # first, but not the second parenthesis, because (i
+    # think) the parenthesis was screwing with the \b (bare
+    # word) separator. if you remove all the bare word
+    # separators that follow closing parentheses, the sun
+    # comes back out again
+    alias_regex_str = alias_regex_str.replace(")\\b", ")")
+
+    # compile the regexp
+    # impLinkRegexp = re.compile(aliasRegexpStr, re.IGNORECASE|re.MULTILINE)
+    compiled_regex = re.compile(alias_regex_str, re.MULTILINE)
+    return compiled_regex
+
+
+def find_matchranges_for_aliases(
+    compiled_regex_of_aliases: re.Pattern, txt_to_search: str
+):
+    """
+    Return a list of (beg,end) tuples for all the matching implicit
+    links in the provided string.
+    """
+    # get the start and endpoints for the matchranges
+    matchranges = [
+        list(match.span())
+        for match in compiled_regex_of_aliases.finditer(txt_to_search)
+    ]
+
+    # return the matchranges
+    return matchranges
+
+
+# txt = """There was a young European man called Friedrich Nietzsche, who most went by just "Nietzsche" (and but never 'Friedrich'). He was a friend of Little Hans and Little Richard but he was not little or Little."""
+
+# aliasRegexpStr, impLinkRegexp = update_implicit_link_regexp_original(all_aliases)
+# match_ranges = get_all_matching_implicit_links_original(impLinkRegexp, txt)
+
+# for match_range in match_ranges:
+#     match_txt = txt[match_range[0]:match_range[1]]
+#     matched_name = name_from_alias[match_txt]
+#     print('MATCHED: ', match_txt, ' <- NAME: ', matched_name, sep='')

gjdutils/requirements_dev.txt

@@ -0,0 +1,2 @@
+requirements_dev.txt
+# playsound

gjdutils/runtime.py

@@ -0,0 +1,19 @@
+import sys
+
+
+def in_notebook() -> bool:
+    # from https://stackoverflow.com/q/15411967
+    try:
+        shell = get_ipython().__class__.__name__  # type: ignore
+        if shell == "ZMQInteractiveShell":
+            return True  # Jupyter notebook or qtconsole
+        elif shell == "TerminalInteractiveShell":
+            return False  # Terminal running IPython
+        else:
+            return False  # Other type (?)
+    except NameError:
+        return False  # Probably standard Python interpreter
+
+
+def in_colab():
+    return "google.colab" in sys.modules

gjdutils/sets.py

@@ -0,0 +1,5 @@
+def assert_sets_identical(set1: set, set2: set):
+    # assert that the sets are the same, and print the difference if not
+    if set1 != set2:
+        assert not set1 - set2, f"Set 1 has extra elements: {set1 - set2}"
+        assert not set2 - set1, f"Set 2 has extra elements: {set2 - set1}"

gjdutils/shell.py

@@ -0,0 +1,69 @@
+"""Shell and command-line utilities."""
+
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import venv
+from contextlib import contextmanager
+from typing import Optional, Union
+
+
+@contextmanager
+def temp_venv(path: Union[str, Path]):
+    """Create and manage a temporary virtualenv.
+
+    Args:
+        path: Path where the virtualenv should be created
+
+    Yields:
+        Path to the Python executable in the virtualenv
+
+    Example:
+        ```python
+        with temp_venv("/tmp/my-venv") as python_path:
+            run_cmd([python_path, "-m", "pip", "install", "some-package"])
+        ```
+    """
+    path = Path(path)
+
+    # Clean up any existing venv first
+    if path.exists():
+        shutil.rmtree(path)
+
+    venv.create(path, with_pip=True)
+
+    # Get the correct python executable path for this venv
+    if sys.platform == "win32":
+        python_path = path / "Scripts" / "python.exe"
+    else:
+        python_path = path / "bin" / "python"
+
+    try:
+        yield python_path
+    finally:
+        if path.exists():
+            shutil.rmtree(path)
+
+
+def fatal_error_msg(msg: str, stderr: Optional[str] = None) -> None:
+    """Print a fatal error message and exit with code 1.
+
+    Args:
+        msg: The error message to display
+        stderr: Optional stderr output to display after the message
+
+    Example:
+        ```python
+        if result.returncode != 0:
+            fatal_error_msg("Failed to build package", result.stderr)
+        ```
+    """
+    from rich.console import Console
+
+    console = Console()
+
+    console.print(f"[red]{msg}[/red]")
+    if stderr:
+        console.print(stderr)
+    sys.exit(1)

gjdutils/sorteddict.py

@@ -0,0 +1,34 @@
+# This dict is always sorted by `sortfunc` when you iterate over it.
+#
+# For example, if you have a dict of str -> Nodes, and you want to iterate over them in
+# order of their `idx` attribute, you can do:
+#
+#   d = SortedDict[str,Node](lambda node: node.idx)
+#   [insert, remove, Nodes]
+#   d.values()  # this will always be sorted by idx
+#
+# based on code from ChatGPT
+
+from typing import Any, Callable, Dict, Generic, TypeVar
+
+
+# Define type variables for keys and values
+K = TypeVar("K")
+V = TypeVar("V")
+
+
+class SortedDict(Dict[K, V], Generic[K, V]):
+    def __init__(self, sortfunc: Callable[[V], Any], *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.sortfunc = sortfunc
+        self._reorder()
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        self._reorder()
+
+    def _reorder(self):
+        sorted_items = sorted(self.items(), key=lambda item: self.sortfunc(item[1]))
+        super().clear()
+        for key, value in sorted_items:
+            super().__setitem__(key, value)

gjdutils/stopwatch.py

@@ -0,0 +1,79 @@
+########################################################
+# from http://effbot.org/librarybook/timing.htm
+# File: timing-example-2.py
+#
+# copied from ~/fmri/distpat/trunk/users/greg/context/time_context.py
+
+"""
+This is my wrapper for the time module. There's probably an
+easier way to time the duration of things, but when I looked
+into timing stuff, this was the best I could come up with...
+
+To use:
+
+    t = Stopwatch()
+    # t.start()
+    
+    # do something
+
+    elapsed = t.finish()
+"""
+
+import time
+
+
+class Stopwatch:
+    """
+    Creates stopwatch timer objects.
+    """
+
+    # stores the time the stopwatch was started
+    t0 = None
+
+    # stores the time the stopwatch was last looked at
+    t1 = None
+
+    def __init__(self, do_start: bool = True):
+        self.t0 = 0
+        self.t1 = 0
+        if do_start:
+            self.start()
+
+    def start(self):
+        """
+        Stores the current time in t0.
+        """
+
+        self.t0 = time.time()
+
+    def finish(self, milliseconds: bool = True):
+        """
+        Returns the elapsed duration in milliseconds. This
+        stores the current time in t1, and calculates the
+        difference between t0 (the stored start time) and
+        t1, so if you call this multiple times, you'll get a
+        larger answer each time.
+
+        You have to call this in order to update t1.
+        """
+
+        self.t1 = time.time()
+        return self.milli() if milliseconds else self.seconds()
+
+    def seconds(self):
+        """
+        Returns t1 - t0 in seconds. Does not update t1.
+        """
+        return int(self.t1 - self.t0)
+
+    def milli(self):
+        """
+        Returns t1 - t0 in milliseconds. Does not update t1.
+        """
+        return int((self.t1 - self.t0) * 1000)
+
+    def micro(self):
+        """
+        Returns t1 - t0 in microseconds. Does not update t1.
+        """
+        return int((self.t1 - self.t0) * 1000000)