GJDutils 0.2.2__tar.gz → 0.3.0__tar.gz

{gjdutils-0.2.2 → gjdutils-0.3.0}/.gitignore

@@ -88,6 +88,7 @@ celerybeat-schedule
 
 # Environments
 .env
+.env.*
 .venv
 env/
 venv/

gjdutils-0.3.0/PKG-INFO

@@ -0,0 +1,273 @@
+Metadata-Version: 2.4
+Name: GJDutils
+Version: 0.3.0
+Summary: A collection of useful utility functions (basics, data science/AI, web development, etc)
+Project-URL: Homepage, https://github.com/gregdetre/gjdutils
+Project-URL: Repository, https://github.com/gregdetre/gjdutils
+Author-email: Greg Detre <greg@gregdetre.com>
+License-File: LICENSE
+Keywords: ai,data science,dates,llm,strings,utilities,web development
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: ipython
+Requires-Dist: jinja2
+Requires-Dist: pydantic
+Requires-Dist: python-dotenv
+Provides-Extra: all-no-dev
+Requires-Dist: azure-cognitiveservices-speech; extra == 'all-no-dev'
+Requires-Dist: bs4; extra == 'all-no-dev'
+Requires-Dist: cachetools; extra == 'all-no-dev'
+Requires-Dist: elevenlabs; extra == 'all-no-dev'
+Requires-Dist: google-cloud-texttospeech; extra == 'all-no-dev'
+Requires-Dist: google-cloud-translate; extra == 'all-no-dev'
+Requires-Dist: humanize; extra == 'all-no-dev'
+Requires-Dist: openai; extra == 'all-no-dev'
+Requires-Dist: openai-whisper; extra == 'all-no-dev'
+Requires-Dist: pendulum; extra == 'all-no-dev'
+Requires-Dist: pillow; extra == 'all-no-dev'
+Requires-Dist: playsound; extra == 'all-no-dev'
+Requires-Dist: pyaudio; extra == 'all-no-dev'
+Requires-Dist: pygame; extra == 'all-no-dev'
+Requires-Dist: python-vlc; extra == 'all-no-dev'
+Requires-Dist: speechrecognition; extra == 'all-no-dev'
+Provides-Extra: audio-lang
+Requires-Dist: azure-cognitiveservices-speech; extra == 'audio-lang'
+Requires-Dist: cachetools; extra == 'audio-lang'
+Requires-Dist: elevenlabs; extra == 'audio-lang'
+Requires-Dist: google-cloud-texttospeech; extra == 'audio-lang'
+Requires-Dist: google-cloud-translate; extra == 'audio-lang'
+Requires-Dist: openai-whisper; extra == 'audio-lang'
+Requires-Dist: playsound; extra == 'audio-lang'
+Requires-Dist: pyaudio; extra == 'audio-lang'
+Requires-Dist: pygame; extra == 'audio-lang'
+Requires-Dist: python-vlc; extra == 'audio-lang'
+Requires-Dist: speechrecognition; extra == 'audio-lang'
+Provides-Extra: dev
+Requires-Dist: black; extra == 'dev'
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: rich; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Requires-Dist: typer; extra == 'dev'
+Requires-Dist: wheel; extra == 'dev'
+Provides-Extra: dt
+Requires-Dist: humanize; extra == 'dt'
+Requires-Dist: pendulum; extra == 'dt'
+Provides-Extra: html-web
+Requires-Dist: bs4; extra == 'html-web'
+Provides-Extra: llm
+Requires-Dist: cachetools; extra == 'llm'
+Requires-Dist: openai; extra == 'llm'
+Requires-Dist: pillow; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# gjdutils
+
+A collection of useful utility functions (strings, dates, data science/AI, web development, types, etc).
+
+This is a smorgasbord of utility functions, patterns and convenient wrappers that I've found myself rewriting and reusing across multiple projects, gathered into one place.
+
+Probably many of these exist elsewhere in libraries - if so, please let me know, because I'd probably rather use something cleaner and better-maintained.
+
+
+## Highlights
+
+### Audio: convenient microphone voice recognition with Whisper, and text-to-speech using ElevenLabs
+```python
+from gjdutils.outloud_text_to_speech import outloud
+from gjdutils.voice_speechrecognition import recognise_speech
+
+# Record speech and play it back in a different voice
+text = recognise_speech("Say something!")  # Records from microphone until you press ENTER
+outloud(text, prog="elevenlabs", mp3_filen="recording.mp3", should_play=True)  # Plays back what you said
+```
+
+
+### Run shell commands clearly & conveniently
+```python
+from gjdutils.cmd import run_cmd
+from pathlib import Path
+
+# Get Python version and capture the output
+retcode, stdout, extra = run_cmd(
+    "python --version",  # you can also provide as a list-of-strings
+    before_msg="Checking Python version...",
+    fatal_msg="Some problem running Python",  # will show up in red
+    verbose=0,  # Run silently unless there's an error
+    **{"timeout": 5}  # Pass additional arguments to subprocess
+)
+print(f"Python version: {stdout}")  # e.g. "Python 3.9.7"
+print(f"Ran command: {extra['cmd_str']}")  # plus lots of other stuff stored
+```
+
+
+### Environment variables with type validation and helpful error messages
+```bash
+$ source scripts/export_all.sh .env
+```
+
+```python
+from gjdutils.env import get_env_var
+
+api_key = get_env_var("OPENAI_API_KEY")  # Ensures non-empty by default
+num_workers = get_env_var("NUM_WORKERS", typ=int)  # Validates and converts to int
+```
+
+### Strict Jinja templating that catches both undefined and unused variables
+```python
+from gjdutils.strings import jinja_render
+
+template = "{{name}} is {{age}} years old"
+context = {"name": "Bob", "unused": True}
+text = jinja_render(template, context)  # will fail both because `age` is missing and `unused` is superfluous
+```
+
+
+### Set random seeds across Python, NumPy, PyTorch for reproducibility
+```python
+from gjdutils.rand import set_seeds
+
+set_seeds(42)  # Sets seeds for random, numpy, torch if available
+```
+
+### Call Claude/OpenAI APIs with function calling, image analysis & JSON support
+```python
+from gjdutils.llms_claude import call_claude_gpt
+from gjdutils.llm_utils import image_to_base64
+
+response, extra = call_claude_gpt(
+    "What's in this image?",
+    image_filens=["path/to/image.jpg"],  # Can pass multiple images
+    temperature=0.001
+)
+```
+
+
+### Translate text between languages with Google Translate
+```python
+from gjdutils.google_translate import translate_text, detect_language
+
+# First detect the language
+text = "Bonjour le monde"
+lang, confidence = detect_language(text)  # Returns ("fr", 0.98)
+
+# Then translate to English
+english_text, _ = translate_text(text, lang_src_code=lang, lang_tgt_code="en")  # Returns "Hello world"
+```
+
+
+### Calculate text similarity using longest common substring analysis
+```python
+from gjdutils.strings import calc_proportion_longest_common_substring
+similarity = calc_proportion_longest_common_substring(["hello world", "hello there"])  # Returns ~0.45 for "hello" match
+```
+
+
+### Measure data uniformity & distribution with simple proportion analysis
+```python
+from gjdutils.dsci import calc_proportion_identical
+uniformity = calc_proportion_identical(['a', 'a', 'a', 'b'])  # Returns 0.75 (75% are 'a')
+```
+
+
+
+### Generate deterministic cache keys for complex Python objects
+```python
+from gjdutils.caching import generate_mckey
+cache_key = generate_mckey("myprefix", {"user_id": 123, "action": "login"})  # Creates deterministic cache key
+```
+
+
+### Generate consistent hashes for caching/comparison
+```python
+from gjdutils.hashing import hash_readable
+
+# Same input always produces same hash, even across sessions
+config = {"model": "gpt-4", "temperature": 0.7}
+cache_key = hash_readable(config)  # e.g. "8f4e5d3..."
+```
+
+
+### Pretty-print and process HTML with customizable indentation
+```python
+from gjdutils.html import prettify_html
+
+# Prettify a string of HTML (also useful for testing two HTML strings are identical without caring about whitespace)
+html = "<div><p>Hello</p><p>World</p></div>" # Also works with BeautifulSoup elements
+pretty = prettify_html(html, indent=4)  # Custom indentation
+print(pretty)
+# <div>
+#     <p>Hello</p>
+#     <p>World</p>
+# </div>
+```
+
+### Debug by printing local variables, excluding noise
+```python
+from gjdutils.misc import print_locals
+
+def my_function(x, y):
+    z = x + y
+    some_func = lambda x: x * 2
+    _internal = "temp"
+    # Print all local vars except functions and _prefixed
+    print_locals(locals(), ignore_functions=True, ignore_underscores=True)
+    # Output: {'x': 1, 'y': 2, 'z': 3}
+```
+
+### Generate readable random IDs (no confusing characters)
+```python
+from gjdutils.rand import gen_readable_rand_id
+
+# Generate random ID without confusing chars (0/O, 1/I/l, etc)
+uid = gen_readable_rand_id(n=7)  # e.g. "k8m5p3h"
+```
+
+----
+
+## Installation
+
+```bash
+pip install gjdutils
+```
+
+For optional features:
+```bash
+pip install "gjdutils[dt]"   # Date/time utilities
+pip install "gjdutils[llm]"  # AI/LLM integrations
+pip install "gjdutils[audio_lang]"  # Speech/translation, language-related
+pip install "gjdutils[html_web]"    # Web scraping
+
+pip install "gjdutils[dev]"  # Development tools (for tweaking `gjdutils` itself, e.g. pytest)
+
+# Install all optional dependencies at once (except `dev`, which is used for developing `gjdutils` itself)
+pip install "gjdutils[all_no_dev]"
+```
+
+### Development Setup
+
+If you're developing `gjdutils` itself, install in editable mode:
+```bash
+# (Assumes you have already setup your virtualenv)
+# from the gjdutils root directory
+pip install -e ".[all_no_dev, dev]"     # Install all optional dependencies
+```
+
+Or if you're feeling lazy and can't remember that command, just use:
+
+```bash
+python scripts/install_all_dev_dependencies.py
+```
+
+
+### Adding to requirements.txt
+
+To add to your `requirements.txt` in editable mode, e.g. to install all optional dependencies:
+```text
+-e "git+https://github.com/gregdetre/gjdutils.git#egg=gjdutils[all_no_dev]"
+```

gjdutils-0.3.0/README.md

@@ -0,0 +1,205 @@
+# gjdutils
+
+A collection of useful utility functions (strings, dates, data science/AI, web development, types, etc).
+
+This is a smorgasbord of utility functions, patterns and convenient wrappers that I've found myself rewriting and reusing across multiple projects, gathered into one place.
+
+Probably many of these exist elsewhere in libraries - if so, please let me know, because I'd probably rather use something cleaner and better-maintained.
+
+
+## Highlights
+
+### Audio: convenient microphone voice recognition with Whisper, and text-to-speech using ElevenLabs
+```python
+from gjdutils.outloud_text_to_speech import outloud
+from gjdutils.voice_speechrecognition import recognise_speech
+
+# Record speech and play it back in a different voice
+text = recognise_speech("Say something!")  # Records from microphone until you press ENTER
+outloud(text, prog="elevenlabs", mp3_filen="recording.mp3", should_play=True)  # Plays back what you said
+```
+
+
+### Run shell commands clearly & conveniently
+```python
+from gjdutils.cmd import run_cmd
+from pathlib import Path
+
+# Get Python version and capture the output
+retcode, stdout, extra = run_cmd(
+    "python --version",  # you can also provide as a list-of-strings
+    before_msg="Checking Python version...",
+    fatal_msg="Some problem running Python",  # will show up in red
+    verbose=0,  # Run silently unless there's an error
+    **{"timeout": 5}  # Pass additional arguments to subprocess
+)
+print(f"Python version: {stdout}")  # e.g. "Python 3.9.7"
+print(f"Ran command: {extra['cmd_str']}")  # plus lots of other stuff stored
+```
+
+
+### Environment variables with type validation and helpful error messages
+```bash
+$ source scripts/export_all.sh .env
+```
+
+```python
+from gjdutils.env import get_env_var
+
+api_key = get_env_var("OPENAI_API_KEY")  # Ensures non-empty by default
+num_workers = get_env_var("NUM_WORKERS", typ=int)  # Validates and converts to int
+```
+
+### Strict Jinja templating that catches both undefined and unused variables
+```python
+from gjdutils.strings import jinja_render
+
+template = "{{name}} is {{age}} years old"
+context = {"name": "Bob", "unused": True}
+text = jinja_render(template, context)  # will fail both because `age` is missing and `unused` is superfluous
+```
+
+
+### Set random seeds across Python, NumPy, PyTorch for reproducibility
+```python
+from gjdutils.rand import set_seeds
+
+set_seeds(42)  # Sets seeds for random, numpy, torch if available
+```
+
+### Call Claude/OpenAI APIs with function calling, image analysis & JSON support
+```python
+from gjdutils.llms_claude import call_claude_gpt
+from gjdutils.llm_utils import image_to_base64
+
+response, extra = call_claude_gpt(
+    "What's in this image?",
+    image_filens=["path/to/image.jpg"],  # Can pass multiple images
+    temperature=0.001
+)
+```
+
+
+### Translate text between languages with Google Translate
+```python
+from gjdutils.google_translate import translate_text, detect_language
+
+# First detect the language
+text = "Bonjour le monde"
+lang, confidence = detect_language(text)  # Returns ("fr", 0.98)
+
+# Then translate to English
+english_text, _ = translate_text(text, lang_src_code=lang, lang_tgt_code="en")  # Returns "Hello world"
+```
+
+
+### Calculate text similarity using longest common substring analysis
+```python
+from gjdutils.strings import calc_proportion_longest_common_substring
+similarity = calc_proportion_longest_common_substring(["hello world", "hello there"])  # Returns ~0.45 for "hello" match
+```
+
+
+### Measure data uniformity & distribution with simple proportion analysis
+```python
+from gjdutils.dsci import calc_proportion_identical
+uniformity = calc_proportion_identical(['a', 'a', 'a', 'b'])  # Returns 0.75 (75% are 'a')
+```
+
+
+
+### Generate deterministic cache keys for complex Python objects
+```python
+from gjdutils.caching import generate_mckey
+cache_key = generate_mckey("myprefix", {"user_id": 123, "action": "login"})  # Creates deterministic cache key
+```
+
+
+### Generate consistent hashes for caching/comparison
+```python
+from gjdutils.hashing import hash_readable
+
+# Same input always produces same hash, even across sessions
+config = {"model": "gpt-4", "temperature": 0.7}
+cache_key = hash_readable(config)  # e.g. "8f4e5d3..."
+```
+
+
+### Pretty-print and process HTML with customizable indentation
+```python
+from gjdutils.html import prettify_html
+
+# Prettify a string of HTML (also useful for testing two HTML strings are identical without caring about whitespace)
+html = "<div><p>Hello</p><p>World</p></div>" # Also works with BeautifulSoup elements
+pretty = prettify_html(html, indent=4)  # Custom indentation
+print(pretty)
+# <div>
+#     <p>Hello</p>
+#     <p>World</p>
+# </div>
+```
+
+### Debug by printing local variables, excluding noise
+```python
+from gjdutils.misc import print_locals
+
+def my_function(x, y):
+    z = x + y
+    some_func = lambda x: x * 2
+    _internal = "temp"
+    # Print all local vars except functions and _prefixed
+    print_locals(locals(), ignore_functions=True, ignore_underscores=True)
+    # Output: {'x': 1, 'y': 2, 'z': 3}
+```
+
+### Generate readable random IDs (no confusing characters)
+```python
+from gjdutils.rand import gen_readable_rand_id
+
+# Generate random ID without confusing chars (0/O, 1/I/l, etc)
+uid = gen_readable_rand_id(n=7)  # e.g. "k8m5p3h"
+```
+
+----
+
+## Installation
+
+```bash
+pip install gjdutils
+```
+
+For optional features:
+```bash
+pip install "gjdutils[dt]"   # Date/time utilities
+pip install "gjdutils[llm]"  # AI/LLM integrations
+pip install "gjdutils[audio_lang]"  # Speech/translation, language-related
+pip install "gjdutils[html_web]"    # Web scraping
+
+pip install "gjdutils[dev]"  # Development tools (for tweaking `gjdutils` itself, e.g. pytest)
+
+# Install all optional dependencies at once (except `dev`, which is used for developing `gjdutils` itself)
+pip install "gjdutils[all_no_dev]"
+```
+
+### Development Setup
+
+If you're developing `gjdutils` itself, install in editable mode:
+```bash
+# (Assumes you have already setup your virtualenv)
+# from the gjdutils root directory
+pip install -e ".[all_no_dev, dev]"     # Install all optional dependencies
+```
+
+Or if you're feeling lazy and can't remember that command, just use:
+
+```bash
+python scripts/install_all_dev_dependencies.py
+```
+
+
+### Adding to requirements.txt
+
+To add to your `requirements.txt` in editable mode, e.g. to install all optional dependencies:
+```text
+-e "git+https://github.com/gregdetre/gjdutils.git#egg=gjdutils[all_no_dev]"
+```

gjdutils-0.3.0/__VERSION__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{gjdutils-0.2.2 → gjdutils-0.3.0}/pyproject.toml

@@ -37,11 +37,14 @@ audio_lang = [
     "elevenlabs",
     # for playing mp3
     "playsound",
+    # for voice_speechrecognition Microphone
+    "pyaudio",
     # for playing mp3
     "pygame",
     # for playing mp3
     "python-vlc",
     "SpeechRecognition",
+    "openai-whisper",  # for OpenAI Whisper speech recognition
 ]
 dev = [
     "black",

gjdutils-0.3.0/scripts/check_git_clean.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+from rich.console import Console
+from gjdutils.shell import fatal_error_msg
+from gjdutils.cmd import run_cmd
+
+console = Console()
+
+
+def check_git_status():
+    """Check if git working directory is clean."""
+    # Check for unstaged changes
+    retcode, stdout, _ = run_cmd("git diff --quiet", check=False)
+    if retcode != 0:
+        _, diff_output, _ = run_cmd("git --no-pager diff --stat")
+        fatal_error_msg("Unstaged changes present:\n" + diff_output)
+
+    # Check for staged but uncommitted changes
+    retcode, stdout, _ = run_cmd("git diff --cached --quiet", check=False)
+    if retcode != 0:
+        _, diff_output, _ = run_cmd("git --no-pager diff --cached --stat")
+        fatal_error_msg("Uncommitted staged changes present:\n" + diff_output)
+
+    # Check for untracked files
+    _, untracked, _ = run_cmd("git ls-files --others --exclude-standard")
+    if untracked.strip():
+        fatal_error_msg(f"Untracked files present:\n{untracked}")
+
+    console.print("[green]Git: clean[/green]")
+
+
+def main():
+    console.rule("[yellow]Checking Git Status")
+    check_git_status()
+
+
+if __name__ == "__main__":
+    main()

{gjdutils-0.2.2 → gjdutils-0.3.0}/scripts/deploy_pypiprod.py

@@ -13,6 +13,8 @@ from gjdutils.pypi_build import (
     upload_to_pypi,
 )
 from gjdutils.shell import fatal_error_msg
+from gjdutils.cmd import run_cmd
+from scripts.check_git_clean import check_git_status
 
 console = Console()
 
@@ -20,6 +22,9 @@ console = Console()
 def main():
     console.rule("[yellow]Starting Production PyPI Deployment")
 
+    # Check git status first
+    check_git_status()
+
     # Check if version already exists
     if check_version_exists(Version(__version__), pypi_env="prod"):
         fatal_error_msg(

gjdutils-0.3.0/scripts/export_all.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# https://stackoverflow.com/a/20909045/230523
+# e.g. source scripts/export_all.sh .env && echo $TESTME
+
+# Check if the script is being sourced
+if [ "$0" = "$BASH_SOURCE" ]; then
+    echo "Error: This script needs to be sourced. Please run: source $0 <env-file>"
+    exit 1
+fi
+
+if [ $# -eq 0 ]; then
+    echo "Error: Environment file path is required"
+    echo "Usage: source $0 <env-file>"
+    return 1
+fi
+
+ENV_FILE="$1"
+
+if [ ! -f "$ENV_FILE" ]; then
+    echo "Error: File '$ENV_FILE' does not exist"
+    return 1
+fi
+
+export $(grep -v '^#' "$ENV_FILE" | xargs)

gjdutils-0.3.0/scripts/install_all_dev_dependencies.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from gjdutils.cmd import run_cmd
+
+if __name__ == "__main__":
+    run_cmd("pip install -e '.[all_no_dev,dev]'", verbose=4)

gjdutils-0.2.2/src/gjdutils/cacheing.py → gjdutils-0.3.0/src/gjdutils/caching.py

@@ -2,6 +2,9 @@ from functools import update_wrapper
 import inspect
 import re
 
+from gjdutils.sorteddict import SortedDict
+from gjdutils.typ import isiterable
+
 
 # from abracadjabra.utils
 def generate_mckey(prefix, d):
@@ -42,13 +45,13 @@ def generate_mckey(prefix, d):
         Returns a SortedDict, with the keys sorted alphabetically.
 
         This might not be necessary, since I think the order
-        of a python dict's keys() is deterministic, but by
+        of a Python 3 dict's keys() is deterministic, but by
         sorting by dictionary keys, it's easier to know in
         advance what the generated MCKEY should look like.
         The idea is to ensure that no matter how D was
         created, you'll know what the key should be.
         """
-        sorted_d = SortedDict()
+        sorted_d = SortedDict(lambda x: x)
         for k in sorted(d.keys()):
             sorted_d[k] = d[k]
         return sorted_d

gjdutils-0.2.2/src/gjdutils/collections.py → gjdutils-0.3.0/src/gjdutils/collection_utils.py

@@ -1,10 +1,18 @@
-from collections import OrderedDict
-from typing import Any, Callable, Literal, Sequence, TypeVar
+from typing import Literal, Sequence, TypeVar
 
 T = TypeVar("T")
 
 
 def found_one(lst: Sequence[T]) -> T | Literal[False]:
+    """
+    e.g.
+    >>> found_one([])
+    False
+    >>> found_one([10])
+    10
+    >>> found_one([10, 20])
+    False
+    """
     if len(lst) == 0:
         return False
     elif len(lst) == 1:
@@ -16,9 +24,7 @@ def found_one(lst: Sequence[T]) -> T | Literal[False]:
 
 
 def find_duplicates(lst: Sequence[T]) -> list[T]:
-    return [item for item in lst if lst.count(item) > 1]
-
-    from collections import OrderedDict
+    return [item for item in set(lst) if lst.count(item) > 1]
 
 
 # def uniquify(items: Sequence[T], key: Callable[[T], Any] | None = None) -> list[T]:

{gjdutils-0.2.2 → gjdutils-0.3.0}/src/gjdutils/env.py

@@ -3,11 +3,17 @@ from pathlib import Path
 from typing import Any, TypeVar, cast
 from pydantic import StrictStr, TypeAdapter
 
+from gjdutils.vprint import vprint
+
 T = TypeVar("T")
 _processed_vars = set()
 
 
-def get_env_var(name: str, typ: Any = StrictStr) -> T:
+# You may find it useful to run `scripts/export_all.sh .env` to first
+# export all the variables in your .env file to your environment.
+
+
+def get_env_var(name: str, typ: Any = StrictStr, verbose: int = 0) -> T:
     """Get environment variable with type validation, e.g.
 
     OPENAI_API_KEY = get_env_var("OPENAI_API_KEY")
@@ -23,6 +29,8 @@ def get_env_var(name: str, typ: Any = StrictStr) -> T:
     Raises:
         ValueError: If variable is missing or fails validation
     """
+    vprint(1, f"Attempting to get environment variable: {name}")
+    vprint(2, f"Current environment variables: {list(os.environ.keys())}")
     try:
         value = os.environ[name]
         _processed_vars.add(name)