GJDutils 0.3.0__tar.gz → 0.3.4__tar.gz

{gjdutils-0.3.0 → gjdutils-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GJDutils
-Version: 0.3.0
+Version: 0.3.4
 Summary: A collection of useful utility functions (basics, data science/AI, web development, etc)
 Project-URL: Homepage, https://github.com/gregdetre/gjdutils
 Project-URL: Repository, https://github.com/gregdetre/gjdutils
@@ -19,15 +19,18 @@ Requires-Dist: jinja2
 Requires-Dist: pydantic
 Requires-Dist: python-dotenv
 Provides-Extra: all-no-dev
+Requires-Dist: anthropic; extra == 'all-no-dev'
 Requires-Dist: azure-cognitiveservices-speech; extra == 'all-no-dev'
 Requires-Dist: bs4; extra == 'all-no-dev'
-Requires-Dist: cachetools; extra == 'all-no-dev'
 Requires-Dist: elevenlabs; extra == 'all-no-dev'
 Requires-Dist: google-cloud-texttospeech; extra == 'all-no-dev'
 Requires-Dist: google-cloud-translate; extra == 'all-no-dev'
 Requires-Dist: humanize; extra == 'all-no-dev'
+Requires-Dist: lxml; extra == 'all-no-dev'
+Requires-Dist: numpy; extra == 'all-no-dev'
 Requires-Dist: openai; extra == 'all-no-dev'
 Requires-Dist: openai-whisper; extra == 'all-no-dev'
+Requires-Dist: pandas; extra == 'all-no-dev'
 Requires-Dist: pendulum; extra == 'all-no-dev'
 Requires-Dist: pillow; extra == 'all-no-dev'
 Requires-Dist: playsound; extra == 'all-no-dev'
@@ -37,7 +40,6 @@ Requires-Dist: python-vlc; extra == 'all-no-dev'
 Requires-Dist: speechrecognition; extra == 'all-no-dev'
 Provides-Extra: audio-lang
 Requires-Dist: azure-cognitiveservices-speech; extra == 'audio-lang'
-Requires-Dist: cachetools; extra == 'audio-lang'
 Requires-Dist: elevenlabs; extra == 'audio-lang'
 Requires-Dist: google-cloud-texttospeech; extra == 'audio-lang'
 Requires-Dist: google-cloud-translate; extra == 'audio-lang'
@@ -50,18 +52,23 @@ Requires-Dist: speechrecognition; extra == 'audio-lang'
 Provides-Extra: dev
 Requires-Dist: black; extra == 'dev'
 Requires-Dist: build; extra == 'dev'
+Requires-Dist: flake8; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: rich; extra == 'dev'
 Requires-Dist: twine; extra == 'dev'
 Requires-Dist: typer; extra == 'dev'
 Requires-Dist: wheel; extra == 'dev'
+Provides-Extra: dsci
+Requires-Dist: numpy; extra == 'dsci'
+Requires-Dist: pandas; extra == 'dsci'
 Provides-Extra: dt
 Requires-Dist: humanize; extra == 'dt'
 Requires-Dist: pendulum; extra == 'dt'
 Provides-Extra: html-web
 Requires-Dist: bs4; extra == 'html-web'
+Requires-Dist: lxml; extra == 'html-web'
 Provides-Extra: llm
-Requires-Dist: cachetools; extra == 'llm'
+Requires-Dist: anthropic; extra == 'llm'
 Requires-Dist: openai; extra == 'llm'
 Requires-Dist: pillow; extra == 'llm'
 Description-Content-Type: text/markdown
@@ -74,6 +81,8 @@ This is a smorgasbord of utility functions, patterns and convenient wrappers tha
 
 Probably many of these exist elsewhere in libraries - if so, please let me know, because I'd probably rather use something cleaner and better-maintained.
 
+Caveat emptor: some of these are old, and haven't been tested in a while.
+
 
 ## Highlights
 
@@ -97,7 +106,7 @@ from pathlib import Path
 retcode, stdout, extra = run_cmd(
     "python --version",  # you can also provide as a list-of-strings
     before_msg="Checking Python version...",
-    fatal_msg="Some problem running Python",  # will show up in red
+    fatal_msg="Some problem running Python",  # will show up in red, sys.exit(1)
     verbose=0,  # Run silently unless there's an error
     **{"timeout": 5}  # Pass additional arguments to subprocess
 )
@@ -108,7 +117,7 @@ print(f"Ran command: {extra['cmd_str']}")  # plus lots of other stuff stored
 
 ### Environment variables with type validation and helpful error messages
 ```bash
-$ source scripts/export_all.sh .env
+$ python -m gjdutils.scripts.export_all .env
 ```
 
 ```python
@@ -135,6 +144,7 @@ from gjdutils.rand import set_seeds
 set_seeds(42)  # Sets seeds for random, numpy, torch if available
 ```
 
+
 ### Call Claude/OpenAI APIs with function calling, image analysis & JSON support
 ```python
 from gjdutils.llms_claude import call_claude_gpt
@@ -142,7 +152,7 @@ from gjdutils.llm_utils import image_to_base64
 
 response, extra = call_claude_gpt(
     "What's in this image?",
-    image_filens=["path/to/image.jpg"],  # Can pass multiple images
+    image_filens=["path/to/image.jpg"],
     temperature=0.001
 )
 ```
@@ -175,11 +185,10 @@ uniformity = calc_proportion_identical(['a', 'a', 'a', 'b'])  # Returns 0.75 (75
 ```
 
 
-
 ### Generate deterministic cache keys for complex Python objects
 ```python
 from gjdutils.caching import generate_mckey
-cache_key = generate_mckey("myprefix", {"user_id": 123, "action": "login"})  # Creates deterministic cache key
+cache_key = generate_mckey("myprefix", {"a": 100, "b": "foo"})  # Creates deterministic cache key
 ```
 
 
@@ -188,7 +197,7 @@ cache_key = generate_mckey("myprefix", {"user_id": 123, "action": "login"})  # C
 from gjdutils.hashing import hash_readable
 
 # Same input always produces same hash, even across sessions
-config = {"model": "gpt-4", "temperature": 0.7}
+config = {"foo": "bar"}
 cache_key = hash_readable(config)  # e.g. "8f4e5d3..."
 ```
 
@@ -207,6 +216,7 @@ print(pretty)
 # </div>
 ```
 
+
 ### Debug by printing local variables, excluding noise
 ```python
 from gjdutils.misc import print_locals
@@ -220,6 +230,7 @@ def my_function(x, y):
     # Output: {'x': 1, 'y': 2, 'z': 3}
 ```
 
+
 ### Generate readable random IDs (no confusing characters)
 ```python
 from gjdutils.rand import gen_readable_rand_id
@@ -261,7 +272,7 @@ pip install -e ".[all_no_dev, dev]"     # Install all optional dependencies
 Or if you're feeling lazy and can't remember that command, just use:
 
 ```bash
-python scripts/install_all_dev_dependencies.py
+python -m gjdutils.scripts.install_all_dev_dependencies
 ```
 
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/README.md

@@ -6,6 +6,8 @@ This is a smorgasbord of utility functions, patterns and convenient wrappers tha
 
 Probably many of these exist elsewhere in libraries - if so, please let me know, because I'd probably rather use something cleaner and better-maintained.
 
+Caveat emptor: some of these are old, and haven't been tested in a while.
+
 
 ## Highlights
 
@@ -29,7 +31,7 @@ from pathlib import Path
 retcode, stdout, extra = run_cmd(
     "python --version",  # you can also provide as a list-of-strings
     before_msg="Checking Python version...",
-    fatal_msg="Some problem running Python",  # will show up in red
+    fatal_msg="Some problem running Python",  # will show up in red, sys.exit(1)
     verbose=0,  # Run silently unless there's an error
     **{"timeout": 5}  # Pass additional arguments to subprocess
 )
@@ -40,7 +42,7 @@ print(f"Ran command: {extra['cmd_str']}")  # plus lots of other stuff stored
 
 ### Environment variables with type validation and helpful error messages
 ```bash
-$ source scripts/export_all.sh .env
+$ python -m gjdutils.scripts.export_all .env
 ```
 
 ```python
@@ -67,6 +69,7 @@ from gjdutils.rand import set_seeds
 set_seeds(42)  # Sets seeds for random, numpy, torch if available
 ```
 
+
 ### Call Claude/OpenAI APIs with function calling, image analysis & JSON support
 ```python
 from gjdutils.llms_claude import call_claude_gpt
@@ -74,7 +77,7 @@ from gjdutils.llm_utils import image_to_base64
 
 response, extra = call_claude_gpt(
     "What's in this image?",
-    image_filens=["path/to/image.jpg"],  # Can pass multiple images
+    image_filens=["path/to/image.jpg"],
     temperature=0.001
 )
 ```
@@ -107,11 +110,10 @@ uniformity = calc_proportion_identical(['a', 'a', 'a', 'b'])  # Returns 0.75 (75
 ```
 
 
-
 ### Generate deterministic cache keys for complex Python objects
 ```python
 from gjdutils.caching import generate_mckey
-cache_key = generate_mckey("myprefix", {"user_id": 123, "action": "login"})  # Creates deterministic cache key
+cache_key = generate_mckey("myprefix", {"a": 100, "b": "foo"})  # Creates deterministic cache key
 ```
 
 
@@ -120,7 +122,7 @@ cache_key = generate_mckey("myprefix", {"user_id": 123, "action": "login"})  # C
 from gjdutils.hashing import hash_readable
 
 # Same input always produces same hash, even across sessions
-config = {"model": "gpt-4", "temperature": 0.7}
+config = {"foo": "bar"}
 cache_key = hash_readable(config)  # e.g. "8f4e5d3..."
 ```
 
@@ -139,6 +141,7 @@ print(pretty)
 # </div>
 ```
 
+
 ### Debug by printing local variables, excluding noise
 ```python
 from gjdutils.misc import print_locals
@@ -152,6 +155,7 @@ def my_function(x, y):
     # Output: {'x': 1, 'y': 2, 'z': 3}
 ```
 
+
 ### Generate readable random IDs (no confusing characters)
 ```python
 from gjdutils.rand import gen_readable_rand_id
@@ -193,7 +197,7 @@ pip install -e ".[all_no_dev, dev]"     # Install all optional dependencies
 Or if you're feeling lazy and can't remember that command, just use:
 
 ```bash
-python scripts/install_all_dev_dependencies.py
+python -m gjdutils.scripts.install_all_dev_dependencies
 ```
 
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/planning/250215_publishing_to_pypi.md

@@ -16,7 +16,7 @@
 1. Build and test package locally:
    ```bash
    # Option 1: Automated testing script (recommended)
-   ./scripts/check_locally.sh
+   python -m gjdutils.scripts.check_locally
    
    # Option 2: Manual steps
    # Clean any existing builds
@@ -29,7 +29,7 @@
 2. Test PyPI Deployment:
    ```bash
    # Option 1: Automated testing script (recommended)
-   ./scripts/check_test_pypi.sh
+   python -m gjdutils.scripts.check_test_pypi
    
    # Option 2: Manual steps
    # Upload to test.pypi.org

{gjdutils-0.3.0 → gjdutils-0.3.4}/pyproject.toml

@@ -1,8 +1,7 @@
 [project]
 # case-insensitive
 name = "GJDutils"
-# https://hatch.pypa.io/latest/config/metadata/
-dynamic = ["version"]
+version = "0.3.4"
 authors = [
   { name="Greg Detre", email="greg@gregdetre.com" },
 ]
@@ -31,7 +30,6 @@ dependencies = [
 [project.optional-dependencies]
 audio_lang = [
     "azure-cognitiveservices-speech",
-    "cachetools",
     "google-cloud-texttospeech",
     "google-cloud-translate",
     "elevenlabs",
@@ -48,12 +46,17 @@ audio_lang = [
 ]
 dev = [
     "black",
-    "pytest",
     "build",
-    "twine",
-    "wheel",
+    "flake8",
+    "pytest",
     "rich",  # for console output formatting
+    "twine",
     "typer",  # for CLI tools
+    "wheel",
+]
+dsci = [
+    "numpy",
+    "pandas",
 ]
 dt = [
     "humanize",
@@ -61,9 +64,10 @@ dt = [
 ]
 html_web = [
     "bs4",
+    "lxml",
 ]
 llm = [
-    "cachetools",
+    "anthropic",
     # "llm", no longer using this
     "openai",
     "pillow", # for images
@@ -71,6 +75,7 @@ llm = [
 
 all_no_dev = [
     "gjdutils[audio_lang]",
+    "gjdutils[dsci]",
     "gjdutils[dt]",
     "gjdutils[html_web]",
     "gjdutils[llm]",
@@ -94,6 +99,3 @@ exclude = [
     "gjdutils/todo/",
     "tests/fixme_*.py",
 ]
-
-[tool.hatch.version]
-path = "__VERSION__.py"

{gjdutils-0.3.0 → gjdutils-0.3.4}/scripts/check_locally.py

@@ -37,6 +37,9 @@ def install_and_test_package(python_path: Path, wheel_file: Path):
         fatal_msg="Failed to install package",
     )
 
+    # Install all optional dependencies first
+    check_install_optional_features(python_path)
+
     # Command: pip install ".[dev]"
     run_cmd(
         f"{python_path} -m pip install '.[dev]'",
@@ -65,7 +68,6 @@ def main():
         install_and_test_package(python_path, wheel_file)
         verify_installation(python_path)
         run_test_suite(python_path)
-        check_install_optional_features(python_path)
 
     console.print("\nLocal testing completed successfully!", style="green")
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/scripts/check_pypiprod.py

@@ -13,12 +13,15 @@ console = Console()
 
 def install_from_pypiprod(python_path: Path):
     # Command: pip install gjdutils
-    return run_cmd(
+    run_cmd(
         f"{python_path} -m pip install gjdutils",
         before_msg="Installing package from PyPI...",
         fatal_msg="Failed to install package from PyPI",
     )
 
+    # Install all optional dependencies
+    check_install_optional_features(python_path, from_test_pypi=False)
+
 
 def main():
     console.rule("[yellow]Starting Production PyPI package testing")
@@ -27,7 +30,6 @@ def main():
     with temp_venv(venv_path) as python_path:
         install_from_pypiprod(python_path)
         verify_installation(python_path)
-        check_install_optional_features(python_path, from_test_pypi=False)
 
     console.print("\nProduction PyPI testing completed successfully!", style="green")
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/scripts/check_pypitest.py

@@ -13,13 +13,16 @@ console = Console()
 
 def install_from_test_pypi(python_path: Path):
     # Command: pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ gjdutils
-    return run_cmd(
+    run_cmd(
         f"{python_path} -m pip install --index-url https://test.pypi.org/simple/ "
         "--extra-index-url https://pypi.org/simple/ gjdutils",
         before_msg="Installing package from Test PyPI...",
         fatal_msg="Failed to install package from Test PyPI",
     )
 
+    # Install all optional dependencies
+    check_install_optional_features(python_path, from_test_pypi=True)
+
 
 def main():
     console.rule("[yellow]Starting Test PyPI package testing")
@@ -28,7 +31,6 @@ def main():
     with temp_venv(venv_path) as python_path:
         install_from_test_pypi(python_path)
         verify_installation(python_path)
-        check_install_optional_features(python_path, from_test_pypi=True)
 
     console.print("\nTest PyPI testing completed successfully!", style="green")
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/scripts/deploy_all.py

@@ -2,11 +2,9 @@
 
 from rich.console import Console
 
-from check_locally import main as check_locally_main
-from deploy_pypitest import main as deploy_pypitest_main
-from check_pypitest import main as check_pypitest_main
-from deploy_pypiprod import main as deploy_pypiprod_main
-from check_pypiprod import main as check_pypiprod_main
+from .check_locally import main as check_locally_main
+from .deploy_pypitest import main as deploy_pypitest_main
+from .deploy_pypiprod import main as deploy_pypiprod_main
 
 console = Console()
 
@@ -16,9 +14,7 @@ def main():
 
     check_locally_main()
     deploy_pypitest_main()
-    check_pypitest_main()
     deploy_pypiprod_main()
-    check_pypiprod_main()
 
     console.print("\n[green]Full deployment process completed successfully! 🎉[/green]")
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/scripts/deploy_pypiprod.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 
 from rich.console import Console
-from pathlib import Path
 from packaging.version import Version
 
 from gjdutils import __version__
-from gjdutils.decorators import console_print_doc
 from gjdutils.pypi_build import (
     check_version_exists,
     clean_build_dirs,
@@ -13,8 +11,9 @@ from gjdutils.pypi_build import (
     upload_to_pypi,
 )
 from gjdutils.shell import fatal_error_msg
-from gjdutils.cmd import run_cmd
-from scripts.check_git_clean import check_git_status
+from .check_git_clean import check_git_status
+from .check_pypiprod import main as check_pypiprod_main
+
 
 console = Console()
 
@@ -28,7 +27,7 @@ def main():
     # Check if version already exists
     if check_version_exists(Version(__version__), pypi_env="prod"):
         fatal_error_msg(
-            f"Version {__version__} already exists on PyPI.\nPlease update __VERSION__.py to a new version number first."
+            f"Version {__version__} already exists on PyPI.\nPlease update the version in pyproject.toml to a new version number first."
         )
 
     # Confirm with user before proceeding
@@ -45,7 +44,8 @@ def main():
     upload_to_pypi(pypi_env="prod")
 
     console.print("\n[green]Deployment to Production PyPI completed![/green]")
-    console.print("Run ./scripts/check_pypiprod.py to verify the deployment")
+
+    check_pypiprod_main()
 
 
 if __name__ == "__main__":

{gjdutils-0.3.0 → gjdutils-0.3.4}/scripts/deploy_pypitest.py

@@ -5,13 +5,13 @@ from pathlib import Path
 from packaging.version import Version
 
 from gjdutils import __version__
-from gjdutils.decorators import console_print_doc
 from gjdutils.pypi_build import (
     check_version_exists,
     clean_build_dirs,
     build_package,
     upload_to_pypi,
 )
+from .check_pypitest import main as check_pypitest_main
 from gjdutils.shell import fatal_error_msg
 
 console = Console()
@@ -23,7 +23,7 @@ def main():
     # Check if version already exists
     if check_version_exists(Version(__version__), pypi_env="test"):
         fatal_error_msg(
-            f"Version {__version__} already exists on Test PyPI.\nPlease update __VERSION__.py to a new version number first."
+            f"Version {__version__} already exists on Test PyPI.\nPlease update the version in pyproject.toml to a new version number first."
         )
 
     # Execute deployment steps
@@ -32,7 +32,8 @@ def main():
     upload_to_pypi(pypi_env="test")
 
     console.print("\n[green]Deployment to Test PyPI completed![/green]")
-    console.print("Run ./scripts/check_pypitest.py to verify the deployment")
+
+    check_pypitest_main()
 
 
 if __name__ == "__main__":

gjdutils-0.3.4/src/gjdutils/__init__.py

@@ -0,0 +1,7 @@
+"""
+GJDutils - A collection of useful utility functions
+"""
+
+from gjdutils.pypi_build import get_version
+
+__version__ = get_version()

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/caching.py

@@ -2,7 +2,6 @@ from functools import update_wrapper
 import inspect
 import re
 
-from gjdutils.sorteddict import SortedDict
 from gjdutils.typ import isiterable
 
 
@@ -51,7 +50,7 @@ def generate_mckey(prefix, d):
         The idea is to ensure that no matter how D was
         created, you'll know what the key should be.
         """
-        sorted_d = SortedDict(lambda x: x)
+        sorted_d = dict()
         for k in sorted(d.keys()):
             sorted_d[k] = d[k]
         return sorted_d

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/dsci.py

@@ -1,16 +1,16 @@
 from collections import Counter
 import itertools
-import numpy as np
-import pandas as pd
-import random
-from scipy import spatial, sparse
 import sys
-from typing import Any, Iterable, Sequence
+from typing import Any, Sequence
 
+# keep this, because it makes sense for the user to be able to import from here
 from .rand import set_seeds
 
 
 def init_display_options():
+    import numpy as np
+    import pandas as pd
+
     # so that it's easier to see things in the terminal
     pd.set_option("display.max_rows", 10000)
     pd.set_option("display.max_columns", 1000)
@@ -74,6 +74,8 @@ def calc_normalised_std_tightness(vals: Sequence[float]) -> float:
     [19,  1, 40, 20] -> 0.31
     [ 9,  1, 70,  0] -> 0
     """
+    import numpy as np
+
     n = len(vals)
     if n == 0:
         raise Exception("Empty")
@@ -143,6 +145,8 @@ def convert_sim_dist_oneminus(val: float) -> float:
 
 
 def square_df_from_square(sq, features):
+    import pandas as pd
+
     df = pd.DataFrame(sq)
     # create a 'Feature' column
     df["Feature"] = features
@@ -154,6 +158,8 @@ def square_df_from_square(sq, features):
 
 
 def long_df_from_flat(dists_flat, features):
+    import pandas as pd
+
     combos = [(f1, f2) for f1, f2 in itertools.permutations(features, 2)]
     assert len(combos) == len(dists_flat)
     dists_triplet = [
@@ -164,6 +170,8 @@ def long_df_from_flat(dists_flat, features):
 
 
 def square_df_from_flat(dists_flat, features):
+    from scipy import spatial
+
     nFeatures = len(features)
     dists_sq = spatial.distance.squareform(np.array(dists_flat))
     assert dists_sq.shape == (nFeatures, nFeatures)

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/env.py

@@ -3,13 +3,13 @@ from pathlib import Path
 from typing import Any, TypeVar, cast
 from pydantic import StrictStr, TypeAdapter
 
-from gjdutils.vprint import vprint
+from gjdutils.print_utils import vprint
 
 T = TypeVar("T")
 _processed_vars = set()
 
 
-# You may find it useful to run `scripts/export_all.sh .env` to first
+# You may find it useful to run `python -m gjdutils.scripts.export_all .env` to first
 # export all the variables in your .env file to your environment.
 
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/files.py

@@ -3,6 +3,8 @@ from pathlib import Path
 from typing import Sequence
 
 from .cmd import run_cmd
+
+# keep this, because it makes sense for the user to be able to import from here
 from .strings import is_string, PathOrStr
 
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/functions.py

@@ -4,7 +4,7 @@ from typing import Any
 
 def func_name():
     # https://stackoverflow.com/a/13514318/230523
-    return inspect.currentframe().f_back.f_code.co_name
+    return inspect.currentframe().f_back.f_code.co_name  # type: ignore
 
 
 def variable_from_caller(var_name: str, frame_depth: int = 1) -> Any:
@@ -21,18 +21,18 @@ def variable_from_caller(var_name: str, frame_depth: int = 1) -> Any:
     try:
         # Go back the specified number of frames
         for _ in range(frame_depth + 1):
-            if frame.f_back is None:
+            if frame.f_back is None:  # type: ignore
                 raise ValueError(f"Cannot go back {frame_depth} frames")
-            frame = frame.f_back
+            frame = frame.f_back  # type: ignore
 
-        if var_name not in frame.f_locals:
-            caller_name = frame.f_code.co_name
+        if var_name not in frame.f_locals:  # type: ignore
+            caller_name = frame.f_code.co_name  # type: ignore
             raise ValueError(
                 f"Variable '{var_name}' not found in caller function '{caller_name}'. "
                 f"Make sure to define a '{var_name}' parameter or variable."
             )
 
-        return frame.f_locals[var_name]
+        return frame.f_locals[var_name]  # type: ignore
     finally:
         # Clean up circular references
         del frame

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/html.py

@@ -1,8 +1,7 @@
 from bs4 import BeautifulSoup
-from lxml import html as lxml_html
-from lxml.html import tostring, fromstring
-from lxml.etree import Element, _Element as ElementType
-from typing import Optional, Iterable, Union
+from lxml.html import tostring
+from lxml.etree import _Element as ElementType
+from typing import Optional, Union
 
 
 def remove_html_tags(html: str):
@@ -65,15 +64,15 @@ def prettify_html(
     if isinstance(html, list):
         # then we'll handle it as a string in a moment
         html = "".join(
-            [tostring(e, method="html").decode() for e in html]
+            [tostring(e, method="html").decode() for e in html]  # type: ignore
         )  #  type: ignore
     if isinstance(html, ElementType):
         # this will do some cleaning and fixing. but
         # you need document_fromstring() if you want to make sure
         # that it's a full html doc, e.g. with html, body
-        html = tostring(html, method="html").decode()
+        html = tostring(html, method="html").decode()  # type: ignore
 
-    soup = BeautifulSoup(html, "html.parser")
+    soup = BeautifulSoup(html, "html.parser")  # type: ignore
     # html2 = tostring(html, pretty_print=True, method="html").decode()  # type: ignore
     # the lxml pretty_print just isn't as good as BS4, e.g. with a list of elements
     # it wraps things in a div fragment, but the pretty-print of that isn't right

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/jsons.py

@@ -6,10 +6,12 @@ def jsonify(x):
     def json_dumper_robust(obj):
         try:
             return obj.toJSON()
-        except:
+        except AttributeError:
+            # Object doesn't have toJSON method, try string conversion
             try:
                 return str(obj)
-            except:
+            except (ValueError, TypeError):
+                # If string conversion fails, return None to skip this field
                 return None
 
     return json.dumps(x, sort_keys=True, indent=4, default=json_dumper_robust)
@@ -49,6 +51,8 @@ def to_json(
     Convert a list of dicts to a JSON string, with only the fields we want,
     and in the same order as FIELDS.
     """
+    if fields is None:
+        fields = []
     outs = []
     for inp in inps:
         if fields is None:
@@ -56,7 +60,7 @@ def to_json(
         # we want to make sure to return a dict with only the fields we want,
         # and in the same order as FIELDS
         out = {}
-        for k in fields:
+        for k in fields:  # type: ignore
             if skip_if_missing and (k not in inp):
                 continue
             v = inp[k]  # will error if missing and !SKIP_IF_MISSING

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/llm_utils.py

@@ -33,9 +33,9 @@ def proc_llm_out_json(s: str):
         s = s[7:-3]
     try:
         j = json.loads(s)
-    except:
-        print(s)
-        raise Exception("Could not parse JSON from LLM output")
+    except json.JSONDecodeError:
+        print("Failed to parse JSON:", s)
+        raise
     return j
 
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/llms_claude.py

@@ -1,13 +1,12 @@
 import json
 from pathlib import Path
 from anthropic import Anthropic, NOT_GIVEN
-import os
 from typing import Optional
 
-from gjdutils.rand import DEFAULT_RANDOM_SEED
 from gjdutils.llm_utils import image_to_base64_basic
+from gjdutils.env import get_env_var
 
-CLAUDE_API_KEY = os.environ.get("CLAUDE_API_KEY")
+CLAUDE_API_KEY = get_env_var("CLAUDE_API_KEY")
 # https://docs.anthropic.com/en/docs/about-claude/models
 MODEL_NAME_CLAUDE_SONNET_GOOD_PINNED = "claude-3-5-sonnet-20240620"
 MODEL_NAME_CLAUDE_SONNET_GOOD_LATEST = "claude-3-5-sonnet-latest"

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/llms_openai.py

@@ -1,17 +1,15 @@
 import json
 from openai import OpenAI, NOT_GIVEN
-import os
-from pprint import pprint
-from typing import Any, Literal, Optional
+from typing import Literal, Optional
 
 from .prompt_templates import summarise_list_of_texts_as_one, summarise_text
 from .rand import DEFAULT_RANDOM_SEED
 from .strings import jinja_render
 from gjdutils.llm_utils import contents_for_images
+from gjdutils.env import get_env_var
 
 
-# openai.api_key = OPENAI_API_KEY
-OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+OPENAI_API_KEY = get_env_var("OPENAI_API_KEY")
 MODEL_NAME_GPT4 = "gpt-4"
 MODEL_NAME_GPT35 = "gpt-3.5-turbo"
 MODEL_NAME_GPT4_TURBO = "gpt-4-turbo"  # -1106-preview"

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/num.py

@@ -20,7 +20,7 @@ def discretise(
     increment: Union[int, float] = 0.1,
     lower: Union[int, float] = 0.0,
     upper: Union[int, float] = 1.0,
-    enforce_range: bool = False,
+    enforce_range: bool = True,
 ):
     """
     You will probably want to cache this.
@@ -40,10 +40,10 @@ def discretise(
 
     if pd.isnull(val):
         return upper
-    if enforce_range:
-        assert (
-            lower <= val <= upper
-        ), f"Required: {lower:.2f} < {val:.2f} <= {upper:.2f}"
+    if enforce_range and (val < lower or val > upper):
+        raise ValueError(
+            f"Value {val:.2f} is outside the valid range [{lower:.2f}, {upper:.2f}]"
+        )
     increments = calc_increments(increment, lower, upper)
     if val < lower:
         return increments[0]

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/outloud_text_to_speech.py

@@ -1,5 +1,6 @@
 import os
 from typing import Optional
+from gjdutils.env import get_env_var
 
 
 def outloud(
@@ -33,11 +34,12 @@ def outloud(
             verbose=verbose,
         )
     elif prog == "elevenlabs":
+        assert speed is None, "speed not supported for elevenlabs"
         response = outloud_elevenlabs(
             text=text,
             mp3_filen=mp3_filen,
             bot_name=bot_name,
-            speed=speed,
+            # speed=speed,
             verbose=verbose,
         )
     else:
@@ -65,8 +67,8 @@ def outloud_azure(
 
     # This example requires environment variables named "SPEECH_KEY" and "SPEECH_REGION"
     speech_config = speechsdk.SpeechConfig(
-        subscription=os.environ.get("SPEECH_KEY"),
-        region=os.environ.get("SPEECH_REGION"),
+        subscription=get_env_var("SPEECH_KEY"),
+        region=get_env_var("SPEECH_REGION"),
     )
     # https://learn.microsoft.com/en-us/answers/questions/693848/can-azure-text-to-speech-support-more-audio-files.html
     speech_config.set_speech_synthesis_output_format(
@@ -78,7 +80,7 @@ def outloud_azure(
 
     # The language of the voice that speaks.
     # speech_config.speech_synthesis_voice_name = "en-US-JennyNeural"
-    speech_config.speech_synthesis_voice_name = bot_name
+    speech_config.speech_synthesis_voice_name = bot_name  # type: ignore
 
     speech_synthesizer = speechsdk.SpeechSynthesizer(
         speech_config=speech_config, audio_config=audio_config
@@ -100,13 +102,13 @@ def outloud_azure(
     speech_synthesis_result = speech_synthesizer.speak_ssml_async(ssml).get()
 
     if (
-        speech_synthesis_result.reason
+        speech_synthesis_result.reason  # type: ignore
         == speechsdk.ResultReason.SynthesizingAudioCompleted
     ):
         if verbose > 0:
             print("Speech synthesized for text [{}]".format(text))
-    elif speech_synthesis_result.reason == speechsdk.ResultReason.Canceled:
-        cancellation_details = speech_synthesis_result.cancellation_details
+    elif speech_synthesis_result.reason == speechsdk.ResultReason.Canceled:  # type: ignore
+        cancellation_details = speech_synthesis_result.cancellation_details  # type: ignore
         print("Speech synthesis canceled: {}".format(cancellation_details.reason))
         if cancellation_details.reason == speechsdk.CancellationReason.Error:
             if cancellation_details.error_details:
@@ -193,7 +195,7 @@ def outloud_elevenlabs(
     from elevenlabs.client import ElevenLabs
 
     if api_key is None:
-        api_key = os.environ.get("ELEVENLABS_API_KEY")
+        api_key = get_env_var("ELEVENLABS_API_KEY")
     # i should figure out a better way to handle playing an mp3 file
     # assert (
     #     not mp3_filen and should_play

gjdutils-0.3.0/src/gjdutils/vprint.py → gjdutils-0.3.4/src/gjdutils/print_utils.py

@@ -2,9 +2,12 @@ from gjdutils.functions import variable_from_caller
 
 
 def vprint(min_verbosity: int, msg: str, /, **kwargs):
-    """Print if the caller's verbosity level is >= min_verbosity.
+    """Print if the caller's verbosity level is >= min_verbosity, e.g.
 
-    This function looks for a 'verbose' variable in the caller's scope.
+        verbose = 2
+        vprint(1, "Hello, world!")  # prints because verbose >= 1
+
+    This function looks for a `verbose` variable in the caller's scope.
 
     Args:
         min_verbosity: Minimum verbosity level required to print (positional-only)

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/pypi_build.py

@@ -4,16 +4,30 @@ from pathlib import Path
 import urllib.request
 import urllib.error
 import shutil
+import tomllib
 from typing import Literal
 from rich.console import Console
 from rich.progress import track
 from packaging.version import Version
+from importlib.metadata import metadata
 
 from gjdutils.cmd import run_cmd
-from gjdutils import __version__
 
 console = Console()
 
+
+def get_version() -> str:
+    """Get package version from pyproject.toml.
+
+    We read directly from pyproject.toml rather than using Python's packaging machinery
+    because we've had issues with the correct version being accessed in deployment/checking
+    machinery that creates virtualenvs. This provides a more reliable way to access the
+    true version from the source."""
+    with open("pyproject.toml", "rb") as f:
+        data = tomllib.load(f)
+    return data["project"]["version"]
+
+
 # Type for PyPI environment
 PyPIEnv = Literal["test", "prod"]
 
@@ -25,26 +39,27 @@ def verify_installation(python_path: Path):
         before_msg="Verify package installation by importing and checking version...",
         fatal_msg="Failed to import gjdutils",
     )
+    expected_version = get_version()
     assert (
-        installed_version == __version__
-    ), f"Installed version {installed_version} does not match expected version {__version__}"
+        installed_version == expected_version
+    ), f"Installed version {installed_version} does not match expected version {expected_version}"
     console.print(f"gjdutils version: {installed_version}")
     return installed_version
 
 
 def check_install_optional_features(python_path: Path, *, from_test_pypi: bool = False):
     """Test installation of optional feature sets."""
-    features = [
-        "audio_lang",
-        "dt",
-        "llm",
-        "html_web",
-    ]
+    # Get optional dependency groups from package metadata
+    pkg_metadata = metadata("gjdutils")
+    # Parse the provides-extra field to get optional dependency groups
+    # get_all() returns None if the field doesn't exist
+    extra_features = pkg_metadata.get_all("Provides-Extra") or []
+    features = [group for group in extra_features if group not in ["dev", "all_no_dev"]]
 
     for feature in track(features, description="Installing features"):
         console.print(f"\nTesting feature set: {feature}", style="yellow")
         if from_test_pypi:
-            cmd = f"{python_path} -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ gjdutils"
+            cmd = f"{python_path} -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ gjdutils[{feature}]"
         else:
             cmd = f"{python_path} -m pip install '.[{feature}]'"
         run_cmd(

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/runtime.py

@@ -1,6 +1,6 @@
 import sys
 
-from gjdutils.env import get_env_var
+# keep this, because it makes sense for the user to be able to import this from here
 from gjdutils.pytest_utils import in_pytest
 
 

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/shell.py

@@ -2,7 +2,6 @@
 
 from pathlib import Path
 import shutil
-import subprocess
 import sys
 import venv
 from contextlib import contextmanager

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/strings.py

@@ -121,7 +121,7 @@ def jinja_render(
 
     Will raise an error if CONTEXT is missing any variables.
     """
-    from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
+    from jinja2 import Environment, FileSystemLoader, StrictUndefined
 
     loader = None if filesystem_loader is None else FileSystemLoader(filesystem_loader)
     env = Environment(loader=loader, undefined=StrictUndefined)
@@ -200,11 +200,18 @@ def append_fullstop(s: str):
 
 
 def str_from_num(x):
+    try:
+        import numpy as np
+
+        bools = (bool, np.bool_)
+    except ImportError:
+        bools = (bool,)
+
     # todo, find a better way of doing this, e.g.
     # if isinstance(Iterable), recur. otherwise try str(x)
     if x is None:
         return str(x)
-    elif isinstance(x, (bool, np.bool_)):
+    elif isinstance(x, bools):
         return str(x)
     elif isinstance(x, str):
         return x

{gjdutils-0.3.0 → gjdutils-0.3.4}/src/gjdutils/typ.py

@@ -1,3 +1,4 @@
+# keep this, because it makes sense for the user to be able to import from here
 from inspect import isfunction
 
 # these are occasionally useful for if statements,
@@ -32,6 +33,6 @@ def isiterable(x):
     """
     from http://stackoverflow.com/questions/1952464/in-python-how-do-i-determine-if-a-variable-is-iterable
     """
-    import collections
+    import collections.abc
 
-    return isinstance(x, collections.Iterable)
+    return isinstance(x, collections.abc.Iterable)

gjdutils-0.3.4/tests/test_num.py

@@ -0,0 +1,18 @@
+from pytest import approx, raises
+
+from gjdutils.num import discretise
+
+
+def test_discretise():
+    assert discretise(0.00, increment=0.05) == approx(0.00)
+    assert discretise(0.01, increment=0.05) == approx(0.00)
+    assert discretise(0.06, increment=0.05) == approx(0.05)
+    assert discretise(0.99, increment=0.05) == approx(0.95)
+    assert discretise(1.00, increment=0.05) == approx(1.00)
+    # check values outside the range
+    with raises(Exception):
+        discretise(5.00, increment=0.05)
+    with raises(Exception):
+        discretise(-1.00, increment=0.05)
+    assert discretise(-1.00, increment=0.05, enforce_range=False) == approx(0.00)
+    assert discretise(5.00, increment=0.05, enforce_range=False) == approx(1.00)

gjdutils-0.3.0/tests/test_vprint.py → gjdutils-0.3.4/tests/test_print_utils.py

@@ -1,5 +1,5 @@
 import pytest
-from gjdutils.vprint import vprint
+from gjdutils.print_utils import vprint
 
 
 def test_vprint_basic():

gjdutils-0.3.0/__VERSION__.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"

gjdutils-0.3.0/src/gjdutils/__init__.py

@@ -1,12 +0,0 @@
-import os
-import sys
-
-# Add the root directory to Python path to import __VERSION__
-root_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-if root_dir not in sys.path:
-    sys.path.append(root_dir)
-
-from __VERSION__ import __version__
-
-# Export version at package level
-__version__ = __version__

gjdutils-0.3.0/src/gjdutils/sorteddict.py

@@ -1,34 +0,0 @@
-# This dict is always sorted by `sortfunc` when you iterate over it.
-#
-# For example, if you have a dict of str -> Nodes, and you want to iterate over them in
-# order of their `idx` attribute, you can do:
-#
-#   d = SortedDict[str,Node](lambda node: node.idx)
-#   [insert, remove, Nodes]
-#   d.values()  # this will always be sorted by idx
-#
-# based on code from ChatGPT
-
-from typing import Any, Callable, Dict, Generic, TypeVar
-
-
-# Define type variables for keys and values
-K = TypeVar("K")
-V = TypeVar("V")
-
-
-class SortedDict(Dict[K, V], Generic[K, V]):
-    def __init__(self, sortfunc: Callable[[V], Any], *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.sortfunc = sortfunc
-        self._reorder()
-
-    def __setitem__(self, key, value):
-        super().__setitem__(key, value)
-        self._reorder()
-
-    def _reorder(self):
-        sorted_items = sorted(self.items(), key=lambda item: self.sortfunc(item[1]))
-        super().clear()
-        for key, value in sorted_items:
-            super().__setitem__(key, value)