pynight 0.2.2__tar.gz → 0.2.3__tar.gz

{pynight-0.2.2 → pynight-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pynight
-Version: 0.2.2
+Version: 0.2.3
 Summary: My Python utility library.
 License: MIT
 Author: NightMachinary
@@ -17,7 +17,7 @@ Requires-Dist: brish (>=0.3.3,<0.4.0)
 Requires-Dist: dnspython (>=2.1.0,<3.0.0)
 Requires-Dist: executing (>=1.2.0,<2.0.0)
 Requires-Dist: matplotlib (>=3.7.1,<4.0.0)
-Requires-Dist: openai (>=0.27.2,<0.28.0)
+Requires-Dist: openai (>=1.14.2,<2.0.0)
 Requires-Dist: pyperclip (>=1.8.2,<2.0.0)
 Requires-Dist: spacy (>=3.5.1,<4.0.0)
-Requires-Dist: tiktoken (>=0.3.3,<0.4.0)
+Requires-Dist: tiktoken (>=0.6,<0.7)

pynight-0.2.3/pynight/batched_iterable.py

@@ -0,0 +1,101 @@
+from collections.abc import Mapping
+from pynight.common_icecream import ic
+from pynight.common_dict import (
+    SimpleObject,
+    simple_obj,
+    BatchedDict,
+)
+
+
+class BatchedIterable:
+    def __init__(
+        self,
+        data,
+        batch_size,
+        drop_last_batch=False,
+        skip_none_p=True,
+        autoadjust_batch_size_mode=False,
+        # autoadjust_batch_size_mode=True,
+    ):
+        self.data = data
+        self.batch_size = batch_size
+        self.drop_last_batch = drop_last_batch
+        self.skip_none_p = skip_none_p
+        self.autoadjust_batch_size_mode = autoadjust_batch_size_mode
+        self.extras = []
+        #: We will drop the default empty `extras`, which might not be summable with `res`. Its only purpose is to make using `len(self.extras)` easy without having to check for None.
+
+    def __iter__(self):
+        length = len(self.data)
+        i = 0
+
+        while i < length or len(self.extras) >= self.batch_size:
+            # ic(type(self.extras))
+
+            if len(self.extras) >= self.batch_size:
+                #: Yield a batch from extras
+                res = self.extras[: self.batch_size]
+                self.extras = self.extras[self.batch_size :]
+                yield res
+                continue
+
+            advance_by = self.batch_size - len(self.extras)
+            res = self.data[i : i + advance_by]
+            # ic(type(res))
+            # ic(len(res), advance_by, len(self.extras))
+
+            i += advance_by
+
+            if self.skip_none_p and res is None:
+                continue
+
+            if self.autoadjust_batch_size_mode:
+                res_size = len(res)
+                if res_size < self.batch_size and self.autoadjust_batch_size_mode in [
+                    True,
+                    "grow",
+                ]:
+                    #: Merge with extras
+                    if len(self.extras) >= 1:
+                        res = self.extras + res
+
+                    if len(res) >= self.batch_size:
+                        self.extras = res[self.batch_size :]
+                        res = res[: self.batch_size]
+                    else:
+                        self.extras = res
+                        continue
+                elif res_size > self.batch_size and self.autoadjust_batch_size_mode in [
+                    True,
+                    "shrink",
+                ]:
+                    #: Keep extras for the next batch
+                    if len(self.extras) >= 1:
+                        self.extras += res[self.batch_size :]
+                    else:
+                        self.extras = res[self.batch_size :]
+
+                    res = res[: self.batch_size]
+
+                #: This assertion should be true, you can enable it for debugging:
+                # if self.autoadjust_batch_size_mode is True:
+                #     assert (
+                #         len(res) == self.batch_size
+                #     ), f"autoadjust_batch_size_mode has not adjusted the size: len={len(res)}, should_be={self.batch_size}"
+
+            # ic(len(res), self.batch_size, self.autoadjust_batch_size_mode)
+            yield res
+
+        if not self.drop_last_batch and len(self.extras) > 0:
+            #: Yield the remaining extras as the last batch
+            yield self.extras
+
+    def __len__(self):
+        length = len(self.data)
+        num_batches = length // self.batch_size
+        if not self.drop_last_batch:
+            num_batches += length % self.batch_size != 0
+        return num_batches
+
+
+##

pynight-0.2.3/pynight/common_anthropic.py

@@ -0,0 +1,33 @@
+import anthropic
+from brish import z, zp
+
+
+anthropic_client = None
+##
+anthropic_key = None
+
+
+def setup_anthropic_key():
+    global anthropic_key
+    global anthropic_client
+
+    anthropic_key = z("var-get anthropic_api_key").outrs
+    assert anthropic_key, "setup_anthropic_key: could not get Anthropic API key!"
+
+    anthropic_client = anthropic.Anthropic(
+        # defaults to os.environ.get("ANTHROPIC_API_KEY")
+        api_key=anthropic_key,
+    )
+    return anthropic_client
+
+
+def anthropic_key_get():
+    global anthropic_key
+
+    if anthropic_key is None:
+        setup_anthropic_key()
+
+    return anthropic_key
+
+
+##

pynight-0.2.3/pynight/common_arxiv.py

@@ -0,0 +1,21 @@
+import re
+
+
+def normalize_path_for_arxiv(
+    path,
+    *,
+    disallowed_regex=r"[ :&\\\"]",
+):
+    """
+    Normalize a file path by replacing spaces, slashes, colons, ampersands,
+    and backslashes with underscores.
+
+    Args:
+        path (str): The file path to normalize
+
+    Returns:
+        str: The normalized file path
+    """
+    path = re.sub(disallowed_regex, "_", path)
+
+    return path

{pynight-0.2.2 → pynight-0.2.3}/pynight/common_async.py

@@ -2,6 +2,7 @@ import functools
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
 
+
 ##
 def force_async(f):
     @functools.wraps(f)

pynight-0.2.3/pynight/common_attr.py

@@ -0,0 +1,225 @@
+import torch
+import torch.nn.functional as F
+import re
+
+from pynight.common_icecream import ic
+from pynight.common_iterable import to_iterable
+from pynight.common_dict import simple_obj
+from pynight.common_datasets import (
+    TransformResult,
+)
+from pynight.common_torch import (
+    rank_tensor,
+)
+
+
+##
+def normalize_map(
+    attributions,
+    normalize,
+    # outlier_quantile=0.1,
+    outlier_quantile=0,
+    num_prefix_tokens=0,
+    bias_token_p=False,
+    clone_p=True,
+    pixel_p=False,
+):
+    if pixel_p:
+        attributions_skipped = attributions
+    else:
+        attributions_skipped = attributions[..., num_prefix_tokens:]  #: Skips CLS
+        if bias_token_p:
+            attributions_skipped = attributions_skipped[..., :-1]
+            #: Skips the bias token
+
+        # ic(num_prefix_tokens, attributions.shape, attributions_skipped.shape)
+
+    attributions_normalized = attributions_skipped
+    if clone_p:
+        attributions_normalized = attributions_normalized.clone()
+
+    normalize = to_iterable(normalize)
+
+    if normalize:
+        for normalizer in normalize:
+            #: Computing these quantiles is expensive, so we do it only when needed.
+            if normalizer in [
+                "scale_by_max_signed_attr",
+            ]:
+                #: higher outlier_quantile makes discerning the attributions in the middle easier, but the higher attributions become the same
+                if outlier_quantile == 0:
+                    max_attr = torch.max(
+                        attributions_normalized,
+                        dim=-1,
+                        keepdim=True,
+                    ).values
+                else:
+                    max_attr = torch.quantile(
+                        attributions_normalized,
+                        1 - outlier_quantile,
+                        dim=-1,
+                        keepdim=True,
+                    )
+
+                # max_attr = torch.max(attributions_normalized).item()
+            else:
+                max_attr = None
+
+            if normalizer in [
+                "scale_by_max_signed_attr",
+                "shift_min_to_zero",
+            ]:
+                if outlier_quantile == 0:
+                    min_attr = torch.min(
+                        attributions_normalized,
+                        dim=-1,
+                        keepdim=True,
+                    ).values
+                else:
+                    min_attr = torch.quantile(
+                        attributions_normalized,
+                        outlier_quantile,
+                        dim=-1,
+                        keepdim=True,
+                    )
+
+                # min_attr = torch.min(attributions_normalized).item()
+            else:
+                min_attr = None
+
+            if normalizer in [
+                "scale_by_max_abs_attr",
+            ]:
+                if outlier_quantile == 0:
+                    max_abs_attr = torch.max(
+                        torch.abs(attributions_normalized),
+                        dim=-1,
+                        keepdim=True,
+                    ).values
+                else:
+                    max_abs_attr = torch.quantile(
+                        torch.abs(attributions_normalized),
+                        1 - outlier_quantile,
+                        dim=-1,
+                        keepdim=True,
+                    )
+
+                # max_abs_attr = torch.max(torch.abs(attributions_normalized)).item()
+            else:
+                max_abs_attr = None
+
+            # ic(
+            #     attributions_normalized.shape,
+            #     max_attr.shape,
+            #     min_attr.shape,
+            #     max_abs_attr.shape,
+            # )
+            # attributions_normalized.shape: torch.Size([10, 196])
+            # max_attr.shape: torch.Size([10, 1])
+            # min_attr.shape: torch.Size([10, 1])
+            # max_abs_attr.shape: torch.Size([10, 1])
+
+            if normalizer == "scale_by_max_abs_attr":
+                attributions_normalized /= max_abs_attr
+
+            elif normalizer.lower() == "relu":
+                attributions_normalized = F.relu(attributions_normalized)
+
+            elif normalizer == "scale_by_max_signed_attr":
+                attributions_normalized = torch.where(
+                    attributions_normalized < 0,
+                    attributions_normalized / torch.abs(min_attr),
+                    attributions_normalized / max_attr,
+                )
+
+            elif normalizer == "shift_min_to_zero":
+                attributions_normalized -= min_attr
+                attributions_normalized = torch.relu(attributions_normalized)
+            
+            elif normalizer == "rank_uniform":
+                ranks_top_is_1 = rank_tensor(
+                    attributions_normalized,
+                    dim=-1,
+                    descending=True,
+                    increment=1,
+                )
+                #: The most important patch is turned into =1=, and the least important patch is turned into =len(attr)=.
+
+                attributions_normalized = 1.0 - (
+                    (ranks_top_is_1 - 1.0) / (ranks_top_is_1.shape[-1] - 1.0)
+                )
+                #: =ranks_top_is_1 - 1= makes the ranks_top_is_1 start from zero.
+                #: =ranks_top_is_1.shape[-1] - 1= adjusts the denominator for the above change.
+                #: =1 - ...= reverses the order of the ranking, so that the first rank becomes 1, and the last rank becomes 0.
+
+            else:
+                raise ValueError(f"Unsupported normalizer: {normalizer}")
+
+    return simple_obj(
+        attributions_skipped=attributions_skipped,
+        attributions_normalized=attributions_normalized,
+    )
+
+
+##
+def transform_attr_threshold(
+    batch,
+    *,
+    attr_name,
+    normalize_opts=None,
+    gt_threshold=None,
+    lt_threshold=None,
+    device=None,
+    return_mode="mutate",
+):
+    attr = batch[attr_name]
+    attr_new = attr  # .clone()
+    #: =normalize_map= clones its input, so we don't need to do so.
+
+    if device is not None:
+        attr_new = attr_new.to(device)
+
+    attr_name_new = attr_name
+
+    attr_name_new = re.sub(r"^attributions_s_", "attributions_p_", attr_name_new)
+    #: _s stands for "signed scalar", and _p stands for "per-patch".
+
+    # ic(attr_name, attr_name_new)
+
+    if normalize_opts is not None:
+        normalize = normalize_opts["normalize"]
+        normalize = to_iterable(normalize)
+        attr_new = normalize_map(
+            attr_new,
+            **normalize_opts,
+        ).attributions_normalized
+        attr_name_new += f"""_N:{'->'.join(normalize)}"""
+
+    if gt_threshold is not None:
+        attr_name_new += f"_GT{gt_threshold}"
+
+        mask = attr_new > gt_threshold
+        attr_new = mask * attr_new
+
+    if lt_threshold is not None:
+        attr_name_new += f"_LT{lt_threshold}"
+
+        mask = attr_name < lt_threshold
+        attr_new = mask * attr_new
+
+    if return_mode == "mutate":
+        batch[attr_name_new] = attr_new
+    elif return_mode == "new_only":
+        batch = dict()
+        batch[attr_name_new] = attr_new
+    else:
+        raise ValueError(f"Unsupported return_mode: {return_mode}")
+
+    return TransformResult(
+        result=batch,
+        name=attr_name_new,
+        #: =name= is used in =segment_by_attr=.
+    )
+
+
+##

pynight-0.2.3/pynight/common_base64.py

@@ -0,0 +1,44 @@
+import base64
+import tempfile
+import os
+from PIL import Image
+
+
+##
+def base64_encode_file(file_path):
+    with open(file_path, "rb") as file_file:
+        return base64.b64encode(file_file.read()).decode("utf-8")
+
+
+def convert_to_jpeg_and_base64_encode(file_path, url_p=True):
+    if os.path.exists(file_path):
+        with Image.open(file_path) as img:
+            if img.format.lower() not in ["jpg", "jpeg"]:
+                #: I don't know if =format= is normalized first or not, so I took the safest approach above.
+
+                rgb_im = img.convert(
+                    "RGB"
+                )  # Convert to RGB if necessary (e.g., RGBA, P mode)
+
+                # Save the image to a temporary file
+                with tempfile.NamedTemporaryFile(
+                    suffix=".jpeg", delete=True
+                ) as tmpfile:
+                    temp_path = tmpfile.name
+                    rgb_im.save(temp_path, format="JPEG")
+
+                    # Use the temporary file for base64 encoding
+                    file_base64 = base64_encode_file(temp_path)
+            else:
+                #: If the image is already a JPEG, encode it directly
+                file_base64 = base64_encode_file(file_path)
+
+            if url_p:
+                return f"data:image/jpeg;base64,{file_base64}"
+            else:
+                return file_base64
+    else:
+        raise FileNotFoundError(f"The file {file_path} does not exist.")
+
+
+##

pynight-0.2.3/pynight/common_bells.py

@@ -0,0 +1,21 @@
+from brish import (
+    z,
+    z_background,
+)
+
+
+def bello():
+    z("bello")
+
+
+def bell_gpt():
+    z("bell-gpt")
+
+
+def bell_call_remote(bell_name):
+    try:
+        if bell_name:
+            z_background("bell-call-remote {bell_name}")
+
+    except Exception as e:
+        print(f"Error remote-calling bell:\n  bell_name: {bell_name}\n  {e}")

pynight-0.2.3/pynight/common_clip.py

@@ -0,0 +1,4 @@
+from pynight.common_model_name import (
+    open_clip_sep,
+    #: @backcompat This used to be located here.
+)

pynight-0.2.3/pynight/common_clipboard.py

@@ -0,0 +1,34 @@
+import pyperclip
+import threading
+import time
+
+
+def clipboard_copy(
+    obj,
+    *,
+    ignore_errors=True,
+):
+    try:
+        return pyperclip.copy(obj)
+    except:
+        if ignore_errors:
+            return None
+        else:
+            raise
+
+
+def clipboard_copy_multi_sync(
+    texts,
+    sleep=0.5,
+):
+    for text in texts:
+        if text:  #: Check if the text is not empty or None
+            clipboard_copy(text)
+            time.sleep(sleep)
+            #: to allow polling-based clipboard managers to capture the text
+
+
+def clipboard_copy_multi(*args):
+    copy_thread = threading.Thread(target=clipboard_copy_multi_sync, args=(args,))
+
+    copy_thread.start()

{pynight-0.2.2 → pynight-0.2.3}/pynight/common_combinatorics.py

@@ -3,6 +3,7 @@ import numpy
 
 np = numpy
 
+
 ##
 def partition_int_into_fixed_gen(n, boxes, min=1):
     "Partitions the integer 'n' into 'boxes' boxes, each box having at least 'min' elements. Order matters. Returns a generator."

{pynight-0.2.2 → pynight-0.2.3}/pynight/common_condition.py

@@ -1,24 +1,32 @@
 import os
 from os import getenv
 
+
 ##
 def jupyter_p():
     return jupyter_p_v1() or jupyter_p_v2()
-    
+
+
 def jupyter_p_v1():
-    return getenv('INSIDE_JUPYTER_NOTEBOOK_P', default='') == 'y'
+    return getenv("INSIDE_JUPYTER_NOTEBOOK_P", default="") == "y"
+
 
 def jupyter_p_v2():
     #: [[https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook][python - How can I check if code is executed in the IPython notebook? - Stack Overflow]]
     ##
     try:
         shell = get_ipython().__class__.__name__
-        if shell == 'ZMQInteractiveShell':
-            return True   # Jupyter notebook or qtconsole
-        elif shell == 'TerminalInteractiveShell':
+        if shell == "ZMQInteractiveShell":
+            return True  # Jupyter notebook or qtconsole
+
+        elif shell == "TerminalInteractiveShell":
             return False  # Terminal running IPython
+
         else:
             return False  # Other type (?)
+
     except NameError:
-        return False      # Probably standard Python interpreter
+        return False  # Probably standard Python interpreter
+
+
 ##

{pynight-0.2.2 → pynight-0.2.3}/pynight/common_csv.py

@@ -34,4 +34,5 @@ def dict_to_csv(data, header_p=True):
 
     return csv_string
 
+
 ##