zhmiscellany 6.2.4__py3-none-any.whl → 6.2.5__py3-none-any.whl

zhmiscellany/_processing_supportfuncs.py

@@ -1,11 +1,5 @@
 # these lines are purposefully the first thing to run when zhmiscellany is imported
-import threading, logging, os, inspect, tempfile, shutil
-from itertools import chain
-import zhmiscellany.fileio
-from io import StringIO
-import sys
-import io
-from unittest.mock import patch
+import sys  # cannot be moved
 
 # Ray availability check
 if sys.platform == "win32" or True:
@@ -13,8 +7,13 @@ if sys.platform == "win32" or True:
 else:
     RAY_AVAILABLE = False
 
+import os  # needed for module-level log clearing and cause detection
 
 def clear_logs():
+    import tempfile
+    import os
+    import shutil
+    import zhmiscellany.fileio
     ray_dir = tempfile.gettempdir()
     ray_dir = os.path.join(ray_dir, 'ray')
     
@@ -46,6 +45,7 @@ if 'ray_logs_cleared' not in os.environ:
 
 
 def safe_open_log(path, unbuffered=False, **kwargs):
+    import os
     try:
         kwargs.setdefault("buffering", 1)
         kwargs.setdefault("mode", "a")
@@ -65,6 +65,8 @@ def safe_open_log(path, unbuffered=False, **kwargs):
 
 
 def ray_init(auto=False):
+    import threading
+    import os
     if not RAY_AVAILABLE:
         print("ray_init() only supports Windows! Functionality disabled")
         return
@@ -96,6 +98,10 @@ def _ray_init():
 
     try:
         def safe_ray_init():
+            import sys
+            import io
+            import ray
+
             def ensure_valid_handles():
                 """Ensure stdout and stderr are valid file-like objects"""
                 if not hasattr(sys.stdout, 'write') or sys.stdout.closed:
@@ -136,6 +142,7 @@ def _ray_init():
 
 
 def get_import_chain():
+    import inspect
     frame = inspect.currentframe()
     chain = []
     while frame:
@@ -149,6 +156,8 @@ def get_import_chain():
         frame = frame.f_back
     return chain[::-1]
 
+
+# Cause detection for auto-initializing ray
 cause_strings = [
     'processing.multiprocess(',
     'processing.batch_multiprocess(',
@@ -171,6 +180,7 @@ for file in cause_files:
             cause = True
             break
 
+import threading
 _ray_init_thread = threading.Thread()  # initialize variable to completed thread
 _ray_init_thread.start()
 
@@ -194,6 +204,8 @@ class ThreadWithResult(threading.Thread):
 
 
 def batch_multiprocess(targets_and_args, max_retries=0, expect_crashes=False, disable_warning=False, flatten=False):
+    import logging
+    from itertools import chain
     if not RAY_AVAILABLE:
         print("batch_multiprocess() only supports Windows! Returning empty list")
         return []
@@ -213,6 +225,7 @@ from zhmiscellany._processing_supportfuncs import _ray_init_thread; _ray_init_th
     _ray_init_thread.join()
     
     if not expect_crashes:
+        import ray
         @ray.remote(max_retries=max_retries, num_cpus=0)
         def worker(func, *args):
             return func(*args)
@@ -223,12 +236,14 @@ from zhmiscellany._processing_supportfuncs import _ray_init_thread; _ray_init_th
             results = list(chain.from_iterable(results))
         return results
     else:
+        import ray
         def wrap_exception(task, disable_warning, max_retries):
             try:
                 result = multiprocess(*task, disable_warning=disable_warning, max_retries=max_retries)
                 return result
             except ray.exceptions.WorkerCrashedError:
                 return None
+        import threading  # this import is explicitly in the original code
         threads = []
         for task in targets_and_args:
             t = ThreadWithResult(
@@ -244,6 +259,7 @@ from zhmiscellany._processing_supportfuncs import _ray_init_thread; _ray_init_th
             results = list(chain.from_iterable(results))
         return results
 
+
 def multiprocess(target, args=(), max_retries=0, disable_warning=False):
     if not RAY_AVAILABLE:
         print("multiprocess() only supports Windows! Returning None")
@@ -255,10 +271,12 @@ class RayActorWrapper:
     def __init__(self, actor_instance):
         self._actor = actor_instance
         
+        import ray
         ray.get(self._actor._ready.remote())
     
     def __getattr__(self, name):
         # When you access an attribute, assume it's a remote method.
+        import ray
         remote_method = getattr(self._actor, name)
         if not callable(remote_method):
             # If it's not callable, try to get its value.
@@ -274,6 +292,8 @@ class RayActorWrapper:
 
 
 def synchronous_class_multiprocess(cls, *args, disable_warning=False, **kwargs):
+    import logging
+    import ray
     if not RAY_AVAILABLE:
         print("synchronous_class_multiprocess() only supports Windows! Returning None")
         return None
@@ -299,4 +319,4 @@ from zhmiscellany._processing_supportfuncs import _ray_init_thread; _ray_init_th
 
     remote_cls = ray.remote(num_cpus=0)(cls)
     actor_instance = remote_cls.remote(*args, **kwargs)
-    return RayActorWrapper(actor_instance)
+    return RayActorWrapper(actor_instance)

zhmiscellany/_py_resources.py

@@ -1,5 +1,5 @@
-import os, zlib
 def gen():
+    import os, zlib
     os.makedirs('resources', exist_ok=True)
     os.makedirs('resources\\random_header_generator', exist_ok=True)
     os.makedirs('resources\\random_header_generator\\data', exist_ok=True)

zhmiscellany/dict.py

@@ -1,5 +1,3 @@
-import json
-
-
 def print_dict(ldict):
+    import json
     print(json.dumps(ldict, indent=4))

zhmiscellany/discord.py

@@ -1,32 +1,16 @@
-import time
-import sys
-import requests
-import copy
-import zhmiscellany.fileio
-import zhmiscellany.netio
-import zhmiscellany.processing
+import sys  # cannot be touched because it's needed
 from ._discord_supportfuncs import scrape_guild
 
-import base64
-import os
-import json
-import re
-
 # Windows-specific imports
 if sys.platform == "win32":
-    try:
-        import win32crypt
-        from Crypto.Cipher import AES
-        WIN32_AVAILABLE = True
-    except ImportError:
-        WIN32_AVAILABLE = False
-        print("Warning: Windows modules not available - local Discord user detection disabled")
+    WIN32_AVAILABLE = True
 else:
     WIN32_AVAILABLE = False
 
 
 def add_reactions_to_message(user_token, emojis, channel_id, message_id):
-
+    import time
+    import requests
     for emoji in emojis:
         url = f'https://discord.com/api/v9/channels/{channel_id}/messages/{message_id}/reactions/{emoji}/@me'
         #headers = {**zhmiscellany.netio.generate_headers(url), 'Authorization': user_token}
@@ -54,7 +38,9 @@ def get_channel_messages(user_token, channel_id, limit=0, use_cache=True, show_p
     '''
     Function to get all client messages in a specific channel. Script by @z_h_ on discord.
     '''
-
+    import requests
+    import zhmiscellany.fileio
+    import os
     if use_cache:
         cache_folder = 'zhmiscellany_cache'
         zhmiscellany.fileio.create_folder(cache_folder)
@@ -118,10 +104,18 @@ def get_channel_messages(user_token, channel_id, limit=0, use_cache=True, show_p
 
 
 def get_local_discord_user(show_output=False):
+    import requests
+    import os
+    import json
+    import re
+    import base64
+    import win32crypt
+    from Crypto.Cipher import AES
+    import zhmiscellany.netio
     if not WIN32_AVAILABLE:
         print("get_local_discord_user() only supports Windows! Returning None")
         return None
-        
+
     global _cached_user_info
     try:
         a = _cached_user_info
@@ -241,6 +235,9 @@ def get_local_discord_user(show_output=False):
 
 
 def get_guild_channels(user_token, guild_id, use_cache=True):
+    import requests
+    import os
+    import zhmiscellany.netio
     if use_cache:
         potential_path = os.path.join('zhmiscellany_cache', f'{guild_id}_channels.json')
         if os.path.exists(potential_path):
@@ -260,12 +257,17 @@ def get_guild_channels(user_token, guild_id, use_cache=True):
 
 
 def send_type(user_token, channel_id):  # after sending the typing post request, the account will be shown as "typing" in the given channel for 10 seconds, or until a message is sent.
+    import requests
+    import zhmiscellany.netio
     url = f'https://discord.com/api/v9/channels/{channel_id}/typing'
     headers = {**zhmiscellany.netio.generate_headers(url), 'Authorization': user_token}
     return requests.post(url, headers=headers)
 
 
 def send_message(user_token, text, channel_id, attachments=None, typing_time=0):
+    import time
+    import requests
+    import zhmiscellany.processing
     typing_time_increments = 9.5  # not set to 10 because then every 10 seconds the typing would stop very briefly
     while typing_time > 0:
         zhmiscellany.processing.start_daemon(target=send_type, args=(user_token, channel_id))
@@ -294,6 +296,8 @@ def send_message(user_token, text, channel_id, attachments=None, typing_time=0):
 
 
 def get_message(user_token, channel_id, message_id):
+    import requests
+    import zhmiscellany.netio
     message_url = f'https://discord.com/api/v9/channels/{channel_id}/messages?limit=1&around={message_id}'
     message = requests.get(message_url, headers={**zhmiscellany.netio.generate_headers(message_url), 'Authorization': user_token})
     message = message.json()
@@ -308,6 +312,7 @@ def ids_to_message_url(channel_id, message_id, guild_id=None):
 
 
 def message_url_to_ids(message_url):
+    import re
     # Regular expressions to extract IDs
     guild_channel_message_regex = r'https:\/\/discord\.com\/channels\/(\d+)\/(\d+)\/(\d+)'
     channel_message_regex = r'https:\/\/discord\.com\/channels\/(\d+)\/(\d+)'
@@ -336,6 +341,8 @@ def decode_user_id(user_token):
     padded_base64 = payload_base64 + '=' * (4 - len(payload_base64) % 4)
 
     # Decoding the base64 and converting to a JSON object
+    import base64
+    import json
     payload_json = base64.b64decode(padded_base64).decode('utf-8')
     user_id = json.loads(payload_json)
 
@@ -343,6 +350,9 @@ def decode_user_id(user_token):
 
 
 def get_guilds(user_token, use_cache=True):
+    import requests
+    import os
+    import zhmiscellany.netio
     if use_cache:
         potential_path = os.path.join('zhmiscellany_cache', f'{decode_user_id(user_token)}_guilds.json')
         if os.path.exists(potential_path):
@@ -361,6 +371,9 @@ def get_guilds(user_token, use_cache=True):
 
 
 def get_dm_channels(user_token, use_cache=True):
+    import requests
+    import os
+    import zhmiscellany.netio
     if use_cache:
         potential_path = os.path.join('zhmiscellany_cache', f'{decode_user_id(user_token)}_dm_channels.json')
         if os.path.exists(potential_path):
@@ -379,6 +392,9 @@ def get_dm_channels(user_token, use_cache=True):
 
 
 def get_invite_info(user_token, invite_code, use_cache=True):
+    import requests
+    import os
+    import zhmiscellany.netio
     if use_cache:
         potential_path = os.path.join('zhmiscellany_cache', f'{invite_code}_invite.json')
         if os.path.exists(potential_path):
@@ -397,6 +413,8 @@ def get_invite_info(user_token, invite_code, use_cache=True):
 
 
 def generate_server_invite(user_token, channel_id):
+    import zhmiscellany.netio
+    import requests
     url = f"https://discord.com/api/v9/channels/{channel_id}/invites"
     response = requests.get(url, headers={**zhmiscellany.netio.generate_headers(url), 'Authorization': user_token})
 
@@ -408,6 +426,8 @@ def generate_server_invite(user_token, channel_id):
 
 
 def get_approximate_member_count(user_token, channel_id, use_cache=True):
+    import os
+    import zhmiscellany.netio
     if use_cache:
         potential_path = os.path.join('zhmiscellany_cache', f'{channel_id}_member_count.json')
         if os.path.exists(potential_path):
@@ -434,6 +454,9 @@ def timestamp_to_id(timestamp):
 
 
 def get_user_avatar_url(user_token, user_id, use_cache=True):
+    import requests
+    import os
+    import zhmiscellany.netio
     url = f"https://discord.com/api/v10/users/{user_id}"
 
     if use_cache:

zhmiscellany/fileio.py

@@ -1,22 +1,9 @@
-from ._fileio_supportfuncs import is_junction
-import json, os, shutil, dill, sys, pickle, base64, zlib
-import zhmiscellany.string
-import zhmiscellany.misc
-import hashlib
-from collections import defaultdict
-from itertools import chain
-import tempfile
-import random
-import string
-import orjson
-from datetime import datetime
-import inspect
-
-
 def read_json_file(file_path):
     """
     Reads JSON data from a file and returns it as a dictionary.
     """
+    import json
+    import os
     if os.path.exists(file_path):
         with open(file_path, 'r') as file:
             data = json.load(file)
@@ -31,61 +18,65 @@ def write_json_file(file_path, data):
     """
     Writes a dictionary to a JSON file.
     """
+    import json
     with open(file_path, 'w') as file:
         json.dump(data, file, indent=4)
 
 
 def create_folder(folder_name):
+    import os
     if not os.path.exists(folder_name):
         os.makedirs(folder_name)
 
 
 def remove_folder(folder_name):
+    import os
+    import shutil
     if os.path.exists(folder_name):
         shutil.rmtree(folder_name)
 
 
 def base_name_no_ext(file_path):
+    import os
     base_name = os.path.basename(file_path)
     base_name_without_extension, _ = os.path.splitext(base_name)
     return base_name_without_extension
 
 
 def convert_name_to_filename(name):
+    import zhmiscellany.string
     return zhmiscellany.string.multi_replace(name, [("/","["), (":","]"), (".","+")])
 
 
 def convert_filename_to_name(filename):
+    import zhmiscellany.string
     return zhmiscellany.string.multi_replace(filename, [("[","/"), ("]",":"), ("+",".")])
 
 
 def recursive_copy_files(source_dir, destination_dir, prints=False):
+    import os
+    import shutil
     if prints:
         print('Validating matching directory structure')
     for root, dirs, files in os.walk(source_dir):
         for dir in dirs:
             dir_path = os.path.join(root, dir)
             dest_dir_path = os.path.join(destination_dir, os.path.relpath(dir_path, source_dir))
-
             if not os.path.exists(dest_dir_path):
                 print(f'Creating missing directory {dest_dir_path}')
                 os.makedirs(dest_dir_path)
-
     if prints:
         print('Getting a list of files in the source directory')
     source_files = []
     for root, _, files in os.walk(source_dir):
         for file in files:
             source_files.append(os.path.join(root, file))
-
     if prints:
         print('Getting a list of files in the destination directory')
     dest_files = []
     for root, _, files in os.walk(destination_dir):
         for file in files:
             dest_files.append(os.path.join(root, file))
-
-
     if prints:
         print('Copying files from source to destination, skipping duplicates')
     for root, dirs, files in os.walk(source_dir):
@@ -93,7 +84,6 @@ def recursive_copy_files(source_dir, destination_dir, prints=False):
             source_file = os.path.join(root, file)
             rel_path = os.path.relpath(source_file, source_dir)
             dest_file = os.path.join(destination_dir, rel_path)
-
             if not os.path.exists(dest_file):
                 if prints:
                     print(f'Copying {source_file}')
@@ -105,10 +95,11 @@ def recursive_copy_files(source_dir, destination_dir, prints=False):
 
 
 def empty_directory(directory_path):
+    import os
+    import shutil
     # Iterate over all items in the directory
     for item in os.listdir(directory_path):
         item_path = os.path.join(directory_path, item)
-
         if os.path.isfile(item_path):
             # If it's a file, delete it
             os.unlink(item_path)
@@ -118,10 +109,12 @@ def empty_directory(directory_path):
 
 
 def abs_listdir(path):
+    import os
     return [os.path.join(path, file) for file in os.listdir(path)]
 
 
 def delete_ends_with(directory, string_endswith, avoid=[]):
+    import os
     files = abs_listdir(directory)
     for file in files:
         if file.endswith(string_endswith):
@@ -134,17 +127,20 @@ def read_bytes_section(file_path, section_start, section_end):
         file.seek(section_start)  # Move the file pointer to the 'start' position
         bytes_to_read = section_end - section_start
         data = file.read(bytes_to_read)  # Read 'bytes_to_read' number of bytes
-
     return data
 
 
 def copy_file_with_overwrite(src, dst):
+    import os
+    import shutil
     if os.path.exists(dst):
         os.remove(dst)
     shutil.copy2(src, dst)
 
 
 def fast_dill_dumps(object):
+    import pickle
+    import dill
     try:
         data = pickle.dumps(object, protocol=5)  # pickle is much faster so at least attempt to use it at first
     except:
@@ -153,6 +149,8 @@ def fast_dill_dumps(object):
 
 
 def fast_dill_loads(data):
+    import pickle
+    import dill
     try:
         object = pickle.loads(data)  # pickle is much faster so at least attempt to use it at first
     except:
@@ -161,6 +159,7 @@ def fast_dill_loads(data):
 
 
 def save_object_to_file(object, file_name, compressed=False):
+    import zlib
     with open(file_name, 'wb') as f:
         if compressed:
             f.write(zlib.compress(fast_dill_dumps(object)))
@@ -169,6 +168,7 @@ def save_object_to_file(object, file_name, compressed=False):
 
 
 def load_object_from_file(file_name, compressed=False):
+    import zlib
     with open(file_name, 'rb') as f:
         if compressed:
             return fast_dill_loads(zlib.decompress(f.read()))
@@ -178,26 +178,33 @@ def load_object_from_file(file_name, compressed=False):
 
 def pickle_and_encode(obj):
     """Pickles an object and URL-safe encodes it."""
+    import base64
+    import zlib
     pickled_data = zlib.compress(fast_dill_dumps(obj), 9)  # Serialize the object
     encoded_data = base64.urlsafe_b64encode(pickled_data).decode()  # Base64 encode
     return encoded_data
 
+
 def decode_and_unpickle(encoded_str):
     """Decodes a URL-safe encoded string and unpickles the object."""
+    import base64
+    import zlib
     pickled_data = base64.urlsafe_b64decode(encoded_str)  # Decode from Base64
     obj = fast_dill_loads(zlib.decompress(pickled_data))  # Deserialize
     return obj
 
 
 def list_files_by_modified_time(directory):
+    import os
     files_with_times = [(file, os.path.getmtime(os.path.join(directory, file))) for file in os.listdir(directory) if os.path.isfile(os.path.join(directory, file))]
     sorted_files = sorted(files_with_times, key=lambda x: x[1], reverse=True)
     sorted_file_names = [file for file, _ in sorted_files]
-
     return sorted_file_names
 
 
 def get_script_path():
+    """Returns the path to the current script or executable."""
+    import sys
     if getattr(sys, 'frozen', False):
         # Running as a standalone executable
         return sys.executable
@@ -207,10 +214,21 @@ def get_script_path():
 
 
 def chdir_to_script_dir():
+    import os
     os.chdir(os.path.dirname(get_script_path()))
 
 
 def cache(function, *args, **kwargs):
+    """
+    Caches the result of a function call to disk.
+    """
+    import os
+    import inspect
+    import orjson
+    import hashlib
+    from datetime import datetime
+    import zhmiscellany.fileio
+
     cache_folder = 'zhmiscellany_cache'
 
     def get_hash_orjson(data):
@@ -267,6 +285,10 @@ def cache(function, *args, **kwargs):
 
 
 def load_all_cached():
+    """
+    Loads all cached objects from the cache folder.
+    """
+    import os
     cache_folder = 'zhmiscellany_cache'
     if os.path.exists(cache_folder):
         files = abs_listdir(cache_folder)
@@ -280,6 +302,11 @@ def load_all_cached():
 
 
 def list_files_recursive(folder):
+    """
+    Recursively lists all files in a directory, excluding symlinks and junctions.
+    """
+    import os
+    from ._fileio_supportfuncs import is_junction
     files = []
     try:
         for entry in os.scandir(folder):
@@ -295,6 +322,9 @@ def list_files_recursive(folder):
 
 
 def list_files_recursive_multiprocessed(dir_path, return_folders=False):
+    import os
+    import zhmiscellany.processing
+
     def is_junction(entry):
         try:
             st = entry.stat(follow_symlinks=False)
@@ -342,6 +372,8 @@ def list_files_recursive_multiprocessed(dir_path, return_folders=False):
 
 def encode_safe_filename(s, max_length=16):
     """Encodes a string into a short, URL-safe, and file name-safe string."""
+    import base64
+    import hashlib
     encoded = base64.urlsafe_b64encode(s.encode()).decode().rstrip("=")  # URL-safe encoding
     if len(encoded) > max_length:  # Truncate if too long
         encoded = hashlib.md5(s.encode()).hexdigest()[:max_length]  # Use a hash
@@ -349,6 +381,15 @@ def encode_safe_filename(s, max_length=16):
 
 
 def list_files_recursive_cache_optimised_multiprocessed(dir_path, show_timings=False, cache_in_temp=True):
+    import os
+    import zhmiscellany.processing
+    import zhmiscellany.fileio
+    import tempfile
+    from collections import defaultdict
+    import random
+    from itertools import chain
+    import zhmiscellany.misc
+
     def is_junction(entry):
         try:
             st = entry.stat(follow_symlinks=False)
@@ -501,7 +542,6 @@ def list_files_recursive_cache_optimised_multiprocessed(dir_path, show_timings=F
     
     groups = split_into_n_groups(changed_folders, scan_changed_folders_thread_group_count)
     tasks = [(atom, (group,)) for group in groups]
-
     if not tasks:
         results = []
     else:
@@ -516,19 +556,16 @@ def list_files_recursive_cache_optimised_multiprocessed(dir_path, show_timings=F
     if len(changed_folders) > fully_update_cache_threshold:
         new_folders.update(get_m_times(new_new_folders))
         if show_timings: zhmiscellany.misc.time_it(f'get m times of {len(new_new_folders)} new folders')
-        
         zhmiscellany.fileio.save_object_to_file((files, new_folders), cache_file)
-        
         if show_timings: zhmiscellany.misc.time_it(f'writing to cache')
     
     ret = list(chain.from_iterable(files.values()))
-    
     if show_timings: zhmiscellany.misc.time_it('Everything together', 'lfrcomt')
-    
     return ret
 
 
 def save_chunk(name, data):
+    import zhmiscellany.string
     create_folder(name)
     chunk_path = f'{name}/chunk_{zhmiscellany.string.get_universally_unique_string()}.pkl'
     save_object_to_file(data, chunk_path)
@@ -544,8 +581,12 @@ def load_chunks(name):
 
 
 def clear_chunks(name):
+    import os
     if os.path.exists(name):
         empty_directory(name)
 
+
 def list_drives():
+    import os
+    import string
     return [f"{d}:\\" for d in string.ascii_uppercase if os.path.exists(f"{d}:\\")]