megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

md_utils/process_utils.py

@@ -1,133 +1,157 @@
-########
-#
-# process_utils.py
-#
-# Run something at the command line and capture the output, based on:
-#
-# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
-# 
-# Includes handy example code for doing this on multiple processes/threads.
-#
-########
-
-#%% Constants, imports, and environment
-
-import os
-import subprocess
-
-os.environ["PYTHONUNBUFFERED"] = "1"
-
-def execute(cmd,encoding=None,errors=None,env=None,verbose=False):
-    """
-    Run [cmd] (a single string) in a shell, yielding each line of output to the caller.
-    
-    The "encoding", "errors", and "env" parameters are passed directly to subprocess.Popen().
-    
-    "verbose" only impacts output about process management, it is not related to printing
-    output from the child process.
-    """
-    
-    if verbose: 
-        if encoding is not None:
-            print('Launching child process with non-default encoding {}'.format(encoding))
-        if errors is not None:
-            print('Launching child process with non-default text error handling {}'.format(errors))
-        if env is not None:
-            print('Launching child process with non-default environment {}'.format(str(env)))
-        
-    # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
-    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                             shell=True, universal_newlines=True, encoding=encoding,
-                             errors=errors, env=env)
-    for stdout_line in iter(popen.stdout.readline, ""):
-        yield stdout_line
-    popen.stdout.close()
-    return_code = popen.wait()
-    if return_code:
-        raise subprocess.CalledProcessError(return_code, cmd)
-
-
-def execute_and_print(cmd,print_output=True,encoding=None,errors=None,env=None,verbose=False):
-    """
-    Run [cmd] (a single string) in a shell, capturing and printing output.  Returns
-    a dictionary with fields "status" and "output".
-    
-    The "encoding", "errors", and "env" parameters are passed directly to subprocess.Popen().
-    
-    "verbose" only impacts output about process management, it is not related to printing
-    output from the child process.
-    """
-
-    to_return = {'status':'unknown','output':''}
-    output = []
-    try:
-        for s in execute(cmd,encoding=encoding,errors=errors,env=env,verbose=verbose):
-            output.append(s)
-            if print_output:
-                print(s,end='',flush=True)
-        to_return['status'] = 0
-    except subprocess.CalledProcessError as cpe:
-        print('execute_and_print caught error: {} ({})'.format(cpe.output,str(cpe)))
-        to_return['status'] = cpe.returncode
-    to_return['output'] = output
-   
-    return to_return
-
-
-#%% Single-threaded test driver for execute_and_print
-
-if False:
-    
-    pass
-
-    #%%
-    
-    if os.name == 'nt':
-        execute_and_print('echo hello && ping -n 5 127.0.0.1 && echo goodbye')  
-    else:
-        execute_and_print('echo hello && sleep 1 && echo goodbye')  
- 
-
-#%% Parallel test driver for execute_and_print
-
-if False:
-   
-    pass
-
-    #%%
-   
-    from functools import partial
-    from multiprocessing.pool import ThreadPool as ThreadPool
-    from multiprocessing.pool import Pool as Pool
-   
-    n_workers = 10
-   
-    # Should we use threads (vs. processes) for parallelization?
-    use_threads = True
-   
-    test_data = ['a','b','c','d']
-   
-    def process_sample(s):
-        return execute_and_print('echo ' + s,True)
-       
-    if n_workers == 1:  
-     
-        results = []
-        for i_sample,sample in enumerate(test_data):    
-            results.append(process_sample(sample))
-     
-    else:
-     
-        n_threads = min(n_workers,len(test_data))
-     
-        if use_threads:
-            print('Starting parallel thread pool with {} workers'.format(n_threads))
-            pool = ThreadPool(n_threads)
-        else:
-            print('Starting parallel process pool with {} workers'.format(n_threads))
-            pool = Pool(n_threads)
-   
-        results = list(pool.map(partial(process_sample),test_data))
-      
-        for r in results:
-            print(r)
+"""
+
+process_utils.py
+
+Run something at the command line and capture the output, based on:
+
+https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
+
+Includes handy example code for doing this on multiple processes/threads.
+
+"""
+
+#%% Constants, imports, and environment
+
+import os
+import subprocess
+
+os.environ["PYTHONUNBUFFERED"] = "1"
+
+def execute(cmd,encoding=None,errors=None,env=None,verbose=False):
+    """
+    Run [cmd] (a single string) in a shell, yielding each line of output to the caller.
+    
+    The "encoding", "errors", and "env" parameters are passed directly to subprocess.Popen().
+    
+    "verbose" only impacts output about process management, it is not related to printing
+    output from the child process.
+    
+    Args:
+        cmd (str): command to run
+        encoding (str, optional): stdout encoding, see Popen() documentation
+        errors (str, optional): error handling, see Popen() documentation
+        env (dict, optional): environment variables, see Popen() documentation
+        verbose (bool, optional): enable additional debug console output
+    
+    Returns:
+        int: the command's return code, always zero, otherwise a CalledProcessError is raised    
+    """
+    
+    if verbose: 
+        if encoding is not None:
+            print('Launching child process with non-default encoding {}'.format(encoding))
+        if errors is not None:
+            print('Launching child process with non-default text error handling {}'.format(errors))
+        if env is not None:
+            print('Launching child process with non-default environment {}'.format(str(env)))
+        
+    # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
+    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                             shell=True, universal_newlines=True, encoding=encoding,
+                             errors=errors, env=env)
+    for stdout_line in iter(popen.stdout.readline, ""):
+        yield stdout_line
+    popen.stdout.close()
+    return_code = popen.wait()
+    if return_code:
+        raise subprocess.CalledProcessError(return_code, cmd)
+        
+    return return_code
+
+
+def execute_and_print(cmd,print_output=True,encoding=None,errors=None,env=None,verbose=False):
+    """
+    Run [cmd] (a single string) in a shell, capturing and printing output.  Returns
+    a dictionary with fields "status" and "output".
+    
+    The "encoding", "errors", and "env" parameters are passed directly to subprocess.Popen().
+    
+    "verbose" only impacts output about process management, it is not related to printing
+    output from the child process.
+    
+    Args:
+        cmd (str): command to run
+        print_output (bool, optional): whether to print output from [cmd]
+        encoding (str, optional): stdout encoding, see Popen() documentation
+        errors (str, optional): error handling, see Popen() documentation
+        env (dict, optional): environment variables, see Popen() documentation
+        verbose (bool, optional): enable additional debug console output
+    
+    Returns:
+        dict: a dictionary with fields "status" (the process return code) and "output"
+        (the content of stdout)    
+    """
+
+    to_return = {'status':'unknown','output':''}
+    output = []
+    try:
+        for s in execute(cmd,encoding=encoding,errors=errors,env=env,verbose=verbose):
+            output.append(s)
+            if print_output:
+                print(s,end='',flush=True)
+        to_return['status'] = 0
+    except subprocess.CalledProcessError as cpe:
+        print('execute_and_print caught error: {} ({})'.format(cpe.output,str(cpe)))
+        to_return['status'] = cpe.returncode
+    to_return['output'] = output
+   
+    return to_return
+
+
+#%% Single-threaded test driver for execute_and_print
+
+if False:
+    
+    pass
+
+    #%%
+    
+    if os.name == 'nt':
+        execute_and_print('echo hello && ping -n 5 127.0.0.1 && echo goodbye')  
+    else:
+        execute_and_print('echo hello && sleep 1 && echo goodbye')  
+ 
+
+#%% Parallel test driver for execute_and_print
+
+if False:
+   
+    pass
+
+    #%%
+   
+    from functools import partial
+    from multiprocessing.pool import ThreadPool as ThreadPool
+    from multiprocessing.pool import Pool as Pool
+   
+    n_workers = 10
+   
+    # Should we use threads (vs. processes) for parallelization?
+    use_threads = True
+   
+    test_data = ['a','b','c','d']
+   
+    def process_sample(s):
+        return execute_and_print('echo ' + s,True)
+       
+    if n_workers == 1:  
+     
+        results = []
+        for i_sample,sample in enumerate(test_data):    
+            results.append(process_sample(sample))
+     
+    else:
+     
+        n_threads = min(n_workers,len(test_data))
+     
+        if use_threads:
+            print('Starting parallel thread pool with {} workers'.format(n_threads))
+            pool = ThreadPool(n_threads)
+        else:
+            print('Starting parallel process pool with {} workers'.format(n_threads))
+            pool = Pool(n_threads)
+   
+        results = list(pool.map(partial(process_sample),test_data))
+      
+        for r in results:
+            print(r)

md_utils/sas_blob_utils.py

@@ -1,23 +1,23 @@
-########
-#
-# sas_blob_utils.py
-#
-# This module contains helper functions for dealing with Shared Access Signatures
-# (SAS) tokens for Azure Blob Storage.
-#
-# The default Azure Storage SAS URI format is:
-#
-# https://<account>.blob.core.windows.net/<container>/<blob>?<sas_token>
-#
-# This module assumes azure-storage-blob version 12.5.
-#
-# Documentation for Azure Blob Storage:
-# docs.microsoft.com/en-us/azure/developer/python/sdk/storage/storage-blob-readme
-#
-# Documentation for SAS:
-# docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview
-#
-########
+"""
+
+sas_blob_utils.py
+
+This module contains helper functions for dealing with Shared Access Signatures
+(SAS) tokens for Azure Blob Storage.
+
+The default Azure Storage SAS URI format is:
+
+https://<account>.blob.core.windows.net/<container>/<blob>?<sas_token>
+
+This module assumes azure-storage-blob version 12.5.
+
+Documentation for Azure Blob Storage:
+docs.microsoft.com/en-us/azure/developer/python/sdk/storage/storage-blob-readme
+
+Documentation for SAS:
+docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview
+
+"""
 
 #%% Imports
 

md_utils/split_locations_into_train_val.py

@@ -1,15 +1,15 @@
-########
-#
-# split_locations_into_train_val.py
-#
-# Split a list of location IDs into training and validation, targeting a specific
-# train/val split for each category, but allowing some categories to be tighter or looser
-# than others.  Does nothing particularly clever, just randomly splits locations into 
-# train/val lots of times using the target val fraction, and picks the one that meets the 
-# specified constraints and minimizes weighted error, where "error" is defined as the
-# sum of each class's absolute divergence from the target val fraction.
-#
-########
+"""
+
+split_locations_into_train_val.py
+
+Splits a list of location IDs into training and validation, targeting a specific
+train/val split for each category, but allowing some categories to be tighter or looser
+than others.  Does nothing particularly clever, just randomly splits locations into 
+train/val lots of times using the target val fraction, and picks the one that meets the 
+specified constraints and minimizes weighted error, where "error" is defined as the
+sum of each class's absolute divergence from the target val fraction.
+
+"""
 
 #%% Imports/constants
 
@@ -30,31 +30,44 @@ def split_locations_into_train_val(location_to_category_counts,
                                    category_to_error_weight=None,
                                    default_max_allowable_error=0.1):
     """
-    Split a list of location IDs into training and validation, targeting a specific
+    Splits a list of location IDs into training and validation, targeting a specific
     train/val split for each category, but allowing some categories to be tighter or looser
     than others.  Does nothing particularly clever, just randomly splits locations into 
     train/val lots of times using the target val fraction, and picks the one that meets the 
     specified constraints and minimizes weighted error, where "error" is defined as the
     sum of each class's absolute divergence from the target val fraction.    
     
-    location_to_category_counts should be a dict mapping location IDs to dicts,
-    with each dict mapping a category name to a count.  Any categories not present in a 
-    particular dict are assumed to have a count of zero for that location.
-    
-    If not None, category_to_max_allowable_error should be a dict mapping category names
-    to maximum allowable errors.  These are hard constraints, but you can specify a subset
-    of categories.  Categories not included here have a maximum error of Inf.
-    
-    If not None, category_to_error_weight should be a dict mapping category names to
-    error weights.  You can specify a subset of categories.  Categories not included here
-    have a weight of 1.0.
-    
-    default_max_allowable_error is the maximum allowable error for categories not present in
-    category_to_max_allowable_error.  Set to None (or >= 1.0) to disable hard constraints for 
-    categories not present in category_to_max_allowable_error
-    
-    returns val_locations,category_to_val_fraction
-    
+    Args:
+        location_to_category_counts (dict): a dict mapping location IDs to dicts,
+            with each dict mapping a category name to a count.  Any categories not present 
+            in a particular dict are assumed to have a count of zero for that location.
+            
+            For example:
+                
+            .. code-block:: none
+
+                {'location-000': {'bear':4,'wolf':10},
+                 'location-001': {'bear':12,'elk':20}}
+    
+        n_random_seeds (int, optional): number of random seeds to try, always starting from zero
+        target_val_fraction (float, optional): fraction of images containing each species we'd
+            like to put in the val split
+        category_to_max_allowable_error (dict, optional): a dict mapping category names
+            to maximum allowable errors.  These are hard constraints (i.e., we will error
+            if we can't meet them).  Does not need to include all categories; categories not 
+            included will be assigned a maximum error according to [default_max_allowable_error].
+            If this is None, no hard constraints are applied.
+        category_to_error_weight (dict, optional): a dict mapping category names to
+            error weights.  You can specify a subset of categories; categories not included here
+            have a weight of 1.0.  If None, all categories have the same weight.
+        default_max_allowable_error (float, optional): the maximum allowable error for categories not 
+            present in [category_to_max_allowable_error].  Set to None (or >= 1.0) to disable hard 
+            constraints for categories not present in [category_to_max_allowable_error]
+    
+    Returns:
+        tuple: A two-element tuple:
+            - list of location IDs in the val split
+            - a dict mapping category names to the fraction of images in the val split                
     """
     
     location_ids = list(location_to_category_counts.keys())
@@ -84,7 +97,7 @@ def split_locations_into_train_val(location_to_category_counts,
     # random_seed = 0
     def compute_seed_errors(random_seed):
         """
-        Compute the per-category error for a specific random seed.
+        Computes the per-category error for a specific random seed.
         
         returns weighted_average_error,category_to_val_fraction
         """

md_utils/string_utils.py

@@ -1,16 +1,27 @@
-########
-#
-# string_utils.py
-#
-# Miscellaneous string utilities
-#
-########
+"""
+
+string_utils.py
+
+Miscellaneous string utilities.
+
+"""
+
+#%% Imports
 
 import re
 
+
+#%% Functions
+
 def is_float(s):
     """ 
-    Checks whether a string represents a valid float
+    Checks whether [s] is an object (typically a string) that can be cast to a float
+    
+    Args:
+        s (object): object to evaluate
+        
+    Returns:
+        bool: True if s successfully casts to a float, otherwise False
     """
     
     try:
@@ -23,10 +34,16 @@ def is_float(s):
 def human_readable_to_bytes(size):
     """
     Given a human-readable byte string (e.g. 2G, 10GB, 30MB, 20KB),
-    return the number of bytes.  Will return 0 if the argument has
+    returns the number of bytes.  Will return 0 if the argument has
     unexpected form.
     
     https://gist.github.com/beugley/ccd69945346759eb6142272a6d69b4e0
+    
+    Args:
+        size (str): string representing a size
+        
+    Returns:
+        int: the corresponding size in bytes
     """
     
     size = re.sub(r'\s+', '', size)
@@ -61,9 +78,15 @@ def human_readable_to_bytes(size):
 
 def remove_ansi_codes(s):
     """
-    Remove ANSI escape codes from a string.
+    Removes ANSI escape codes from a string.
     
     https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python#14693789
+    
+    Args:
+        s (str): the string to de-ANSI-i-fy
+        
+    Returns:
+        str: A copy of [s] without ANSI codes
     """
     ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
     return ansi_escape.sub('', s)