megadetector 10.0.15__py3-none-any.whl

megadetector/utils/process_utils.py

@@ -0,0 +1,172 @@
+"""
+
+process_utils.py
+
+Run something at the command line and capture the output, based on:
+
+https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
+
+Includes handy example code for doing this on multiple processes/threads.
+
+"""
+
+#%% Constants, imports, and environment
+
+import os
+import subprocess
+
+def execute(cmd,encoding=None,errors=None,env=None,verbose=False):
+    """
+    Run [cmd] (a single string) in a shell, yielding each line of output to the caller.
+
+    The "encoding", "errors", and "env" parameters are passed directly to subprocess.Popen().
+
+    "verbose" only impacts output about process management, it is not related to printing
+    output from the child process.
+
+    Args:
+        cmd (str): command to run
+        encoding (str, optional): stdout encoding, see Popen() documentation
+        errors (str, optional): error handling, see Popen() documentation
+        env (dict, optional): environment variables, see Popen() documentation
+        verbose (bool, optional): enable additional debug console output
+
+    Returns:
+        int: the command's return code, always zero, otherwise a CalledProcessError is raised
+    """
+
+    os.environ["PYTHONUNBUFFERED"] = "1"
+
+    if verbose:
+        if encoding is not None:
+            print('Launching child process with non-default encoding {}'.format(encoding))
+        if errors is not None:
+            print('Launching child process with non-default text error handling {}'.format(errors))
+        if env is not None:
+            print('Launching child process with non-default environment {}'.format(str(env)))
+
+    # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
+    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                             shell=True, universal_newlines=True, encoding=encoding,
+                             errors=errors, env=env)
+    for stdout_line in iter(popen.stdout.readline, ""):
+        yield stdout_line
+    popen.stdout.close()
+    return_code = popen.wait()
+    if return_code:
+        raise subprocess.CalledProcessError(return_code, cmd)
+
+    return return_code
+
+
+def execute_and_print(cmd,
+                      print_output=True,
+                      encoding=None,
+                      errors=None,
+                      env=None,
+                      verbose=False,
+                      catch_exceptions=True,
+                      echo_command=False):
+    """
+    Run [cmd] (a single string) in a shell, capturing and printing output.  Returns
+    a dictionary with fields "status" and "output".
+
+    The "encoding", "errors", and "env" parameters are passed directly to subprocess.Popen().
+
+    "verbose" only impacts output about process management, it is not related to printing
+    output from the child process.
+
+    Args:
+        cmd (str): command to run
+        print_output (bool, optional): whether to print output from [cmd] (stdout is
+            captured regardless of the value of print_output)
+        encoding (str, optional): stdout encoding, see Popen() documentation
+        errors (str, optional): error handling, see Popen() documentation
+        env (dict, optional): environment variables, see Popen() documentation
+        verbose (bool, optional): enable additional debug console output
+        catch_exceptions (bool, optional): catch exceptions and include in the output, otherwise raise
+        echo_command (bool, optional): print the command before executing
+
+    Returns:
+        dict: a dictionary with fields "status" (the process return code) and "output"
+        (the content of stdout)
+    """
+
+    if echo_command:
+        print('Running command:\n{}\n'.format(cmd))
+
+    to_return = {'status':'unknown','output':''}
+    output = []
+    try:
+        for s in execute(cmd,encoding=encoding,errors=errors,env=env,verbose=verbose):
+            output.append(s)
+            if print_output:
+                print(s,end='',flush=True)
+        to_return['status'] = 0
+    except subprocess.CalledProcessError as cpe:
+        if not catch_exceptions:
+            raise
+        print('execute_and_print caught error: {} ({})'.format(cpe.output,str(cpe)))
+        to_return['status'] = cpe.returncode
+    to_return['output'] = output
+
+    return to_return
+
+
+#%% Single-threaded test driver for execute_and_print
+
+if False:
+
+    pass
+
+    #%%
+
+    if os.name == 'nt':
+        execute_and_print('echo hello && ping -n 5 127.0.0.1 && echo goodbye')
+    else:
+        execute_and_print('echo hello && sleep 1 && echo goodbye')
+
+
+#%% Parallel test driver for execute_and_print
+
+if False:
+
+    pass
+
+    #%%
+
+    from functools import partial
+    from multiprocessing.pool import ThreadPool as ThreadPool
+    from multiprocessing.pool import Pool as Pool
+
+    n_workers = 10
+
+    # Should we use threads (vs. processes) for parallelization?
+    use_threads = True
+
+    test_data = ['a','b','c','d']
+
+    def _process_sample(s):
+        return execute_and_print('echo ' + s,True)
+
+    if n_workers == 1:
+
+        results = []
+        for i_sample,sample in enumerate(test_data):
+            results.append(_process_sample(sample))
+
+    else:
+
+        n_threads = min(n_workers,len(test_data))
+
+        if use_threads:
+            print('Starting parallel thread pool with {} workers'.format(n_threads))
+            pool = ThreadPool(n_threads)
+        else:
+            print('Starting parallel process pool with {} workers'.format(n_threads))
+            pool = Pool(n_threads)
+
+        results = list(pool.map(partial(_process_sample),test_data))
+
+        for r in results:
+            print(r)

megadetector/utils/split_locations_into_train_val.py

@@ -0,0 +1,237 @@
+"""
+
+split_locations_into_train_val.py
+
+Splits a list of location IDs into training and validation, targeting a specific
+train/val split for each category, but allowing some categories to be tighter or looser
+than others.  Does nothing particularly clever, just randomly splits locations into
+train/val lots of times using the target val fraction, and picks the one that meets the
+specified constraints and minimizes weighted error, where "error" is defined as the
+sum of each class's absolute divergence from the target val fraction.
+
+"""
+
+#%% Imports/constants
+
+import random
+import numpy as np
+
+from collections import defaultdict
+from megadetector.utils.ct_utils import sort_dictionary_by_value
+from tqdm import tqdm
+
+
+#%% Main function
+
+def split_locations_into_train_val(location_to_category_counts,
+                                   n_random_seeds=10000,
+                                   target_val_fraction=0.15,
+                                   category_to_max_allowable_error=None,
+                                   category_to_error_weight=None,
+                                   default_max_allowable_error=0.1,
+                                   require_complete_coverage=True):
+    """
+    Splits a list of location IDs into training and validation, targeting a specific
+    train/val split for each category, but allowing some categories to be tighter or looser
+    than others.  Does nothing particularly clever, just randomly splits locations into
+    train/val lots of times using the target val fraction, and picks the one that meets the
+    specified constraints and minimizes weighted error, where "error" is defined as the
+    sum of each class's absolute divergence from the target val fraction.
+
+    Args:
+        location_to_category_counts (dict): a dict mapping location IDs to dicts,
+            with each dict mapping a category name to a count.  Any categories not present
+            in a particular dict are assumed to have a count of zero for that location.
+
+            For example:
+
+            .. code-block:: none
+
+                {'location-000': {'bear':4,'wolf':10},
+                 'location-001': {'bear':12,'elk':20}}
+
+        n_random_seeds (int, optional): number of random seeds to try, always starting from zero
+        target_val_fraction (float, optional): fraction of images containing each species we'd
+            like to put in the val split
+        category_to_max_allowable_error (dict, optional): a dict mapping category names
+            to maximum allowable errors.  These are hard constraints (i.e., we will error
+            if we can't meet them).  Does not need to include all categories; categories not
+            included will be assigned a maximum error according to [default_max_allowable_error].
+            If this is None, no hard constraints are applied.
+        category_to_error_weight (dict, optional): a dict mapping category names to
+            error weights.  You can specify a subset of categories; categories not included here
+            have a weight of 1.0.  If None, all categories have the same weight.
+        default_max_allowable_error (float, optional): the maximum allowable error for categories not
+            present in [category_to_max_allowable_error].  Set to None (or >= 1.0) to disable hard
+            constraints for categories not present in [category_to_max_allowable_error]
+        require_complete_coverage (bool, optional): require that every category appear in both train
+            and val
+
+    Returns:
+        tuple: A two-element tuple:
+            - list of location IDs in the val split
+            - a dict mapping category names to the fraction of images in the val split
+    """
+
+    location_ids = list(location_to_category_counts.keys())
+
+    n_val_locations = int(target_val_fraction*len(location_ids))
+
+    if category_to_max_allowable_error is None:
+        category_to_max_allowable_error = {}
+
+    if category_to_error_weight is None:
+        category_to_error_weight = {}
+
+    # category ID to total count; the total count is used only for printouts
+    category_id_to_count = {}
+    for location_id in location_to_category_counts:
+        for category_id in location_to_category_counts[location_id].keys():
+            if category_id not in category_id_to_count:
+                category_id_to_count[category_id] = 0
+            category_id_to_count[category_id] += \
+                location_to_category_counts[location_id][category_id]
+
+    category_ids = set(category_id_to_count.keys())
+
+    print('Splitting {} categories over {} locations'.format(
+        len(category_ids),len(location_ids)))
+
+    # random_seed = 0
+    def compute_seed_errors(random_seed):
+        """
+        Computes the per-category error for a specific random seed.
+
+        returns weighted_average_error,category_to_val_fraction
+        """
+
+        # Randomly split into train/val
+        random.seed(random_seed)
+        val_locations = random.sample(location_ids,k=n_val_locations)
+        val_locations_set = set(val_locations)
+
+        # For each category, measure the % of images that went into the val set
+        category_to_val_fraction = defaultdict(float)
+
+        for category_id in category_ids:
+            category_val_count = 0
+            category_train_count = 0
+            for location_id in location_to_category_counts:
+                if category_id not in location_to_category_counts[location_id]:
+                    location_category_count = 0
+                else:
+                    location_category_count = location_to_category_counts[location_id][category_id]
+                if location_id in val_locations_set:
+                    category_val_count += location_category_count
+                else:
+                    category_train_count += location_category_count
+            category_val_fraction = category_val_count / (category_val_count + category_train_count)
+            category_to_val_fraction[category_id] = category_val_fraction
+
+        # Absolute deviation from the target val fraction for each category
+        category_errors = {}
+        weighted_category_errors = {}
+
+        # category = next(iter(category_to_val_fraction))
+        for category in category_to_val_fraction:
+
+            category_val_fraction = category_to_val_fraction[category]
+
+            category_error = abs(category_val_fraction-target_val_fraction)
+            category_errors[category] = category_error
+
+            category_weight = 1.0
+            if category in category_to_error_weight:
+                category_weight = category_to_error_weight[category]
+            weighted_category_error = category_error * category_weight
+            weighted_category_errors[category] = weighted_category_error
+
+        weighted_average_error = np.mean(list(weighted_category_errors.values()))
+
+        return weighted_average_error,weighted_category_errors,category_to_val_fraction
+
+    # ... def compute_seed_errors(...)
+
+    # This will only include random seeds that satisfy the hard constraints
+    random_seed_to_weighted_average_error = {}
+
+    # random_seed = 0
+    for random_seed in tqdm(range(0,n_random_seeds)):
+
+        weighted_average_error,weighted_category_errors,category_to_val_fraction = \
+            compute_seed_errors(random_seed)
+
+        seed_satisfies_hard_constraints = True
+
+        for category in category_to_val_fraction:
+            if category in category_to_max_allowable_error:
+                max_allowable_error = category_to_max_allowable_error[category]
+            else:
+                if default_max_allowable_error is None:
+                    continue
+                max_allowable_error = default_max_allowable_error
+            val_fraction = category_to_val_fraction[category]
+
+            # If necessary, verify that this category doesn't *only* appear in train or val
+            if require_complete_coverage:
+                if (val_fraction == 0.0) or (val_fraction == 1.0):
+                    seed_satisfies_hard_constraints = False
+                    break
+
+            # Check whether this category exceeds the hard maximum deviation
+            category_error = abs(val_fraction - target_val_fraction)
+            if category_error > max_allowable_error:
+                seed_satisfies_hard_constraints = False
+                break
+
+        # ...for each category
+
+        if seed_satisfies_hard_constraints:
+            random_seed_to_weighted_average_error[random_seed] = weighted_average_error
+
+    # ...for each random seed
+
+    assert len(random_seed_to_weighted_average_error) > 0, \
+        'No random seed met all the hard constraints'
+
+    print('\n{} of {} random seeds satisfied hard constraints'.format(
+        len(random_seed_to_weighted_average_error),n_random_seeds))
+
+    min_error = None
+    min_error_seed = None
+
+    for random_seed in random_seed_to_weighted_average_error.keys():
+        error_metric = random_seed_to_weighted_average_error[random_seed]
+        if min_error is None or error_metric < min_error:
+            min_error = error_metric
+            min_error_seed = random_seed
+
+    random.seed(min_error_seed)
+    val_locations = random.sample(location_ids,k=n_val_locations)
+    train_locations = []
+    for location_id in location_ids:
+        if location_id not in val_locations:
+            train_locations.append(location_id)
+
+    print('\nVal locations:\n')
+    for loc in val_locations:
+        print('{}'.format(loc))
+    print('')
+
+    weighted_average_error,weighted_category_errors,category_to_val_fraction = \
+        compute_seed_errors(min_error_seed)
+
+    category_to_val_fraction = sort_dictionary_by_value(category_to_val_fraction,
+                                                        sort_values=category_id_to_count,
+                                                        reverse=True)
+
+    print('Val fractions by category:\n')
+
+    for category in category_to_val_fraction:
+        print('{} ({}) {:.2f}'.format(
+            category,category_id_to_count[category],
+            category_to_val_fraction[category]))
+
+    return val_locations,category_to_val_fraction
+
+# ...def split_locations_into_train_val(...)

megadetector/utils/string_utils.py

@@ -0,0 +1,234 @@
+"""
+
+string_utils.py
+
+Miscellaneous string utilities.
+
+"""
+
+#%% Imports
+
+import re
+
+
+#%% Functions
+
+def is_float(s):
+    """
+    Checks whether [s] is an object (typically a string) that can be cast to a float
+
+    Args:
+        s (object): object to evaluate
+
+    Returns:
+        bool: True if s successfully casts to a float, otherwise False
+    """
+
+    if s is None:
+        return False
+
+    try:
+        _ = float(s)
+    except ValueError:
+        return False
+    return True
+
+
+def is_int(s):
+    """
+    Checks whether [s] is an object (typically a string) that can be cast to a int
+
+    Args:
+        s (object): object to evaluate
+
+    Returns:
+        bool: True if s successfully casts to a int, otherwise False
+    """
+
+    if s is None:
+        return False
+
+    try:
+        _ = int(s)
+    except ValueError:
+        return False
+    return True
+
+
+def human_readable_to_bytes(size):
+    """
+    Given a human-readable byte string (e.g. 2G, 10GB, 30MB, 20KB),
+    returns the number of bytes.  Will return 0 if the argument has
+    unexpected form.
+
+    https://gist.github.com/beugley/ccd69945346759eb6142272a6d69b4e0
+
+    Args:
+        size (str): string representing a size
+
+    Returns:
+        int: the corresponding size in bytes
+    """
+
+    size = re.sub(r'\s+', '', size)
+
+    if not size: # Handle empty string case after stripping spaces
+        return 0
+
+    if (size[-1] == 'B'):
+        size = size[:-1]
+
+    if not size: # Handle case where size was just "B"
+        return 0
+
+    if (size.isdigit()):
+        bytes_val = int(size) # Renamed to avoid conflict with built-in 'bytes'
+    elif (is_float(size)):
+        bytes_val = float(size) # Renamed
+    else:
+        # Handle cases like "1KB" where size[:-1] might be "1K" before this block
+        # The original code would try to float("1K") which fails.
+        # Need to separate numeric part from unit more carefully.
+        numeric_part = ''
+        unit_part = ''
+
+        # Iterate from the end to find the unit (K, M, G, T)
+        # This handles cases like "10KB" or "2.5GB"
+        for i in range(len(size) -1, -1, -1):
+            if size[i].isalpha():
+                unit_part = size[i] + unit_part
+            else:
+                numeric_part = size[:i+1]
+                break
+
+        # If no unit found, or numeric part is empty after stripping unit
+        if not unit_part or not numeric_part:
+            return 0
+
+        try:
+            bytes_val = float(numeric_part)
+            unit = unit_part
+            if (unit == 'T'):
+                bytes_val *= 1024*1024*1024*1024
+            elif (unit == 'G'):
+                bytes_val *= 1024*1024*1024
+            elif (unit == 'M'):
+                bytes_val *= 1024*1024
+            elif (unit == 'K'):
+                bytes_val *= 1024
+            else:
+                # If it's a known unit (like 'B' already stripped) but not T/G/M/K,
+                # and it was floatable, it's just bytes.  If it's an unknown unit, it's
+                # an error.
+                if unit not in ['B', '']: # 'B' was stripped, '' means just a number
+                     bytes_val = 0
+        except ValueError:
+            bytes_val = 0
+
+    return bytes_val
+
+
+def remove_ansi_codes(s):
+    """
+    Removes ANSI escape codes from a string.
+
+    https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python#14693789
+
+    Args:
+        s (str): the string to de-ANSI-i-fy
+
+    Returns:
+        str: A copy of [s] without ANSI codes
+    """
+
+    ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+    return ansi_escape.sub('', s)
+
+
+#%% Tests
+
+
+class TestStringUtils:
+    """
+    Tests for string_utils.py
+    """
+
+
+    def test_is_float(self):
+        """
+        Test the is_float function.
+        """
+
+        assert is_float("1.23")
+        assert is_float("-0.5")
+        assert is_float("0")
+        assert is_float(1.23)
+        assert is_float(0)
+        assert not is_float("abc")
+        assert not is_float("1.2.3")
+        assert not is_float("")
+        assert not is_float(None)
+        assert not is_float("1,23")
+
+
+    def test_human_readable_to_bytes(self):
+        """
+        Test the human_readable_to_bytes function.
+        """
+
+        assert human_readable_to_bytes("10B") == 10
+        assert human_readable_to_bytes("10") == 10
+        assert human_readable_to_bytes("1K") == 1024
+        assert human_readable_to_bytes("1KB") == 1024
+        assert human_readable_to_bytes("1M") == 1024*1024
+        assert human_readable_to_bytes("1MB") == 1024*1024
+        assert human_readable_to_bytes("1G") == 1024*1024*1024
+        assert human_readable_to_bytes("1GB") == 1024*1024*1024
+        assert human_readable_to_bytes("1T") == 1024*1024*1024*1024
+        assert human_readable_to_bytes("1TB") == 1024*1024*1024*1024
+
+        assert human_readable_to_bytes("2.5K") == 2.5 * 1024
+        assert human_readable_to_bytes("0.5MB") == 0.5 * 1024 * 1024
+
+        # Test with spaces
+        assert human_readable_to_bytes(" 2 G ") == 2 * 1024*1024*1024
+        assert human_readable_to_bytes("500 KB") == 500 * 1024
+
+        # Invalid inputs
+        assert human_readable_to_bytes("abc") == 0
+        assert human_readable_to_bytes("1X") == 0
+        assert human_readable_to_bytes("1KBB") == 0
+        assert human_readable_to_bytes("K1") == 0
+        assert human_readable_to_bytes("") == 0
+        assert human_readable_to_bytes("1.2.3K") == 0
+        assert human_readable_to_bytes("B") == 0
+
+
+    def test_remove_ansi_codes(self):
+        """
+        Test the remove_ansi_codes function.
+        """
+
+        assert remove_ansi_codes("text without codes") == "text without codes"
+        assert remove_ansi_codes("\x1b[31mRed text\x1b[0m") == "Red text"
+        assert remove_ansi_codes("\x1b[1m\x1b[4mBold and Underline\x1b[0m") == "Bold and Underline"
+        assert remove_ansi_codes("Mixed \x1b[32mgreen\x1b[0m and normal") == "Mixed green and normal"
+        assert remove_ansi_codes("") == ""
+
+        # More complex/varied ANSI codes
+        assert remove_ansi_codes("text\x1b[1Aup") == "textup"
+        assert remove_ansi_codes("\x1b[2Jclearscreen") == "clearscreen"
+
+
+def test_string_utils():
+    """
+    Runs all tests in the TestStringUtils class.
+    """
+
+    test_instance = TestStringUtils()
+    test_instance.test_is_float()
+    test_instance.test_human_readable_to_bytes()
+    test_instance.test_remove_ansi_codes()
+
+# from IPython import embed; embed()
+# test_string_utils()