megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/utils/split_locations_into_train_val.py

@@ -4,8 +4,8 @@ split_locations_into_train_val.py
 
 Splits a list of location IDs into training and validation, targeting a specific
 train/val split for each category, but allowing some categories to be tighter or looser
-than others.  Does nothing particularly clever, just randomly splits locations into 
-train/val lots of times using the target val fraction, and picks the one that meets the 
+than others.  Does nothing particularly clever, just randomly splits locations into
+train/val lots of times using the target val fraction, and picks the one that meets the
 specified constraints and minimizes weighted error, where "error" is defined as the
 sum of each class's absolute divergence from the target val fraction.
 
@@ -26,60 +26,63 @@ from tqdm import tqdm
 def split_locations_into_train_val(location_to_category_counts,
                                    n_random_seeds=10000,
                                    target_val_fraction=0.15,
-                                   category_to_max_allowable_error=None,                                   
+                                   category_to_max_allowable_error=None,
                                    category_to_error_weight=None,
-                                   default_max_allowable_error=0.1):
+                                   default_max_allowable_error=0.1,
+                                   require_complete_coverage=True):
     """
     Splits a list of location IDs into training and validation, targeting a specific
     train/val split for each category, but allowing some categories to be tighter or looser
-    than others.  Does nothing particularly clever, just randomly splits locations into 
-    train/val lots of times using the target val fraction, and picks the one that meets the 
+    than others.  Does nothing particularly clever, just randomly splits locations into
+    train/val lots of times using the target val fraction, and picks the one that meets the
     specified constraints and minimizes weighted error, where "error" is defined as the
-    sum of each class's absolute divergence from the target val fraction.    
-    
+    sum of each class's absolute divergence from the target val fraction.
+
     Args:
         location_to_category_counts (dict): a dict mapping location IDs to dicts,
-            with each dict mapping a category name to a count.  Any categories not present 
+            with each dict mapping a category name to a count.  Any categories not present
             in a particular dict are assumed to have a count of zero for that location.
-            
+
             For example:
-                
+
             .. code-block:: none
 
                 {'location-000': {'bear':4,'wolf':10},
                  'location-001': {'bear':12,'elk':20}}
-    
+
         n_random_seeds (int, optional): number of random seeds to try, always starting from zero
         target_val_fraction (float, optional): fraction of images containing each species we'd
             like to put in the val split
         category_to_max_allowable_error (dict, optional): a dict mapping category names
             to maximum allowable errors.  These are hard constraints (i.e., we will error
-            if we can't meet them).  Does not need to include all categories; categories not 
+            if we can't meet them).  Does not need to include all categories; categories not
             included will be assigned a maximum error according to [default_max_allowable_error].
             If this is None, no hard constraints are applied.
         category_to_error_weight (dict, optional): a dict mapping category names to
             error weights.  You can specify a subset of categories; categories not included here
             have a weight of 1.0.  If None, all categories have the same weight.
-        default_max_allowable_error (float, optional): the maximum allowable error for categories not 
-            present in [category_to_max_allowable_error].  Set to None (or >= 1.0) to disable hard 
+        default_max_allowable_error (float, optional): the maximum allowable error for categories not
+            present in [category_to_max_allowable_error].  Set to None (or >= 1.0) to disable hard
             constraints for categories not present in [category_to_max_allowable_error]
-    
+        require_complete_coverage (bool, optional): require that every category appear in both train and
+            val
+
     Returns:
         tuple: A two-element tuple:
             - list of location IDs in the val split
-            - a dict mapping category names to the fraction of images in the val split                
+            - a dict mapping category names to the fraction of images in the val split
     """
-    
+
     location_ids = list(location_to_category_counts.keys())
-    
+
     n_val_locations = int(target_val_fraction*len(location_ids))
-    
+
     if category_to_max_allowable_error is None:
         category_to_max_allowable_error = {}
-        
+
     if category_to_error_weight is None:
         category_to_error_weight = {}
-        
+
     # category ID to total count; the total count is used only for printouts
     category_id_to_count = {}
     for location_id in location_to_category_counts:
@@ -88,28 +91,28 @@ def split_locations_into_train_val(location_to_category_counts,
                 category_id_to_count[category_id] = 0
             category_id_to_count[category_id] += \
                 location_to_category_counts[location_id][category_id]
-    
+
     category_ids = set(category_id_to_count.keys())
-    
+
     print('Splitting {} categories over {} locations'.format(
         len(category_ids),len(location_ids)))
-    
+
     # random_seed = 0
     def compute_seed_errors(random_seed):
         """
         Computes the per-category error for a specific random seed.
-        
+
         returns weighted_average_error,category_to_val_fraction
         """
-        
+
         # Randomly split into train/val
         random.seed(random_seed)
         val_locations = random.sample(location_ids,k=n_val_locations)
         val_locations_set = set(val_locations)
-        
+
         # For each category, measure the % of images that went into the val set
         category_to_val_fraction = defaultdict(float)
-        
+
         for category_id in category_ids:
             category_val_count = 0
             category_train_count = 0
@@ -124,42 +127,42 @@ def split_locations_into_train_val(location_to_category_counts,
                     category_train_count += location_category_count
             category_val_fraction = category_val_count / (category_val_count + category_train_count)
             category_to_val_fraction[category_id] = category_val_fraction
-        
-        # Absolute deviation from the target val fraction for each categorys
+
+        # Absolute deviation from the target val fraction for each category
         category_errors = {}
         weighted_category_errors = {}
-        
+
         # category = next(iter(category_to_val_fraction))
         for category in category_to_val_fraction:
-            
+
             category_val_fraction = category_to_val_fraction[category]
-            
+
             category_error = abs(category_val_fraction-target_val_fraction)
             category_errors[category] = category_error
-        
+
             category_weight = 1.0
             if category in category_to_error_weight:
                 category_weight = category_to_error_weight[category]
             weighted_category_error = category_error * category_weight
             weighted_category_errors[category] = weighted_category_error
-        
+
         weighted_average_error = np.mean(list(weighted_category_errors.values()))
-        
+
         return weighted_average_error,weighted_category_errors,category_to_val_fraction
-    
+
     # ... def compute_seed_errors(...)
-    
+
     # This will only include random seeds that satisfy the hard constraints
     random_seed_to_weighted_average_error = {}
-    
+
     # random_seed = 0
     for random_seed in tqdm(range(0,n_random_seeds)):
-        
+
         weighted_average_error,weighted_category_errors,category_to_val_fraction = \
             compute_seed_errors(random_seed)
-            
+
         seed_satisfies_hard_constraints = True
-        
+
         for category in category_to_val_fraction:
             if category in category_to_max_allowable_error:
                 max_allowable_error = category_to_max_allowable_error[category]
@@ -168,61 +171,71 @@ def split_locations_into_train_val(location_to_category_counts,
                     continue
                 max_allowable_error = default_max_allowable_error
             val_fraction = category_to_val_fraction[category]
+
+            # If necessary, verify that this category doesn't *only* appear in train or val
+            if require_complete_coverage:
+                if (val_fraction == 0.0) or (val_fraction == 1.0):
+                    seed_satisfies_hard_constraints = False
+                    break
+
+            # Check whether this category exceeds the hard maximum deviation
             category_error = abs(val_fraction - target_val_fraction)
             if category_error > max_allowable_error:
                 seed_satisfies_hard_constraints = False
                 break
-        
-        if seed_satisfies_hard_constraints:            
+
+        # ...for each category
+
+        if seed_satisfies_hard_constraints:
             random_seed_to_weighted_average_error[random_seed] = weighted_average_error
-        
+
     # ...for each random seed
-    
+
     assert len(random_seed_to_weighted_average_error) > 0, \
         'No random seed met all the hard constraints'
-        
+
     print('\n{} of {} random seeds satisfied hard constraints'.format(
         len(random_seed_to_weighted_average_error),n_random_seeds))
-    
+
     min_error = None
     min_error_seed = None
-    
+
     for random_seed in random_seed_to_weighted_average_error.keys():
         error_metric = random_seed_to_weighted_average_error[random_seed]
         if min_error is None or error_metric < min_error:
             min_error = error_metric
             min_error_seed = random_seed
-    
+
     random.seed(min_error_seed)
     val_locations = random.sample(location_ids,k=n_val_locations)
     train_locations = []
     for location_id in location_ids:
         if location_id not in val_locations:
             train_locations.append(location_id)
-            
-    print('\nVal locations:\n')        
+
+    print('\nVal locations:\n')
     for loc in val_locations:
         print('{}'.format(loc))
     print('')
-    
+
     weighted_average_error,weighted_category_errors,category_to_val_fraction = \
         compute_seed_errors(min_error_seed)
-        
+
     random_seed = min_error_seed
-    
+
     category_to_val_fraction = sort_dictionary_by_value(category_to_val_fraction,reverse=True)
     category_to_val_fraction = sort_dictionary_by_value(category_to_val_fraction,
                                                         sort_values=category_id_to_count,
                                                         reverse=True)
-    
-    
+
+
     print('Val fractions by category:\n')
-    
+
     for category in category_to_val_fraction:
         print('{} ({}) {:.2f}'.format(
             category,category_id_to_count[category],
             category_to_val_fraction[category]))
-          
+
     return val_locations,category_to_val_fraction
 
 # ...def split_locations_into_train_val(...)

megadetector/utils/string_utils.py

@@ -14,15 +14,18 @@ import re
 #%% Functions
 
 def is_float(s):
-    """ 
+    """
     Checks whether [s] is an object (typically a string) that can be cast to a float
-    
+
     Args:
         s (object): object to evaluate
-        
+
     Returns:
         bool: True if s successfully casts to a float, otherwise False
     """
+
+    if s is None:
+        return False
     
     try:
         _ = float(s)
@@ -36,57 +39,175 @@ def human_readable_to_bytes(size):
     Given a human-readable byte string (e.g. 2G, 10GB, 30MB, 20KB),
     returns the number of bytes.  Will return 0 if the argument has
     unexpected form.
-    
+
     https://gist.github.com/beugley/ccd69945346759eb6142272a6d69b4e0
-    
+
     Args:
         size (str): string representing a size
-        
+
     Returns:
         int: the corresponding size in bytes
     """
-    
+
     size = re.sub(r'\s+', '', size)
-    
+
+    if not size: # Handle empty string case after stripping spaces
+        return 0
+        
     if (size[-1] == 'B'):
         size = size[:-1]
-        
+
+    if not size: # Handle case where size was just "B"
+        return 0
+
     if (size.isdigit()):
-        bytes = int(size)
+        bytes_val = int(size) # Renamed to avoid conflict with built-in 'bytes'
     elif (is_float(size)):
-        bytes = float(size)
+        bytes_val = float(size) # Renamed
     else:
-        bytes = size[:-1]
-        unit = size[-1]
-        try:        
-            bytes = float(bytes)
+        # Handle cases like "1KB" where size[:-1] might be "1K" before this block
+        # The original code would try to float("1K") which fails.
+        # Need to separate numeric part from unit more carefully.
+        numeric_part = ''
+        unit_part = ''
+        
+        # Iterate from the end to find the unit (K, M, G, T)
+        # This handles cases like "10KB" or "2.5GB"
+        for i in range(len(size) -1, -1, -1):
+            if size[i].isalpha():
+                unit_part = size[i] + unit_part
+            else:
+                numeric_part = size[:i+1]
+                break
+        
+        # If no unit found, or numeric part is empty after stripping unit
+        if not unit_part or not numeric_part:
+            return 0
+
+        try:
+            bytes_val = float(numeric_part)
+            unit = unit_part
             if (unit == 'T'):
-                bytes *= 1024*1024*1024*1024
+                bytes_val *= 1024*1024*1024*1024
             elif (unit == 'G'):
-                bytes *= 1024*1024*1024
+                bytes_val *= 1024*1024*1024
             elif (unit == 'M'):
-                bytes *= 1024*1024
+                bytes_val *= 1024*1024
             elif (unit == 'K'):
-                bytes *= 1024
+                bytes_val *= 1024
             else:
-                bytes = 0
+                # If it's a known unit (like 'B' already stripped) but not T/G/M/K,
+                # and it was floatable, it's just bytes.  If it's an unknown unit, it's 
+				# an error.
+                if unit not in ['B', '']: # 'B' was stripped, '' means just a number
+                     bytes_val = 0
         except ValueError:
-            bytes = 0
-            
-    return bytes
+            bytes_val = 0
+
+    return bytes_val
 
 
 def remove_ansi_codes(s):
     """
     Removes ANSI escape codes from a string.
-    
+
     https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python#14693789
-    
+
     Args:
         s (str): the string to de-ANSI-i-fy
-        
+
     Returns:
         str: A copy of [s] without ANSI codes
     """
+
     ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
     return ansi_escape.sub('', s)
+
+
+#%% Tests
+
+
+class TestStringUtils:
+    """
+    Tests for string_utils.py
+    """
+
+
+    def test_is_float(self):
+        """
+        Test the is_float function.
+        """
+
+        assert is_float("1.23")
+        assert is_float("-0.5")
+        assert is_float("0")
+        assert is_float(1.23)
+        assert is_float(0)
+        assert not is_float("abc")
+        assert not is_float("1.2.3")
+        assert not is_float("")
+        assert not is_float(None)
+        assert not is_float("1,23")
+
+
+    def test_human_readable_to_bytes(self):
+        """
+        Test the human_readable_to_bytes function.
+        """
+
+        assert human_readable_to_bytes("10B") == 10
+        assert human_readable_to_bytes("10") == 10
+        assert human_readable_to_bytes("1K") == 1024
+        assert human_readable_to_bytes("1KB") == 1024
+        assert human_readable_to_bytes("1M") == 1024*1024
+        assert human_readable_to_bytes("1MB") == 1024*1024
+        assert human_readable_to_bytes("1G") == 1024*1024*1024
+        assert human_readable_to_bytes("1GB") == 1024*1024*1024
+        assert human_readable_to_bytes("1T") == 1024*1024*1024*1024
+        assert human_readable_to_bytes("1TB") == 1024*1024*1024*1024
+        
+        assert human_readable_to_bytes("2.5K") == 2.5 * 1024
+        assert human_readable_to_bytes("0.5MB") == 0.5 * 1024 * 1024
+
+        # Test with spaces
+        assert human_readable_to_bytes(" 2 G ") == 2 * 1024*1024*1024
+        assert human_readable_to_bytes("500 KB") == 500 * 1024
+
+        # Invalid inputs
+        assert human_readable_to_bytes("abc") == 0
+        assert human_readable_to_bytes("1X") == 0
+        assert human_readable_to_bytes("1KBB") == 0
+        assert human_readable_to_bytes("K1") == 0
+        assert human_readable_to_bytes("") == 0
+        assert human_readable_to_bytes("1.2.3K") == 0
+        assert human_readable_to_bytes("B") == 0
+
+
+    def test_remove_ansi_codes(self):
+        """
+        Test the remove_ansi_codes function.
+        """
+
+        assert remove_ansi_codes("text without codes") == "text without codes"
+        assert remove_ansi_codes("\x1b[31mRed text\x1b[0m") == "Red text"
+        assert remove_ansi_codes("\x1b[1m\x1b[4mBold and Underline\x1b[0m") == "Bold and Underline"
+        assert remove_ansi_codes("Mixed \x1b[32mgreen\x1b[0m and normal") == "Mixed green and normal"
+        assert remove_ansi_codes("") == ""
+
+        # More complex/varied ANSI codes
+        assert remove_ansi_codes("text\x1b[1Aup") == "textup"
+        assert remove_ansi_codes("\x1b[2Jclearscreen") == "clearscreen"
+
+
+def test_string_utils():
+    """
+    Runs all tests in the TestStringUtils class.
+    """
+
+    test_instance = TestStringUtils()
+    test_instance.test_is_float()
+    test_instance.test_human_readable_to_bytes()
+    test_instance.test_remove_ansi_codes()
+
+# from IPython import embed; embed()
+# test_string_utils()