megadetector 5.0.28__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/taxonomy_mapping/preview_lila_taxonomy.py

@@ -27,20 +27,20 @@ html_output_file = os.path.join(preview_base,'index.html')
 
 def parse_taxonomy_string(taxonomy_string):
 
-    taxonomic_match = eval(taxonomy_string)        
+    taxonomic_match = eval(taxonomy_string)
     matched_entity = taxonomic_match[0]
     assert len(matched_entity) == 4
-    
+
     level = matched_entity[1]
-    
+
     scientific_name = matched_entity[2]
-    
+
     common_names = matched_entity[3]
     if len(common_names) == 1:
         common_name = common_names[0]
     else:
         common_name = str(common_names)
-    
+
     return scientific_name,common_name,level,taxonomic_match
 
 def taxonomy_string_to_common_name(taxonomy_string):
@@ -79,14 +79,14 @@ n_taxonomy_changes = 0
 
 # Look for internal inconsistency
 for i_row,row in df.iterrows():
-    
+
     sn = row['scientific_name']
     if not isinstance(sn,str):
         continue
-    
-    ts = row['taxonomy_string'] 
+
+    ts = row['taxonomy_string']
     assert sn == taxonomy_string_to_scientific(ts)
-    
+
     assert row['taxonomy_level'] == taxonomy_string_to_level(ts)
 
 # Look for outdated mappings
@@ -94,18 +94,18 @@ taxonomy_preference = 'inat'
 
 # i_row = 0; row = df.iloc[i_row]
 for i_row,row in tqdm(df.iterrows(),total=len(df)):
-    
+
     sn = row['scientific_name']
     if not isinstance(sn,str):
         continue
-    
+
     m = get_preferred_taxonomic_match(sn,taxonomy_preference)
     assert m.scientific_name == sn
-    
+
     ts = row['taxonomy_string']
     assert m.taxonomy_string[0:50] == ts[0:50], 'Mismatch for {}:\n\n{}\n\n{}\n'.format(
         row['dataset_name'],ts,m.taxonomy_string)
-        
+
     if ts != m.taxonomy_string:
         n_taxonomy_changes += 1
         df.loc[i_row,'taxonomy_string'] = m.taxonomy_string
@@ -161,45 +161,45 @@ suppress_multiple_matches = [
     ['porcupine','Snapshot Kruger','Idaho Camera Traps'],
     ['porcupine','Snapshot Mountain Zebra','Idaho Camera Traps'],
     ['porcupine','Snapshot Serengeti','Idaho Camera Traps'],
-    
+
     ['porcupine','Snapshot Serengeti','Snapshot Mountain Zebra'],
     ['porcupine','Snapshot Serengeti','Snapshot Kruger'],
     ['porcupine','Snapshot Serengeti','Snapshot Kgalagadi'],
     ['porcupine','Snapshot Serengeti','Snapshot Karoo'],
     ['porcupine','Snapshot Serengeti','Snapshot Camdeboo'],
-    
+
     ['porcupine','Snapshot Enonkishu','Snapshot Camdeboo'],
     ['porcupine','Snapshot Enonkishu','Snapshot Mountain Zebra'],
     ['porcupine','Snapshot Enonkishu','Snapshot Kruger'],
     ['porcupine','Snapshot Enonkishu','Snapshot Kgalagadi'],
     ['porcupine','Snapshot Enonkishu','Snapshot Karoo'],
-    
+
     ['kudu','Snapshot Serengeti','Snapshot Mountain Zebra'],
     ['kudu','Snapshot Serengeti','Snapshot Kruger'],
     ['kudu','Snapshot Serengeti','Snapshot Kgalagadi'],
     ['kudu','Snapshot Serengeti','Snapshot Karoo'],
     ['kudu','Snapshot Serengeti','Snapshot Camdeboo'],
-    
+
     ['fox','Caltech Camera Traps','Channel Islands Camera Traps'],
     ['fox','Idaho Camera Traps','Channel Islands Camera Traps'],
     ['fox','Idaho Camera Traps','Caltech Camera Traps'],
-    
+
     ['pangolin','Snapshot Serengeti','SWG Camera Traps'],
-    
+
     ['deer', 'Wellington Camera Traps', 'Idaho Camera Traps'],
     ['deer', 'Wellington Camera Traps', 'Caltech Camera Traps'],
-    
+
     ['unknown cervid', 'WCS Camera Traps', 'Idaho Camera Traps']
-    
+
 ]
 
 for i_row,row in df.iterrows():
-    
+
     query = row['query']
     taxonomy_string = row['taxonomy_string']
-    
+
     for previous_i_row in query_to_rows[query]:
-        
+
         previous_row = df.iloc[previous_i_row]
         assert previous_row['query'] == query
         query_match = False
@@ -209,11 +209,11 @@ for i_row,row in df.iterrows():
             query_match = isnan(row['taxonomy_string'])
         else:
             query_match = previous_row['taxonomy_string'][0:10] == taxonomy_string[0:10]
-        
+
         if not query_match:
-            
+
             suppress = False
-            
+
             # x = suppress_multiple_matches[-1]
             for x in suppress_multiple_matches:
                 if x[0] == query and \
@@ -225,18 +225,18 @@ for i_row,row in df.iterrows():
                     suppress = True
                     n_suppressed += 1
                     break
-                
+
             if not suppress:
                 print('Query {} in {} and {}:\n\n{}\n\n{}\n'.format(
                     query, row['dataset_name'], previous_row['dataset_name'],
                     taxonomy_string, previous_row['taxonomy_string']))
-                
+
             queries_with_multiple_mappings.add(query)
-                
+
     # ...for each row where we saw this query
-    
+
     query_to_rows[query].append(i_row)
-    
+
 # ...for each row
 
 print('Found {} queries with multiple mappings ({} occurrences suppressed)'.format(
@@ -267,9 +267,9 @@ for i_row,row in df.iterrows():
         ) \
         and \
         ('species' in level):
-        
+
         if query not in allowable_unknown_species:
-            
+
             print('Warning: query {}:{} maps to {} {}'.format(
                 row['dataset_name'],
                 row['query'],
@@ -285,7 +285,7 @@ for i_row,row in df.iterrows():
         if 'source' in row:
             assert isinstance(row['source'],str)
         assert isinstance(row['taxonomy_level'],str)
-        
+
 
 #%% Find WCS mappings that aren't species or aren't the same as the input
 
@@ -294,22 +294,22 @@ for i_row,row in df.iterrows():
 
 # row = df.iloc[-500]
 for i_row,row in df.iterrows():
-    
+
     if not isinstance(row['scientific_name'],str):
         continue
     if 'WCS' not in row['dataset_name']:
         continue
-    
+
     query = row['query']
     scientific_name = row['scientific_name']
     common_name = row['common_name']
-    level = row['taxonomy_level']    
+    level = row['taxonomy_level']
     taxonomy_string = row['taxonomy_string']
-    
-    common_name_from_taxonomy = taxonomy_string_to_common_name(taxonomy_string)   
+
+    common_name_from_taxonomy = taxonomy_string_to_common_name(taxonomy_string)
     query_string = query.replace(' sp','')
     query_string = query_string.replace('unknown ','')
-    
+
     # Anything marked "species" or "unknown" by definition doesn't map to a species,
     # so ignore these.
     if (' sp' not in query) and ('unknown' not in query) and \
@@ -317,7 +317,7 @@ for i_row,row in df.iterrows():
         print('WCS query {} ({}) remapped to {} {} ({})'.format(
             query,common_name,level,scientific_name,common_name_from_taxonomy))
 
-    if query_string != scientific_name:        
+    if query_string != scientific_name:
         pass
         # print('WCS query {} ({}) remapped to {} ({})'.format(
         #     query,common_name,scientific_name,common_names_from_taxonomy))
@@ -345,20 +345,20 @@ min_valid_image_size = 3000
 #
 # i_row = 0; row = df.iloc[i_row]
 for i_row,row in df.iterrows():
-    
+
     s = row['scientific_name']
-    
+
     if (not isinstance(s,str)) or (len(s)==0):
         continue
-    
+
     query = s.replace(' ','+')
-    
+
     if query in remapped_queries:
         query = remapped_queries[query]
-        
+
     query_folder = os.path.join(image_base,query)
     os.makedirs(query_folder,exist_ok=True)
-    
+
     # Check whether we already have enough images for this query
     image_files = os.listdir(query_folder)
     image_fullpaths = [os.path.join(query_folder,fn) for fn in image_files]
@@ -371,7 +371,7 @@ for i_row,row in df.iterrows():
     # Check whether we've already run this query for a previous row
     if query in scientific_name_to_paths:
         continue
-    
+
     print('Processing query {} of {} ({})'.format(i_row,len(df),query))
     paths = retrieve_sample_image.download_images(query=query,
                                              output_directory=image_base,
@@ -404,40 +404,40 @@ scientific_name_to_preferred_images = {}
 
 # s = list(scientific_name_to_paths.keys())[0]
 for s in list(df.scientific_name):
-    
+
     if not isinstance(s,str):
         continue
-    
+
     query = s.replace(' ','+')
-    
+
     if query in remapped_queries:
         query = remapped_queries[query]
-    
+
     query_folder = os.path.join(image_base,query)
     assert os.path.isdir(query_folder)
     image_files = os.listdir(query_folder)
-    image_fullpaths = [os.path.join(query_folder,fn) for fn in image_files]    
+    image_fullpaths = [os.path.join(query_folder,fn) for fn in image_files]
     sizes = [os.path.getsize(p) for p in image_fullpaths]
     path_to_size = {}
     for i_fp,fp in enumerate(image_fullpaths):
         path_to_size[fp] = sizes[i_fp]
     paths_by_size = [x for _, x in sorted(zip(sizes, image_fullpaths),reverse=True)]
-    
+
     # Be suspicious of duplicate sizes
     b_duplicate_sizes = [False] * len(paths_by_size)
-    
+
     for i_path,p in enumerate(paths_by_size):
         if i_path == len(paths_by_size) - 1:
             continue
         if path_to_size[p] == path_to_size[paths_by_size[i_path+1]]:
             b_duplicate_sizes[i_path] = True
-    
+
     paths_by_size_non_dup = [i for (i, v) in zip(paths_by_size, b_duplicate_sizes) if not v]
-    
+
     preferred_paths = paths_by_size_non_dup[:max_images_per_query]
     scientific_name_to_preferred_images[s] = preferred_paths
 
-# ...for each scientific name    
+# ...for each scientific name
 
 
 #%% Delete unused images
@@ -445,7 +445,7 @@ for s in list(df.scientific_name):
 used_images = []
 for images in scientific_name_to_preferred_images.values():
     used_images.extend(images)
-    
+
 print('Using a total of {} images'.format(len(used_images)))
 used_images_set = set(used_images)
 
@@ -461,18 +461,18 @@ print('{} of {} files unused (diff {})'.format(len(unused_images),len(all_images
                                                len(all_images) - len(unused_images)))
 
 for fn in tqdm(unused_images):
-    os.remove(fn)    
+    os.remove(fn)
 
 
 #%% Produce HTML preview
 
 with open(html_output_file, 'w', encoding='utf-8') as f:
-    
+
     f.write('<html><head></head><body>\n')
 
     names = scientific_name_to_preferred_images.keys()
     names = sorted(names)
-    
+
     f.write('<p class="speciesinfo_p" style="font-weight:bold;font-size:130%">'
             'dataset_name: <b><u>category</u></b> mapped to taxonomy_level scientific_name (taxonomic_common_name) (manual_common_name)</p>\n'
             '</p>')
@@ -481,10 +481,10 @@ with open(html_output_file, 'w', encoding='utf-8') as f:
     for i_row, row in tqdm(df.iterrows(), total=len(df)):
 
         s = row['scientific_name']
-        
+
         taxonomy_string = row['taxonomy_string']
         if isinstance(taxonomy_string,str):
-            taxonomic_match = eval(taxonomy_string)        
+            taxonomic_match = eval(taxonomy_string)
             matched_entity = taxonomic_match[0]
             assert len(matched_entity) == 4
             common_names = matched_entity[3]
@@ -499,7 +499,7 @@ with open(html_output_file, 'w', encoding='utf-8') as f:
 
         if isinstance(row.scientific_name,str):
             output_string = '{}: <b><u>{}</u></b> mapped to {} {} ({}) ({})</p>\n'.format(
-                row.dataset_name, row.query, 
+                row.dataset_name, row.query,
                 row.taxonomy_level, row.scientific_name, common_name_string,
                 row.common_name)
             f.write(output_string)

megadetector/taxonomy_mapping/retrieve_sample_image.py

@@ -17,21 +17,21 @@ import os
 
 output_folder = os.path.expanduser('~/tmp/image-download-test')
 os.makedirs(output_folder,exist_ok=True)
-                                   
+
 method = 'simple_image_download' # 'google_images_download'
 
 if method == 'simple_image_download':
-    
+
     from megadetector.taxonomy_mapping import simple_image_download
     google_image_downloader = simple_image_download.Downloader()
     google_image_downloader.directory = output_folder
- 
+
 elif method == 'google_images_download':
-    
+
     from google_images_download import google_images_download
 
 else:
-    
+
     raise ValueError('Unrecognized method {}'.format(method))
 
 
@@ -39,33 +39,33 @@ else:
 
 def download_images(query,output_directory,limit=100,verbose=False):
 
-    query = query.replace(' ','+')        
-    
+    query = query.replace(' ','+')
+
     if method == 'simple_image_download':
-        
+
         google_image_downloader.directory = output_directory
         paths = google_image_downloader.download(query, limit=limit,
           verbose=verbose, cache=False, download_cache=False)
         return paths
-        
+
     elif method == 'google_images_download':
-        
-        response = google_images_download.googleimagesdownload()    
+
+        response = google_images_download.googleimagesdownload()
         arguments = {'keywords':query,'limit':limit,'print_urls':verbose,
                      'image-directory':output_directory}
         response.download(arguments)
         return None
 
     else:
-        
+
         raise ValueError('Unrecognized method {}'.format(method))
-        
+
 
 #%% Test driver
 
 if False:
-    
+
     #%%
-    
+
     paths = download_images(query='redunca',output_directory=output_folder,
-                    limit=20,verbose=True) 
+                    limit=20,verbose=True)

megadetector/taxonomy_mapping/simple_image_download.py

@@ -49,7 +49,7 @@ def generate_urls(search):
     """
     Generate Google search URLs for all tokens in the list [search]
     """
-    
+
     return [(BASE_URL+quote(word)+GOOGLE_PICTURE_ID) for word in search]
 
 
@@ -60,7 +60,7 @@ def check_webpage(url):
         if 'html' not in str(request.content):
             checked_url = request
     except Exception as err:
-        print(err)        
+        print(err)
     return checked_url
 
 
@@ -68,7 +68,7 @@ def scan_webpage(webpage, extensions, timer):
     """
     Scan for pictures to download based on keywords
     """
-    
+
     global SCANNER_COUNTER
     scanner = webpage.find
     found = False
@@ -143,7 +143,7 @@ class Downloader:
         urls_ = generate_urls(search)
         timer = timer if timer else 1000
         # max_progressbar = count * (list(range(limit+1))[-1]+1)
-        
+
         # bar = progressbar.ProgressBar(maxval=max_progressbar,
         #                               widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()]).start()
         i = 0
@@ -172,7 +172,7 @@ class Downloader:
             print('==='*15 + ' < ' + 'NO PICTURES FOUND' + ' > ' + '==='*15)
         return cache_out
 
-    def download(self, keywords=None, limit=1, verbose=False, cache=True, download_cache=False, 
+    def download(self, keywords=None, limit=1, verbose=False, cache=True, download_cache=False,
                  timer=None):
         if not download_cache:
             content = self.search_urls(keywords, limit, verbose, cache, timer)
@@ -180,16 +180,16 @@ class Downloader:
             content = self._cached_urls
             if not content:
                 print('Downloader has not URLs saved in Memory yet, run Downloader.search_urls to find pics first')
-        paths = []                
+        paths = []
         for name, (path, url) in content.items():
             fullpath = os.path.join(path, name)
             paths.append(fullpath)
             with open(fullpath, 'wb') as file:
                 file.write(url.content)
             if verbose:
-                print(f'File Name={name}, Downloaded from {url.url}')            
+                print(f'File Name={name}, Downloaded from {url.url}')
         return paths
-    
+
     def _create_directories(self, name):
         dir_path = os.path.join(self._directory, name)
         try:

megadetector/taxonomy_mapping/species_lookup.py

@@ -114,7 +114,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     if (not force_init) and (inat_taxonomy is not None):
         print('Skipping taxonomy re-init')
         return
-    
+
     if (not force_init) and (os.path.isfile(serialized_structures_file)):
 
         print(f'De-serializing taxonomy data from {serialized_structures_file}')
@@ -135,7 +135,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
         gbif_vernacular_to_taxon_id,\
         gbif_taxon_id_to_scientific,\
         gbif_scientific_to_taxon_id = structures_to_serialize
-        
+
         return
 
 
@@ -146,7 +146,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     for taxonomy_name, zip_url in taxonomy_urls.items():
 
         need_to_download = False
-        
+
         if force_init:
             need_to_download = True
 
@@ -267,7 +267,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     # Build iNat dictionaries
 
     print('Building lookup dictionaries for iNat taxonomy')
-    
+
     for i_row, row in tqdm(inat_taxonomy.iterrows(), total=len(inat_taxonomy)):
 
         taxon_id = row['taxonID']
@@ -286,7 +286,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     # Build GBIF dictionaries
 
     print('Building lookup dictionaries for GBIF taxonomy')
-    
+
     for i_row, row in tqdm(gbif_taxonomy.iterrows(), total=len(gbif_taxonomy)):
 
         taxon_id = row['taxonID']
@@ -596,21 +596,21 @@ class TaxonomicMatch:
 
 
 hyphenated_terms = ['crowned', 'backed', 'throated', 'tailed', 'headed', 'cheeked',
-                    'ruffed', 'browed', 'eating', 'striped', 'shanked', 
+                    'ruffed', 'browed', 'eating', 'striped', 'shanked',
                     'fronted', 'bellied', 'spotted', 'eared', 'collared', 'breasted',
                     'necked']
 
 def get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat', retry=True) -> TaxonomicMatch:
     """
-    Wrapper for _get_preferred_taxonomic_match, but expressing a variety of heuristics 
+    Wrapper for _get_preferred_taxonomic_match, but expressing a variety of heuristics
     and preferences that are specific to our scenario.
-    
+
     Args:
         query (str): The common or scientific name we want to look up
         taxonomy_preference (str, optional): 'inat' or 'gbif'
-        retry (bool, optional): if the initial lookup fails, should we try heuristic 
+        retry (bool, optional): if the initial lookup fails, should we try heuristic
             substitutions, e.g. replacing "_" with " ", or "spp" with "species"?
-    
+
     Returns:
         TaxonomicMatch: the best taxonomic match, or None
     """
@@ -618,31 +618,31 @@ def get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat', retr
     m,query = _get_preferred_taxonomic_match(query=query,taxonomy_preference=taxonomy_preference)
     if (len(m.scientific_name) > 0) or (not retry):
         return m
-    
+
     for s in hyphenated_terms:
         query = query.replace(' ' + s,'-' + s)
     m,query = _get_preferred_taxonomic_match(query=query,taxonomy_preference=taxonomy_preference)
     return m
-    
-    
+
+
 def validate_and_convert(data):
     """
     Recursively validates that all elements in the nested structure are only
     tuples, lists, ints, or np.int64, and converts np.int64 to int.
-    
+
     Args:
         data: The nested structure to validate and convert
-        
+
     Returns:
         The validated and converted structure
-        
+
     Raises:
         TypeError: If an invalid type is encountered
     """
-    
-    if isinstance(data, np.int64):        
+
+    if isinstance(data, np.int64):
         return int(data)
-    elif isinstance(data, int) or isinstance(data, str): 
+    elif isinstance(data, int) or isinstance(data, str):
         return data
     elif isinstance(data, (list, tuple)):
         # Process lists and tuples recursively
@@ -654,17 +654,17 @@ def validate_and_convert(data):
 
 # ...def validate_and_convert(...)
 
-    
+
 def _get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat') -> TaxonomicMatch:
-    
+
     query = query.lower().strip().replace('_', ' ')
     query = query.replace('unidentified','')
     query = query.replace('unknown','')
     if query.endswith(' sp'):
         query = query.replace(' sp','')
     if query.endswith(' group'):
-        query = query.replace(' group','')    
-    
+        query = query.replace(' group','')
+
     query = query.strip()
 
     # query = 'person'
@@ -686,17 +686,17 @@ def _get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat') ->
 
     n_inat_matches = len(inat_matches)
     n_gbif_matches = len(gbif_matches)
-    
+
     selected_matches = None
-    
+
     assert taxonomy_preference in ['gbif','inat'],\
         'Unrecognized taxonomy preference: {}'.format(taxonomy_preference)
-        
+
     if n_inat_matches > 0 and taxonomy_preference == 'inat':
         selected_matches = 'inat'
     elif n_gbif_matches > 0:
         selected_matches = 'gbif'
-        
+
     if selected_matches == 'inat':
 
         i_match = 0
@@ -802,7 +802,7 @@ def _get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat') ->
     # Convert np.int64's to ints
     if match is not None:
         match = validate_and_convert(match)
-        
+
     taxonomy_string = str(match)
 
     return TaxonomicMatch(scientific_name, common_name, taxonomic_level, source,
@@ -828,15 +828,15 @@ if False:
     # print(matches)
 
     print_taxonomy_matches(matches,verbose=True)
-    
+
     print('\n\n')
-    
+
     # Print the taxonomy in the taxonomy spreadsheet format
     assert matches[1]['source'] == 'inat'
     t = str(matches[1]['taxonomy'])
     print(t)
     import clipboard; clipboard.copy(t)
-    
+
 
     #%% Directly access the taxonomy tables
 
@@ -848,12 +848,12 @@ if False:
 
 #%% Command-line driver
 
-def main():
+def main(): # noqa
 
     # Read command line inputs (absolute path)
     parser = argparse.ArgumentParser()
     parser.add_argument('input_file')
-        
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()