megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py

@@ -2,7 +2,7 @@
 
 map_lila_taxonomy_to_wi_taxonomy.py
 
-Loads the LILA category mapping (in which taxonomy information comes from an 
+Loads the LILA category mapping (in which taxonomy information comes from an
 iNat taxonomy snapshot) and tries to map each class to the Wildlife Insights taxonomy.
 
 """
@@ -22,9 +22,9 @@ from megadetector.data_management.lila.lila_common import \
 #%% Prevent execution during infrastructural imports
 
 if False:
-    
+
     #%%
-    
+
     lila_local_base = os.path.expanduser('~/lila')
 
     metadata_dir = os.path.join(lila_local_base, 'metadata')
@@ -65,9 +65,9 @@ if False:
 
     #%% Cache WI taxonomy lookups
 
-    def is_empty_wi_item(v):
+    def _is_empty_wi_item(v):
         if isinstance(v, str):
-            return len(v) == 0        
+            return len(v) == 0
         elif v is None:
             return True
         else:
@@ -75,7 +75,7 @@ if False:
             return True
 
 
-    def taxonomy_items_equal(a, b):
+    def _taxonomy_items_equal(a, b):
         if isinstance(a, str) and (not isinstance(b, str)):
             return False
         if isinstance(b, str) and (not isinstance(a, str)):
@@ -121,7 +121,7 @@ if False:
             v = taxon[k]
             if isinstance(v,str):
                 taxon[k] = v.strip()
-                
+
         if taxon['commonNameEnglish'] in ignore_taxa:
             continue
 
@@ -152,37 +152,37 @@ if False:
                 continue
 
         # Do we have a species name?
-        if not is_empty_wi_item(taxon['species']):
+        if not _is_empty_wi_item(taxon['species']):
 
             # If 'species' is populated, 'genus' should always be populated; one item currently breaks
             # this rule.
-            assert not is_empty_wi_item(taxon['genus'])
-            
+            assert not _is_empty_wi_item(taxon['genus'])
+
             taxon_name = (taxon['genus'].strip() + ' ' +
                         taxon['species'].strip()).strip().lower()
-            assert not is_empty_wi_item(taxon['class']) and \
-                not is_empty_wi_item(taxon['order']) and \
-                not is_empty_wi_item(taxon['family'])
+            assert not _is_empty_wi_item(taxon['class']) and \
+                not _is_empty_wi_item(taxon['order']) and \
+                not _is_empty_wi_item(taxon['family'])
 
-        elif not is_empty_wi_item(taxon['genus']):
+        elif not _is_empty_wi_item(taxon['genus']):
 
-            assert not is_empty_wi_item(taxon['class']) and \
-                not is_empty_wi_item(taxon['order']) and \
-                not is_empty_wi_item(taxon['family'])
+            assert not _is_empty_wi_item(taxon['class']) and \
+                not _is_empty_wi_item(taxon['order']) and \
+                not _is_empty_wi_item(taxon['family'])
             taxon_name = taxon['genus'].strip().lower()
 
-        elif not is_empty_wi_item(taxon['family']):
+        elif not _is_empty_wi_item(taxon['family']):
 
-            assert not is_empty_wi_item(taxon['class']) and \
-                not is_empty_wi_item(taxon['order'])
+            assert not _is_empty_wi_item(taxon['class']) and \
+                not _is_empty_wi_item(taxon['order'])
             taxon_name = taxon['family'].strip().lower()
 
-        elif not is_empty_wi_item(taxon['order']):
+        elif not _is_empty_wi_item(taxon['order']):
 
-            assert not is_empty_wi_item(taxon['class'])
+            assert not _is_empty_wi_item(taxon['class'])
             taxon_name = taxon['order'].strip().lower()
 
-        elif not is_empty_wi_item(taxon['class']):
+        elif not _is_empty_wi_item(taxon['class']):
 
             taxon_name = taxon['class'].strip().lower()
 
@@ -204,8 +204,8 @@ if False:
                             level,previous_taxon[level],
                             previous_taxon['taxon_name'],
                             level,taxon[level])
-                        assert taxonomy_items_equal(previous_taxon[level], taxon[level]), error_string
-                    
+                        assert _taxonomy_items_equal(previous_taxon[level], taxon[level]), error_string
+
         taxon['taxon_name'] = taxon_name
         if taxon_name == 'homo sapiens':
             human_taxa.append(taxon)
@@ -234,7 +234,7 @@ if False:
         pass
 
         #%% Manual review of redundant taxa
-        
+
         s = taxon_names_with_multiple_entries[15]
         taxa = wi_taxon_name_to_taxa[s]
         for t in taxa:
@@ -249,19 +249,19 @@ if False:
     taxon_name_to_preferred_taxon_id = {}
 
     # "helmeted guineafowl" vs "domestic guineafowl"
-    taxon_name_to_preferred_taxon_id['numida meleagris'] = '83133617-8358-4910-82ee-4c23e40ba3dc' # 2005826 
+    taxon_name_to_preferred_taxon_id['numida meleagris'] = '83133617-8358-4910-82ee-4c23e40ba3dc' # 2005826
 
     # "domestic turkey" vs. "wild turkey"
-    taxon_name_to_preferred_taxon_id['meleagris gallopavo'] = 'c10547c3-1748-48bf-a451-8066c820f22f' # 2021598 
+    taxon_name_to_preferred_taxon_id['meleagris gallopavo'] = 'c10547c3-1748-48bf-a451-8066c820f22f' # 2021598
 
     # multiple sensible human entries
-    taxon_name_to_preferred_taxon_id['homo sapiens'] = '990ae9dd-7a59-4344-afcb-1b7b21368000' # 2002045 
+    taxon_name_to_preferred_taxon_id['homo sapiens'] = '990ae9dd-7a59-4344-afcb-1b7b21368000' # 2002045
 
     # "domestic dog" and "dog-on-leash"
-    taxon_name_to_preferred_taxon_id['canis familiaris'] = '3d80f1d6-b1df-4966-9ff4-94053c7a902a' # 2021548 
+    taxon_name_to_preferred_taxon_id['canis familiaris'] = '3d80f1d6-b1df-4966-9ff4-94053c7a902a' # 2021548
 
     # "small mammal" vs. "mammal"
-    taxon_name_to_preferred_taxon_id['mammalia'] = 'f2d233e3-80e3-433d-9687-e29ecc7a467a' # 2021108 
+    taxon_name_to_preferred_taxon_id['mammalia'] = 'f2d233e3-80e3-433d-9687-e29ecc7a467a' # 2021108
 
     # "Hispaniolan Mango" vs. NaN
     taxon_name_to_preferred_taxon_id['anthracothorax dominicus'] = 'f94e6d97-59cf-4d38-a05a-a75efdd2863b'
@@ -276,19 +276,19 @@ if False:
     taxon_name_to_preferred_taxon_id['stagonopleura bella'] = '7fec8e7e-fd3b-4d7f-99fd-3ade6f3bbaa5' # 2021939
 
     # "yellow wagtail" vs. "yellow crowned-wagtail"
-    taxon_name_to_preferred_taxon_id['motacilla flava'] = 'ac6669bc-9f9e-4473-b609-b9082f9bf50c' # 2016194 
+    taxon_name_to_preferred_taxon_id['motacilla flava'] = 'ac6669bc-9f9e-4473-b609-b9082f9bf50c' # 2016194
 
     # "dremomys species" vs. "dremomys genus"
     taxon_name_to_preferred_taxon_id['dremomys'] = '1507d153-af11-46f1-bfb8-77918d035ab3' # 2019370
 
     # "elk" vs. "domestic elk"
-    taxon_name_to_preferred_taxon_id['cervus canadensis'] = 'c5ce946f-8f0d-4379-992b-cc0982381f5e' 
+    taxon_name_to_preferred_taxon_id['cervus canadensis'] = 'c5ce946f-8f0d-4379-992b-cc0982381f5e'
 
     # "American bison" vs. "domestic bison"
-    taxon_name_to_preferred_taxon_id['bison bison'] = '539ebd55-081b-429a-9ae6-5a6a0f6999d4' # 2021593 
+    taxon_name_to_preferred_taxon_id['bison bison'] = '539ebd55-081b-429a-9ae6-5a6a0f6999d4' # 2021593
 
     # "woodrat or rat or mouse species" vs. "mouse species"
-    taxon_name_to_preferred_taxon_id['muridae'] = 'e7503287-468c-45af-a1bd-a17821bb62f2' # 2021642 
+    taxon_name_to_preferred_taxon_id['muridae'] = 'e7503287-468c-45af-a1bd-a17821bb62f2' # 2021642
 
     # both "southern sand frog"
     taxon_name_to_preferred_taxon_id['tomopterna adiastola'] = 'a5dc63cb-41be-4090-84a7-b944b16dcee4' # 2021834
@@ -296,18 +296,18 @@ if False:
     # sericornis species vs. scrubwren species
     taxon_name_to_preferred_taxon_id['sericornis'] = 'ad82c0ac-df48-4028-bf71-d2b2f4bc4129' # 2021776
 
-        
+
     # taxon_name = list(taxon_name_to_preferred_taxon_id.keys())[0]
     for taxon_name in taxon_name_to_preferred_taxon_id.keys():
-        
+
         candidate_taxa = wi_taxon_name_to_taxa[taxon_name]
-        
+
         # If we've gotten this far, we should be choosing from multiple taxa.
         #
         # This will become untrue if any of these are resolved later, at which point we should
         # remove them from taxon_name_to_preferred_id
         assert len(candidate_taxa) > 1, 'Only one taxon available for {}'.format(taxon_name)
-        
+
         # Choose the preferred taxa
         selected_taxa = [t for t in candidate_taxa if t[id_column] == \
                         taxon_name_to_preferred_taxon_id[taxon_name]]
@@ -365,7 +365,7 @@ if False:
         query = None
 
         lila_dataset_category = lila_taxon['dataset_name'] + ':' + lila_taxon['query']
-        
+
         # Go from kingdom --> species, choosing the lowest-level description as the query
         for level in lila_taxonomy_levels:
             if isinstance(lila_taxon[level], str):
@@ -455,37 +455,37 @@ if False:
     #%% Map LILA datasets to WI taxa, and count the number of each taxon available in each dataset
 
     with open(wi_mapping_table_file,'w') as f:
-        
+
         f.write('lila_dataset_name,lila_category_name,wi_guid,wi_taxon_name,wi_common,count\n')
-        
+
         # dataset_name = list(lila_dataset_to_categories.keys())[0]
         for dataset_name in lila_dataset_to_categories.keys():
-            
+
             if '_bbox' in dataset_name:
                 continue
-            
+
             dataset_categories = lila_dataset_to_categories[dataset_name]
-            
+
             # dataset_category = dataset_categories[0]
             for category in dataset_categories:
-                
+
                 lila_dataset_category = dataset_name + ':' + category['name'].strip().lower()
                 if '#' in lila_dataset_category:
                     continue
                 assert lila_dataset_category in lila_dataset_category_to_lila_taxon
                 assert lila_dataset_category in lila_dataset_category_to_wi_taxon
                 assert 'count' in category
-        
+
                 wi_taxon = lila_dataset_category_to_wi_taxon[lila_dataset_category]
-                
-                # Write out the dataset name, category name, WI GUID, WI scientific name, WI common name, 
+
+                # Write out the dataset name, category name, WI GUID, WI scientific name, WI common name,
                 # and count
                 s = f"{dataset_name},{category['name']},{wi_taxon['uniqueIdentifier']},"+\
                     f"{wi_taxon['taxon_name']},{wi_taxon['commonNameEnglish']},{category['count']}\n"
                 f.write(s)
-                
+
             # ...for each category in this dataset
-                
-        # ...for each dataset    
+
+        # ...for each dataset
 
     # ...with open()

megadetector/taxonomy_mapping/map_new_lila_datasets.py

@@ -15,10 +15,10 @@ import json
 # Created by get_lila_category_list.py
 input_lila_category_list_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
 
-output_file = os.path.expanduser('~/lila/lila_additions_2025.03.24.csv')
+output_file = os.path.expanduser('~/lila/lila_additions_2025.06.23.csv')
 
 datasets_to_map = [
-    'UNSW Predators'
+    'Nkhotakota Camera Traps'
     ]
 
 
@@ -48,7 +48,7 @@ for s in datasets_to_map:
     assert s in lila_datasets
 
 
-#%% Find all categories    
+#%% Find all categories
 
 category_mappings = []
 
@@ -75,17 +75,17 @@ allow_non_preferred_matches = True
 
 # mapping_string = category_mappings[1]; print(mapping_string)
 for mapping_string in category_mappings:
-    
+
     tokens = mapping_string.split(':')
-    assert len(tokens) == 2    
+    assert len(tokens) == 2
 
     dataset_name = tokens[0]
     query = tokens[1]
 
     taxonomic_match = get_preferred_taxonomic_match(query,taxonomy_preference=taxonomy_preference)
-    
+
     if (taxonomic_match.source == taxonomy_preference) or allow_non_preferred_matches:
-    
+
         output_row = {
             'dataset_name': dataset_name,
             'query': query,
@@ -95,9 +95,9 @@ for mapping_string in category_mappings:
             'common_name': taxonomic_match.common_name,
             'taxonomy_string': taxonomic_match.taxonomy_string
         }
-    
+
     else:
-        
+
         output_row = {
             'dataset_name': dataset_name,
             'query': query,
@@ -107,10 +107,10 @@ for mapping_string in category_mappings:
             'common_name': '',
             'taxonomy_string': ''
         }
-        
+
     output_rows.append(output_row)
-    
-# ...for each mapping    
+
+# ...for each mapping
 
 
 #%% Write output rows
@@ -133,19 +133,19 @@ output_df.to_csv(output_file, index=None, header=True)
 if False:
 
     #%% You probably want to open the .csv file first
-    
+
     from megadetector.utils.path_utils import open_file
     open_file(output_file)
 
-    
+
     #%%
-    
-    q = 'dasyurus maculatus'
-    
+
+    q = 'animalia'
+
     taxonomy_preference = 'inat'
     m = get_preferred_taxonomic_match(q,taxonomy_preference)
     # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
-    
+
     if (m is None) or (len(m.taxonomy_string) == 0):
         print('No match')
     else:
@@ -154,5 +154,4 @@ if False:
             # raise ValueError('')
         print(m.source)
         print(m.taxonomy_string)
-        # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
         import clipboard; clipboard.copy(m.taxonomy_string)

megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py

@@ -2,7 +2,7 @@
 
 prepare_lila_taxonomy_release.py
 
-Given the private intermediate taxonomy mapping (produced by map_new_lila_datasets.py), 
+Given the private intermediate taxonomy mapping (produced by map_new_lila_datasets.py),
 prepare the public (release) taxonomy mapping file.
 
 """
@@ -17,9 +17,9 @@ import pandas as pd
 #%% Prevent execution during infrastructural imports
 
 if False:
-    
+
     #%% Filenames
-    
+
     lila_taxonomy_file = 'c:/git/agentmorrisprivate/lila-taxonomy/lila-taxonomy-mapping.csv'
     release_taxonomy_file = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
     # import clipboard; clipboard.copy(release_taxonomy_file)
@@ -42,7 +42,7 @@ if False:
 
     # dataset_name = datasets_to_map[0]
     for dataset_name in lila_dataset_to_categories.keys():
-        
+
         ds_categories = lila_dataset_to_categories[dataset_name]
         for category in ds_categories:
             category_name = category['name'].lower()
@@ -52,6 +52,8 @@ if False:
 
     df['used'] = False
 
+    n_dropped = 0
+
     # i_row = 0; row = df.iloc[i_row]; row
     for i_row,row in df.iterrows():
         ds_name = row['dataset_name']
@@ -60,8 +62,11 @@ if False:
         if mapping_name in used_category_mappings:
             df.loc[i_row,'used'] = True
         else:
+            n_dropped += 1
             print('Dropping unused mapping {}'.format(mapping_name))
 
+    print('Dropping {} of {} mappings'.format(n_dropped,len(df)))
+
     df = df[df.used]
     df = df.drop('used',axis=1)
 
@@ -71,66 +76,82 @@ if False:
     assert not os.path.isfile(release_taxonomy_file), \
         'File {} exists, delete it manually before proceeding'.format(release_taxonomy_file)
 
-    known_levels = ['stateofmatter', #noqa
-                        'kingdom',
-                        'phylum','subphylum',
-                        'superclass','class','subclass','infraclass',
-                        'superorder','order','parvorder','suborder','infraorder',
-                        'zoosection',
-                        'superfamily','family','subfamily','tribe',
-                        'genus',
-                        'species','subspecies','variety']
-
     levels_to_include = ['kingdom',
-                        'phylum','subphylum',
-                        'superclass','class','subclass','infraclass',
-                        'superorder','order','suborder','infraorder',
-                        'superfamily','family','subfamily','tribe',
-                        'genus',
-                        'species','subspecies','variety']
-
-    levels_to_exclude = ['stateofmatter','zoosection','parvorder','complex','epifamily']
+                         'phylum',
+                         'subphylum',
+                         'superclass',
+                         'class',
+                         'subclass',
+                         'infraclass',
+                         'superorder',
+                         'order',
+                         'suborder',
+                         'infraorder',
+                         'superfamily',
+                         'family',
+                         'subfamily',
+                         'tribe',
+                         'genus',
+                         'subgenus',
+                         'species',
+                         'subspecies',
+                         'variety']
+
+    levels_to_exclude = ['stateofmatter',
+                         'zoosection',
+                         'parvorder',
+                         'complex',
+                         'epifamily']
+
+    for x in [levels_to_include,levels_to_exclude]:
+        assert len(x) == len(set(x))
 
     for s in levels_to_exclude:
         assert s not in levels_to_include
-        
+
+    known_levels = levels_to_include + levels_to_exclude
+
     levels_used = set()
 
     # i_row = 0; row = df.iloc[i_row]; row
     for i_row,row in df.iterrows():
-        
+
         if not isinstance(row['scientific_name'],str):
             assert not isinstance(row['taxonomy_string'],str)
             continue
-        
+
+        # This is a list of length-4 tuples that each look like:
+        #
+        # (41789, 'species', 'taxidea taxus', ['american badger'])
         taxonomic_match = eval(row['taxonomy_string'])
-        
+
         # match_at_level = taxonomic_match[0]
         for match_at_level in taxonomic_match:
             assert len(match_at_level) == 4
+            # E.g. "species"
             levels_used.add(match_at_level[1])
-            
+
     levels_used = [s for s in levels_used if isinstance(s,str)]
 
     for s in levels_used:
-        assert s in levels_to_exclude or s in levels_to_include, 'Unrecognized level {}'.format(s)
+        assert s in known_levels, 'Unrecognized level {}'.format(s)
 
     for s in levels_to_include:
         assert s in levels_used
-        
+
     for s in levels_to_include:
         df[s] = ''
-        
+
     # i_row = 0; row = df.iloc[i_row]; row
     for i_row,row in df.iterrows():
-        
+
         if not isinstance(row['scientific_name'],str):
             assert not isinstance(row['taxonomy_string'],str)
             continue
-        
+
         # E.g.: (43117, 'genus', 'lepus', ['hares and jackrabbits']
         taxonomic_match = eval(row['taxonomy_string'])
-        
+
         for match_at_level in taxonomic_match:
             level = match_at_level[1]
             if level in levels_to_include: