megadetector 5.0.8__py3-none-any.whl → 5.0.9__py3-none-any.whl

taxonomy_mapping/map_new_lila_datasets.py

@@ -1,154 +1,154 @@
-########
-#
-# map_new_lila_datasets.py
-#
-# Given a subset of LILA datasets, find all the categories, and start the taxonomy
-# mapping process.
-#
-########
-
-#%% Constants and imports
-
-import os
-import json
-
-# Created by get_lila_category_list.py
-input_lila_category_list_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
-
-output_file = os.path.expanduser('~/lila/lila_additions_2023.12.29.csv')
-
-datasets_to_map = [
-    'Trail Camera Images of New Zealand Animals'
-    ]
-
-
-#%% Initialize taxonomic lookup
-
-from taxonomy_mapping.species_lookup import (
-    initialize_taxonomy_lookup,
-    get_preferred_taxonomic_match)
-
-# from taxonomy_mapping.species_lookup import (
-#    get_taxonomic_info, print_taxonomy_matche)
-
-initialize_taxonomy_lookup(force_init=False)
-
-
-#%% Read the list of datasets
-
-with open(input_lila_category_list_file,'r') as f:
-    input_lila_categories = json.load(f)
-
-lila_datasets = set()
-
-for dataset_name in input_lila_categories.keys():
-    # The script that generates this dictionary creates a separate entry for bounding box
-    # metadata files, but those don't represent new dataset names
-    lila_datasets.add(dataset_name.replace('_bbox',''))
-    
-for s in datasets_to_map:
-    assert s in lila_datasets
-    
-    
-#%% Find all categories    
-
-category_mappings = []
-
-# dataset_name = datasets_to_map[0]
-for dataset_name in datasets_to_map:
-    
-    ds_categories = input_lila_categories[dataset_name]
-    for category in ds_categories:
-        category_name = category['name']
-        assert ':' not in category_name
-        mapping_name = dataset_name + ':' + category_name
-        category_mappings.append(mapping_name)
-        
-print('Need to create {} mappings'.format(len(category_mappings)))
-
-
-#%% Match every query against our taxonomies
-
-output_rows = []
-
-taxonomy_preference = 'inat'
-
-allow_non_preferred_matches = True
-
-# mapping_string = category_mappings[1]; print(mapping_string)
-for mapping_string in category_mappings:
-    
-    tokens = mapping_string.split(':')
-    assert len(tokens) == 2    
-
-    dataset_name = tokens[0]
-    query = tokens[1]
-
-    taxonomic_match = get_preferred_taxonomic_match(query,taxonomy_preference=taxonomy_preference)
-    
-    if (taxonomic_match.source == taxonomy_preference) or allow_non_preferred_matches:
-    
-        output_row = {
-            'dataset_name': dataset_name,
-            'query': query,
-            'source': taxonomic_match.source,
-            'taxonomy_level': taxonomic_match.taxonomic_level,
-            'scientific_name': taxonomic_match.scientific_name,
-            'common_name': taxonomic_match.common_name,
-            'taxonomy_string': taxonomic_match.taxonomy_string
-        }
-    
-    else:
-        
-        output_row = {
-            'dataset_name': dataset_name,
-            'query': query,
-            'source': '',
-            'taxonomy_level': '',
-            'scientific_name': '',
-            'common_name': '',
-            'taxonomy_string': ''
-        }
-        
-    output_rows.append(output_row)
-    
-# ...for each mapping    
-
-
-#%% Write output rows
-
-import os
-import pandas as pd
-
-assert not os.path.isfile(output_file), 'Delete the output file before re-generating'
-
-output_df = pd.DataFrame(data=output_rows, columns=[
-    'dataset_name', 'query', 'source', 'taxonomy_level',
-    'scientific_name', 'common_name', 'taxonomy_string'])
-output_df.to_csv(output_file, index=None, header=True)
-
-
-#%% Manual lookup
-
-if False:
-    
-    #%%
-    
-    # q = 'white-throated monkey'
-    # q = 'cingulata'
-    # q = 'notamacropus'
-    q = 'porzana'
-    taxonomy_preference = 'inat'
-    m = get_preferred_taxonomic_match(q,taxonomy_preference)
-    # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
-    
-    if m is None:
-        print('No match')
-    else:
-        if m.source != taxonomy_preference:
-            print('\n*** non-preferred match ***\n')
-            # raise ValueError('')
-        print(m.source)
-        print(m.taxonomy_string)
-        # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
-        import clipboard; clipboard.copy(m.taxonomy_string)
+"""
+
+map_new_lila_datasets.py
+
+Given a subset of LILA datasets, find all the categories, and start the taxonomy
+mapping process.
+
+"""
+
+#%% Constants and imports
+
+import os
+import json
+
+# Created by get_lila_category_list.py
+input_lila_category_list_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
+
+output_file = os.path.expanduser('~/lila/lila_additions_2023.12.29.csv')
+
+datasets_to_map = [
+    'Trail Camera Images of New Zealand Animals'
+    ]
+
+
+#%% Initialize taxonomic lookup
+
+from taxonomy_mapping.species_lookup import (
+    initialize_taxonomy_lookup,
+    get_preferred_taxonomic_match)
+
+# from taxonomy_mapping.species_lookup import (
+#    get_taxonomic_info, print_taxonomy_matche)
+
+initialize_taxonomy_lookup(force_init=False)
+
+
+#%% Read the list of datasets
+
+with open(input_lila_category_list_file,'r') as f:
+    input_lila_categories = json.load(f)
+
+lila_datasets = set()
+
+for dataset_name in input_lila_categories.keys():
+    # The script that generates this dictionary creates a separate entry for bounding box
+    # metadata files, but those don't represent new dataset names
+    lila_datasets.add(dataset_name.replace('_bbox',''))
+    
+for s in datasets_to_map:
+    assert s in lila_datasets
+    
+    
+#%% Find all categories    
+
+category_mappings = []
+
+# dataset_name = datasets_to_map[0]
+for dataset_name in datasets_to_map:
+    
+    ds_categories = input_lila_categories[dataset_name]
+    for category in ds_categories:
+        category_name = category['name']
+        assert ':' not in category_name
+        mapping_name = dataset_name + ':' + category_name
+        category_mappings.append(mapping_name)
+        
+print('Need to create {} mappings'.format(len(category_mappings)))
+
+
+#%% Match every query against our taxonomies
+
+output_rows = []
+
+taxonomy_preference = 'inat'
+
+allow_non_preferred_matches = True
+
+# mapping_string = category_mappings[1]; print(mapping_string)
+for mapping_string in category_mappings:
+    
+    tokens = mapping_string.split(':')
+    assert len(tokens) == 2    
+
+    dataset_name = tokens[0]
+    query = tokens[1]
+
+    taxonomic_match = get_preferred_taxonomic_match(query,taxonomy_preference=taxonomy_preference)
+    
+    if (taxonomic_match.source == taxonomy_preference) or allow_non_preferred_matches:
+    
+        output_row = {
+            'dataset_name': dataset_name,
+            'query': query,
+            'source': taxonomic_match.source,
+            'taxonomy_level': taxonomic_match.taxonomic_level,
+            'scientific_name': taxonomic_match.scientific_name,
+            'common_name': taxonomic_match.common_name,
+            'taxonomy_string': taxonomic_match.taxonomy_string
+        }
+    
+    else:
+        
+        output_row = {
+            'dataset_name': dataset_name,
+            'query': query,
+            'source': '',
+            'taxonomy_level': '',
+            'scientific_name': '',
+            'common_name': '',
+            'taxonomy_string': ''
+        }
+        
+    output_rows.append(output_row)
+    
+# ...for each mapping    
+
+
+#%% Write output rows
+
+import os
+import pandas as pd
+
+assert not os.path.isfile(output_file), 'Delete the output file before re-generating'
+
+output_df = pd.DataFrame(data=output_rows, columns=[
+    'dataset_name', 'query', 'source', 'taxonomy_level',
+    'scientific_name', 'common_name', 'taxonomy_string'])
+output_df.to_csv(output_file, index=None, header=True)
+
+
+#%% Manual lookup
+
+if False:
+    
+    #%%
+    
+    # q = 'white-throated monkey'
+    # q = 'cingulata'
+    # q = 'notamacropus'
+    q = 'porzana'
+    taxonomy_preference = 'inat'
+    m = get_preferred_taxonomic_match(q,taxonomy_preference)
+    # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
+    
+    if m is None:
+        print('No match')
+    else:
+        if m.source != taxonomy_preference:
+            print('\n*** non-preferred match ***\n')
+            # raise ValueError('')
+        print(m.source)
+        print(m.taxonomy_string)
+        # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
+        import clipboard; clipboard.copy(m.taxonomy_string)

taxonomy_mapping/prepare_lila_taxonomy_release.py

@@ -1,134 +1,142 @@
-########
-#
-# prepare_lila_taxonomy_release.py
-#
-# Given the private intermediate taxonomy mapping (produced by map_new_lila_datasets.py), 
-# prepare the public (release) taxonomy mapping file.
-#
-########
-
-#%% Imports and constants
-
-import os
-import json
-import pandas as pd
-
-lila_taxonomy_file = 'c:/git/agentmorrisprivate/lila-taxonomy/lila-taxonomy-mapping.csv'
-release_taxonomy_file = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
-# import clipboard; clipboard.copy(release_taxonomy_file)
-
-# Created by get_lila_category_list.py... contains counts for each category
-lila_dataset_to_categories_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
-
-assert os.path.isfile(lila_dataset_to_categories_file)
-assert os.path.isfile(lila_taxonomy_file)
-
-
-#%% Find out which categories are actually used
-
-df = pd.read_csv(lila_taxonomy_file)
-
-with open(lila_dataset_to_categories_file,'r') as f:
-    lila_dataset_to_categories = json.load(f)
-
-used_category_mappings = []
-
-# dataset_name = datasets_to_map[0]
-for dataset_name in lila_dataset_to_categories.keys():
-    
-    ds_categories = lila_dataset_to_categories[dataset_name]
-    for category in ds_categories:
-        category_name = category['name'].lower()
-        assert ':' not in category_name
-        mapping_name = dataset_name + ':' + category_name
-        used_category_mappings.append(mapping_name)
-
-df['used'] = False
-
-# i_row = 0; row = df.iloc[i_row]; row
-for i_row,row in df.iterrows():
-    ds_name = row['dataset_name']
-    query = row['query']
-    mapping_name = ds_name + ':' + query
-    if mapping_name in used_category_mappings:
-        df.loc[i_row,'used'] = True
-    else:
-        print('Dropping unused mapping {}'.format(mapping_name))
-
-df = df[df.used]
-df = df.drop('used',axis=1)
-
-
-#%% Generate the final output file
-
-assert not os.path.isfile(release_taxonomy_file)
-
-known_levels = ['stateofmatter',
-                     'kingdom',
-                     'phylum','subphylum',
-                     'superclass','class','subclass','infraclass',
-                     'superorder','order','parvorder','suborder','infraorder',
-                     'zoosection',
-                     'superfamily','family','subfamily','tribe',
-                     'genus',
-                     'species','subspecies','variety']
-
-levels_to_include = ['kingdom',
-                     'phylum','subphylum',
-                     'superclass','class','subclass','infraclass',
-                     'superorder','order','suborder','infraorder',
-                     'superfamily','family','subfamily','tribe',
-                     'genus',
-                     'species','subspecies','variety']
-
-levels_to_exclude = ['stateofmatter','zoosection','parvorder']
-
-for s in levels_to_exclude:
-    assert s not in levels_to_include
-    
-levels_used = set()
-
-# i_row = 0; row = df.iloc[i_row]; row
-for i_row,row in df.iterrows():
-    
-    if not isinstance(row['scientific_name'],str):
-        assert not isinstance(row['taxonomy_string'],str)
-        continue
-    
-    taxonomic_match = eval(row['taxonomy_string'])
-    
-    # match_at_level = taxonomic_match[0]
-    for match_at_level in taxonomic_match:
-        assert len(match_at_level) == 4
-        levels_used.add(match_at_level[1])
-        
-levels_used = [s for s in levels_used if isinstance(s,str)]
-
-for s in levels_used:
-    assert s in levels_to_exclude or s in levels_to_include, 'Unrecognized level {}'.format(s)
-
-for s in levels_to_include:
-    assert s in levels_used
-    
-for s in levels_to_include:
-    df[s] = ''
-    
-# i_row = 0; row = df.iloc[i_row]; row
-for i_row,row in df.iterrows():
-    
-    if not isinstance(row['scientific_name'],str):
-        assert not isinstance(row['taxonomy_string'],str)
-        continue
-    
-    # E.g.: (43117, 'genus', 'lepus', ['hares and jackrabbits']
-    taxonomic_match = eval(row['taxonomy_string'])
-    
-    for match_at_level in taxonomic_match:
-        level = match_at_level[1]
-        if level in levels_to_include:
-            df.loc[i_row,level] = match_at_level[2]
-
-df = df.drop('source',axis=1)
-df.to_csv(release_taxonomy_file,header=True,index=False)
-
-print('Wrote final output to {}'.format(release_taxonomy_file))
+"""
+
+prepare_lila_taxonomy_release.py
+
+Given the private intermediate taxonomy mapping (produced by map_new_lila_datasets.py), 
+prepare the public (release) taxonomy mapping file.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+import pandas as pd
+
+
+#%% Prevent execution during infrastructural imports
+
+if False:
+    
+    #%% Filenames
+    
+    lila_taxonomy_file = 'c:/git/agentmorrisprivate/lila-taxonomy/lila-taxonomy-mapping.csv'
+    release_taxonomy_file = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
+    # import clipboard; clipboard.copy(release_taxonomy_file)
+
+    # Created by get_lila_category_list.py... contains counts for each category
+    lila_dataset_to_categories_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
+
+    assert os.path.isfile(lila_dataset_to_categories_file)
+    assert os.path.isfile(lila_taxonomy_file)
+
+
+    #%% Find out which categories are actually used
+
+    df = pd.read_csv(lila_taxonomy_file)
+
+    with open(lila_dataset_to_categories_file,'r') as f:
+        lila_dataset_to_categories = json.load(f)
+
+    used_category_mappings = []
+
+    # dataset_name = datasets_to_map[0]
+    for dataset_name in lila_dataset_to_categories.keys():
+        
+        ds_categories = lila_dataset_to_categories[dataset_name]
+        for category in ds_categories:
+            category_name = category['name'].lower()
+            assert ':' not in category_name
+            mapping_name = dataset_name + ':' + category_name
+            used_category_mappings.append(mapping_name)
+
+    df['used'] = False
+
+    # i_row = 0; row = df.iloc[i_row]; row
+    for i_row,row in df.iterrows():
+        ds_name = row['dataset_name']
+        query = row['query']
+        mapping_name = ds_name + ':' + query
+        if mapping_name in used_category_mappings:
+            df.loc[i_row,'used'] = True
+        else:
+            print('Dropping unused mapping {}'.format(mapping_name))
+
+    df = df[df.used]
+    df = df.drop('used',axis=1)
+
+
+    #%% Generate the final output file
+
+    assert not os.path.isfile(release_taxonomy_file)
+
+    known_levels = ['stateofmatter', #noqa
+                        'kingdom',
+                        'phylum','subphylum',
+                        'superclass','class','subclass','infraclass',
+                        'superorder','order','parvorder','suborder','infraorder',
+                        'zoosection',
+                        'superfamily','family','subfamily','tribe',
+                        'genus',
+                        'species','subspecies','variety']
+
+    levels_to_include = ['kingdom',
+                        'phylum','subphylum',
+                        'superclass','class','subclass','infraclass',
+                        'superorder','order','suborder','infraorder',
+                        'superfamily','family','subfamily','tribe',
+                        'genus',
+                        'species','subspecies','variety']
+
+    levels_to_exclude = ['stateofmatter','zoosection','parvorder']
+
+    for s in levels_to_exclude:
+        assert s not in levels_to_include
+        
+    levels_used = set()
+
+    # i_row = 0; row = df.iloc[i_row]; row
+    for i_row,row in df.iterrows():
+        
+        if not isinstance(row['scientific_name'],str):
+            assert not isinstance(row['taxonomy_string'],str)
+            continue
+        
+        taxonomic_match = eval(row['taxonomy_string'])
+        
+        # match_at_level = taxonomic_match[0]
+        for match_at_level in taxonomic_match:
+            assert len(match_at_level) == 4
+            levels_used.add(match_at_level[1])
+            
+    levels_used = [s for s in levels_used if isinstance(s,str)]
+
+    for s in levels_used:
+        assert s in levels_to_exclude or s in levels_to_include, 'Unrecognized level {}'.format(s)
+
+    for s in levels_to_include:
+        assert s in levels_used
+        
+    for s in levels_to_include:
+        df[s] = ''
+        
+    # i_row = 0; row = df.iloc[i_row]; row
+    for i_row,row in df.iterrows():
+        
+        if not isinstance(row['scientific_name'],str):
+            assert not isinstance(row['taxonomy_string'],str)
+            continue
+        
+        # E.g.: (43117, 'genus', 'lepus', ['hares and jackrabbits']
+        taxonomic_match = eval(row['taxonomy_string'])
+        
+        for match_at_level in taxonomic_match:
+            level = match_at_level[1]
+            if level in levels_to_include:
+                df.loc[i_row,level] = match_at_level[2]
+
+    df = df.drop('source',axis=1)
+    df.to_csv(release_taxonomy_file,header=True,index=False)
+
+    print('Wrote final output to {}'.format(release_taxonomy_file))
+