megadetector 5.0.6__py3-none-any.whl → 5.0.8__py3-none-any.whl

taxonomy_mapping/species_lookup.py

@@ -36,14 +36,23 @@ taxonomy_urls = {
 }
 
 files_to_unzip = {
-    'GBIF': ['backbone/Taxon.tsv', 'backbone/VernacularName.tsv'],
+    # GBIF used to put everything in a "backbone" folder within the zipfile, but as of
+    # 12.2023, this is no longer the case.
+    # 'GBIF': ['backbone/Taxon.tsv', 'backbone/VernacularName.tsv'],
+    'GBIF': ['Taxon.tsv', 'VernacularName.tsv'],
     'iNaturalist': ['taxa.csv']
 }
 
 # As of 2020.05.12:
 #
 # GBIF: ~777MB zipped, ~1.6GB taxonomy
-# iNat: ~2.2GB zipped, ~51MB taxonomy
+# iNat: ~2.2GB zipped, ~51MB taxonomy (most of the zipfile is observations)
+
+# As of 2023.12.29:
+#
+# GBIF: ~948MB zipped, ~2.2GB taxonomy
+# iNat: ~6.7GB zipped, ~62MB taxonomy (most of the zipfile is observations)
+
 
 os.makedirs(taxonomy_download_dir, exist_ok=True)
 for taxonomy_name in taxonomy_urls:
@@ -99,15 +108,16 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
         gbif_taxon_id_to_scientific,\
         gbif_scientific_to_taxon_id
 
+
     ## Load serialized taxonomy info if we've already saved it
 
     if (not force_init) and (inat_taxonomy is not None):
         print('Skipping taxonomy re-init')
         return
     
-    if os.path.isfile(serialized_structures_file):
+    if (not force_init) and (os.path.isfile(serialized_structures_file)):
 
-        print(f'Reading taxonomy data from {serialized_structures_file}')
+        print(f'De-serializing taxonomy data from {serialized_structures_file}')
 
         with open(serialized_structures_file, 'rb') as f:
             structures_to_serialize = pickle.load(f)
@@ -125,6 +135,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
         gbif_vernacular_to_taxon_id,\
         gbif_taxon_id_to_scientific,\
         gbif_scientific_to_taxon_id = structures_to_serialize
+        
         return
 
 
@@ -135,6 +146,9 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     for taxonomy_name, zip_url in taxonomy_urls.items():
 
         need_to_download = False
+        
+        if force_init:
+            need_to_download = True
 
         # Don't download the zipfile if we've already unzipped what we need
         for fn in files_to_unzip[taxonomy_name]:
@@ -150,11 +164,11 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
         zipfile_path = os.path.join(
             taxonomy_download_dir, zip_url.split('/')[-1])
 
-        # Bypasses download if the file exists already
+        # Bypasses download if the file exists already (unless force_init is set)
         url_utils.download_url(
             zip_url, os.path.join(zipfile_path),
             progress_updater=url_utils.DownloadProgressBar(),
-            verbose=True)
+            verbose=True,force_download=force_init)
 
         # Unzip the files we need
         files_we_need = files_to_unzip[taxonomy_name]
@@ -166,7 +180,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
                 target_file = os.path.join(
                     taxonomy_download_dir, taxonomy_name, os.path.basename(fn))
 
-                if os.path.isfile(target_file):
+                if (not force_init) and (os.path.isfile(target_file)):
                     print(f'Bypassing unzip of {target_file}, file exists')
                 else:
                     os.makedirs(os.path.basename(target_file),exist_ok=True)
@@ -185,13 +199,16 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     # name file
 
     # Load iNat taxonomy
-    inat_taxonomy = pd.read_csv(os.path.join(taxonomy_download_dir, 'iNaturalist', 'taxa.csv'))
+    inat_taxonomy_file = os.path.join(taxonomy_download_dir, 'iNaturalist', 'taxa.csv')
+    print('Loading iNat taxonomy from {}'.format(inat_taxonomy_file))
+    inat_taxonomy = pd.read_csv(inat_taxonomy_file)
     inat_taxonomy['scientificName'] = inat_taxonomy['scientificName'].fillna('').str.strip()
     inat_taxonomy['vernacularName'] = inat_taxonomy['vernacularName'].fillna('').str.strip()
 
     # Load GBIF taxonomy
-    gbif_taxonomy = pd.read_csv(os.path.join(
-        taxonomy_download_dir, 'GBIF', 'Taxon.tsv'), sep='\t')
+    gbif_taxonomy_file = os.path.join(taxonomy_download_dir, 'GBIF', 'Taxon.tsv')
+    print('Loading GBIF taxonomy from {}'.format(gbif_taxonomy_file))
+    gbif_taxonomy = pd.read_csv(gbif_taxonomy_file, sep='\t')
     gbif_taxonomy['scientificName'] = gbif_taxonomy['scientificName'].fillna('').str.strip()
     gbif_taxonomy['canonicalName'] = gbif_taxonomy['canonicalName'].fillna('').str.strip()
 
@@ -249,7 +266,8 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
 
     # Build iNat dictionaries
 
-    # row = inat_taxonomy.iloc[0]
+    print('Building lookup dictionaries for iNat taxonomy')
+    
     for i_row, row in tqdm(inat_taxonomy.iterrows(), total=len(inat_taxonomy)):
 
         taxon_id = row['taxonID']
@@ -267,6 +285,8 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
 
     # Build GBIF dictionaries
 
+    print('Building lookup dictionaries for GBIF taxonomy')
+    
     for i_row, row in tqdm(gbif_taxonomy.iterrows(), total=len(gbif_taxonomy)):
 
         taxon_id = row['taxonID']
@@ -320,13 +340,13 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
         gbif_scientific_to_taxon_id
     ]
 
-    print('Serializing...', end='')
+    print('Serializing to {}...'.format(serialized_structures_file), end='')
     if not os.path.isfile(serialized_structures_file):
         with open(serialized_structures_file, 'wb') as p:
             pickle.dump(structures_to_serialize, p)
     print(' done')
 
-# ...def initialize_taxonomy_lookup()
+# ...def initialize_taxonomy_lookup(...)
 
 
 def get_scientific_name_from_row(r):

taxonomy_mapping/taxonomy_csv_checker.py

@@ -45,7 +45,7 @@ def check_taxonomy_csv(csv_path: str) -> None:
     num_taxon_level_errors = 0
     num_scientific_name_errors = 0
 
-    for i, row in taxonomy_df.iterrows():
+    for i_row, row in taxonomy_df.iterrows():
         
         ds = row['dataset_name']
         ds_label = row['query']
@@ -81,14 +81,14 @@ def check_taxonomy_csv(csv_path: str) -> None:
             node.add_id(id_source, int(taxon_id))  # np.int64 -> int
             if j == 0:
                 if level != taxon_level:
-                    print(f'row: {i}, {ds}, {ds_label}')
+                    print(f'row: {i_row}, {ds}, {ds_label}')
                     print(f'- taxonomy_level column: {level}, '
                           f'level from taxonomy_string: {taxon_level}')
                     print()
                     num_taxon_level_errors += 1
 
                 if scientific_name != taxon_name:
-                    print(f'row: {i}, {ds}, {ds_label}')
+                    print(f'row: {i_row}, {ds}, {ds_label}')
                     print(f'- scientific_name column: {scientific_name}, '
                           f'name from taxonomy_string: {taxon_name}')
                     print()
@@ -97,7 +97,7 @@ def check_taxonomy_csv(csv_path: str) -> None:
             taxon_child = node
     
     # ...for each row in the taxonomy file        
-            
+
     assert nx.is_directed_acyclic_graph(graph)
 
     for node in graph.nodes:
@@ -123,6 +123,8 @@ def check_taxonomy_csv(csv_path: str) -> None:
     except AssertionError as e:
         print(f'At least one node has unresolved ambiguous parents: {e}')
 
+    print('Processed {} rows from {}'.format(len(taxonomy_df),csv_path))
+    
     print('num taxon level errors:', num_taxon_level_errors)
     print('num scientific name errors:', num_scientific_name_errors)
 
@@ -154,4 +156,4 @@ if False:
     import os
     csv_path = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
     check_taxonomy_csv(csv_path)
-    
+    

md_visualization/visualize_megadb.py

@@ -1,183 +0,0 @@
-########
-#
-# visualize_megadb.py
-#
-# Create visual previews of images/sequences in MegaDB.
-#
-########
-
-#%% Imports
-
-import argparse
-import json
-import os
-import sys
-from random import shuffle
-from multiprocessing.pool import ThreadPool
-from functools import partial
-import io
-
-from tqdm import tqdm
-
-from data_management.megadb.megadb_utils import MegadbUtils
-from md_utils.write_html_image_list import write_html_image_list
-from md_visualization import visualization_utils as vis_utils
-
-
-#%% Support functions
-
-def render_image_info(rendering, args):
-    
-    storage_client = rendering['storage_client']
-    image_obj = io.BytesIO()
-
-    try:
-        storage_client.download_blob(rendering['blob_path']).readinto(image_obj)
-    except Exception as e:
-        print(f'Image not found in blob storage: {rendering["blob_path"]}')
-        print(e)
-        return
-
-    # resize is for displaying them more quickly
-    image = vis_utils.resize_image(
-        vis_utils.open_image(image_obj), args.output_image_width)
-    vis_utils.render_megadb_bounding_boxes(rendering['bbox'], image)
-
-    annotated_img_name = rendering['annotated_img_name']
-    annotated_img_path = os.path.join(
-        args.output_dir, 'rendered_images', annotated_img_name)
-    image.save(annotated_img_path)
-
-
-def visualize_sequences(datasets_table, sequences, args):
-    
-    num_images = 0
-
-    images_html = []
-    rendering_info = []
-
-    for seq in sequences:
-        if 'images' not in seq:
-            continue
-
-        # dataset and seq_id are required fields
-        dataset_name = seq['dataset']
-        seq_id = seq['seq_id']
-
-        # sort the images in the sequence
-
-        images_in_seq = sorted(seq['images'], key=lambda x: x['frame_num']) if len(seq['images']) > 1 else seq['images']
-
-        for im in images_in_seq:
-            if args.trim_to_images_bboxes_labeled and 'bbox' not in im:
-                continue
-
-            num_images += 1
-
-            blob_path = MegadbUtils.get_full_path(
-                datasets_table, dataset_name, im['file'])
-            frame_num = im.get('frame_num', -1)
-
-            # if no class label on the image, show class label on the sequence
-            im_class = im.get('class', None)
-            if im_class is None:
-                im_class = seq.get('class', [])
-
-            rendering = {}
-            rendering['storage_client'] = MegadbUtils.get_storage_client(
-                datasets_table, dataset_name)
-            rendering['blob_path'] = blob_path
-            rendering['bbox'] = im.get('bbox', [])
-
-            annotated_img_name = 'anno_' + blob_path.replace('/', args.pathsep_replacement).replace('\\', args.pathsep_replacement)
-            rendering['annotated_img_name'] = annotated_img_name
-
-            rendering_info.append(rendering)
-
-            images_html.append({
-                'filename': 'rendered_images/{}'.format(annotated_img_name),
-                'title': 'Seq ID: {}. Frame number: {}<br/> Image file: {}<br/> number of boxes: {}, image class labels: {}'.format(seq_id, frame_num, blob_path, len(rendering['bbox']), im_class),
-                'textStyle': 'font-family:verdana,arial,calibri;font-size:80%;text-align:left;margin-top:20;margin-bottom:5'
-            })
-
-        if num_images >= args.num_to_visualize:
-            print('num_images visualized is {}'.format(num_images))
-            break
-
-    # pool = ThreadPool()
-    render_image_info_partial = partial(render_image_info, args=args)
-    # print('len of rendering_info', len(rendering_info))
-    # tqdm(pool.imap_unordered(render_image_info_partial, rendering_info), total=len(rendering_info))
-
-    for rendering in tqdm(rendering_info):
-        render_image_info_partial(rendering)
-
-    print('Making HTML...')
-
-    html_path = os.path.join(args.output_dir, 'index.html')
-    # options = write_html_image_list()
-    # options['headerHtml']
-    write_html_image_list(
-        filename=html_path,
-        images=images_html
-    )
-
-
-#%% Command-line driver
-
-def main():
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'megadb_entries', type=str,
-        help='Path to a json list of MegaDB entries')
-    parser.add_argument(
-        'output_dir', action='store', type=str,
-        help='Output directory for html and rendered images')
-    parser.add_argument(
-        '--trim_to_images_bboxes_labeled', action='store_true',
-        help='Only include images that have been sent for bbox labeling (but '
-             'may be actually empty). Turn this on if QAing annotations.')
-    parser.add_argument(
-        '--num_to_visualize', action='store', type=int, default=200,
-        help='Number of images to visualize (all comformant images in a '
-             'sequence are shown, so may be a few more than specified). '
-             'Sequences are shuffled. Default: 200. Use -1 to visualize all.')
-    parser.add_argument(
-        '--pathsep_replacement', action='store', type=str, default='~',
-        help='Replace path separators in relative filenames with another '
-             'character (default ~)')
-    parser.add_argument(
-        '-w', '--output_image_width', type=int, default=700,
-        help='an integer indicating the desired width in pixels of the output '
-             'annotated images. Use -1 to not resize.')
-
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-
-    args = parser.parse_args()
-
-    assert 'COSMOS_ENDPOINT' in os.environ and 'COSMOS_KEY' in os.environ
-
-    os.makedirs(args.output_dir, exist_ok=True)
-    os.makedirs(os.path.join(args.output_dir, 'rendered_images'))
-
-    print('Connecting to MegaDB to get the datasets table...')
-    megadb_utils = MegadbUtils()
-    datasets_table = megadb_utils.get_datasets_table()
-
-    print('Loading the MegaDB entries...')
-    with open(args.megadb_entries) as f:
-        sequences = json.load(f)
-    print('Total number of sequences: {}'.format(len(sequences)))
-
-    # print('Checking that the MegaDB entries conform to the schema...')
-    # sequences_schema_check.sequences_schema_check(sequences)
-
-    shuffle(sequences)
-    visualize_sequences(datasets_table, sequences, args)
-
-
-if __name__ == '__main__':
-    main()