megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/taxonomy_mapping/species_lookup.py

@@ -114,7 +114,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     if (not force_init) and (inat_taxonomy is not None):
         print('Skipping taxonomy re-init')
         return
-    
+
     if (not force_init) and (os.path.isfile(serialized_structures_file)):
 
         print(f'De-serializing taxonomy data from {serialized_structures_file}')
@@ -135,7 +135,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
         gbif_vernacular_to_taxon_id,\
         gbif_taxon_id_to_scientific,\
         gbif_scientific_to_taxon_id = structures_to_serialize
-        
+
         return
 
 
@@ -146,7 +146,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     for taxonomy_name, zip_url in taxonomy_urls.items():
 
         need_to_download = False
-        
+
         if force_init:
             need_to_download = True
 
@@ -267,7 +267,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     # Build iNat dictionaries
 
     print('Building lookup dictionaries for iNat taxonomy')
-    
+
     for i_row, row in tqdm(inat_taxonomy.iterrows(), total=len(inat_taxonomy)):
 
         taxon_id = row['taxonID']
@@ -286,7 +286,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     # Build GBIF dictionaries
 
     print('Building lookup dictionaries for GBIF taxonomy')
-    
+
     for i_row, row in tqdm(gbif_taxonomy.iterrows(), total=len(gbif_taxonomy)):
 
         taxon_id = row['taxonID']
@@ -596,21 +596,21 @@ class TaxonomicMatch:
 
 
 hyphenated_terms = ['crowned', 'backed', 'throated', 'tailed', 'headed', 'cheeked',
-                    'ruffed', 'browed', 'eating', 'striped', 'shanked', 
+                    'ruffed', 'browed', 'eating', 'striped', 'shanked',
                     'fronted', 'bellied', 'spotted', 'eared', 'collared', 'breasted',
                     'necked']
 
 def get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat', retry=True) -> TaxonomicMatch:
     """
-    Wrapper for _get_preferred_taxonomic_match, but expressing a variety of heuristics 
+    Wrapper for _get_preferred_taxonomic_match, but expressing a variety of heuristics
     and preferences that are specific to our scenario.
-    
+
     Args:
         query (str): The common or scientific name we want to look up
         taxonomy_preference (str, optional): 'inat' or 'gbif'
-        retry (bool, optional): if the initial lookup fails, should we try heuristic 
+        retry (bool, optional): if the initial lookup fails, should we try heuristic
             substitutions, e.g. replacing "_" with " ", or "spp" with "species"?
-    
+
     Returns:
         TaxonomicMatch: the best taxonomic match, or None
     """
@@ -618,31 +618,31 @@ def get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat', retr
     m,query = _get_preferred_taxonomic_match(query=query,taxonomy_preference=taxonomy_preference)
     if (len(m.scientific_name) > 0) or (not retry):
         return m
-    
+
     for s in hyphenated_terms:
         query = query.replace(' ' + s,'-' + s)
     m,query = _get_preferred_taxonomic_match(query=query,taxonomy_preference=taxonomy_preference)
     return m
-    
-    
+
+
 def validate_and_convert(data):
     """
     Recursively validates that all elements in the nested structure are only
     tuples, lists, ints, or np.int64, and converts np.int64 to int.
-    
+
     Args:
         data: The nested structure to validate and convert
-        
+
     Returns:
         The validated and converted structure
-        
+
     Raises:
         TypeError: If an invalid type is encountered
     """
-    
-    if isinstance(data, np.int64):        
+
+    if isinstance(data, np.int64):
         return int(data)
-    elif isinstance(data, int) or isinstance(data, str): 
+    elif isinstance(data, int) or isinstance(data, str):
         return data
     elif isinstance(data, (list, tuple)):
         # Process lists and tuples recursively
@@ -654,17 +654,17 @@ def validate_and_convert(data):
 
 # ...def validate_and_convert(...)
 
-    
+
 def _get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat') -> TaxonomicMatch:
-    
+
     query = query.lower().strip().replace('_', ' ')
     query = query.replace('unidentified','')
     query = query.replace('unknown','')
     if query.endswith(' sp'):
         query = query.replace(' sp','')
     if query.endswith(' group'):
-        query = query.replace(' group','')    
-    
+        query = query.replace(' group','')
+
     query = query.strip()
 
     # query = 'person'
@@ -686,17 +686,17 @@ def _get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat') ->
 
     n_inat_matches = len(inat_matches)
     n_gbif_matches = len(gbif_matches)
-    
+
     selected_matches = None
-    
+
     assert taxonomy_preference in ['gbif','inat'],\
         'Unrecognized taxonomy preference: {}'.format(taxonomy_preference)
-        
+
     if n_inat_matches > 0 and taxonomy_preference == 'inat':
         selected_matches = 'inat'
     elif n_gbif_matches > 0:
         selected_matches = 'gbif'
-        
+
     if selected_matches == 'inat':
 
         i_match = 0
@@ -802,7 +802,7 @@ def _get_preferred_taxonomic_match(query: str, taxonomy_preference = 'inat') ->
     # Convert np.int64's to ints
     if match is not None:
         match = validate_and_convert(match)
-        
+
     taxonomy_string = str(match)
 
     return TaxonomicMatch(scientific_name, common_name, taxonomic_level, source,
@@ -828,15 +828,15 @@ if False:
     # print(matches)
 
     print_taxonomy_matches(matches,verbose=True)
-    
+
     print('\n\n')
-    
+
     # Print the taxonomy in the taxonomy spreadsheet format
     assert matches[1]['source'] == 'inat'
     t = str(matches[1]['taxonomy'])
     print(t)
     import clipboard; clipboard.copy(t)
-    
+
 
     #%% Directly access the taxonomy tables
 
@@ -848,12 +848,12 @@ if False:
 
 #%% Command-line driver
 
-def main():
+def main(): # noqa
 
     # Read command line inputs (absolute path)
     parser = argparse.ArgumentParser()
     parser.add_argument('input_file')
-        
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()

megadetector/taxonomy_mapping/taxonomy_csv_checker.py

@@ -36,7 +36,7 @@ def check_taxonomy_csv(csv_path: str) -> None:
     """
     See module docstring.
     """
-    
+
     taxonomy_df = pd.read_csv(csv_path)
 
     graph = nx.DiGraph()
@@ -46,12 +46,12 @@ def check_taxonomy_csv(csv_path: str) -> None:
     num_scientific_name_errors = 0
 
     for i_row, row in taxonomy_df.iterrows():
-        
+
         ds = row['dataset_name']
         ds_label = row['query']
         scientific_name = row['scientific_name']
         level = row['taxonomy_level']
-        
+
         # This used to represent the source of the mapping: iNat, gbif, or manual.  We've
         # stopped tracking this, so this is now vestigial.
         id_source = 0 # row['source']
@@ -95,8 +95,8 @@ def check_taxonomy_csv(csv_path: str) -> None:
                     num_scientific_name_errors += 1
 
             taxon_child = node
-    
-    # ...for each row in the taxonomy file        
+
+    # ...for each row in the taxonomy file
 
     assert nx.is_directed_acyclic_graph(graph)
 
@@ -124,36 +124,36 @@ def check_taxonomy_csv(csv_path: str) -> None:
         print(f'At least one node has unresolved ambiguous parents: {e}')
 
     print('Processed {} rows from {}'.format(len(taxonomy_df),csv_path))
-    
+
     print('num taxon level errors:', num_taxon_level_errors)
     print('num scientific name errors:', num_scientific_name_errors)
 
 
 #%% Command-line driver
-    
+
 if __name__ == '__main__':
-    
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'taxonomy_csv_path',
         help='path to taxonomy CSV file')
-    
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()
-    
+
     args = parser.parse_args()
 
     check_taxonomy_csv(args.taxonomy_csv_path)
 
 
 #%% Interactive driver
-    
+
 if False:
-    
+
     #%%
-    
+
     import os
     csv_path = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
     check_taxonomy_csv(csv_path)
-    
+

megadetector/taxonomy_mapping/taxonomy_graph.py

@@ -1,4 +1,4 @@
-"""
+r"""
 
 taxonomy_graph.py
 
@@ -69,7 +69,7 @@ class TaxonNode:
     By default, we support multiple parents for each TaxonNode. See discussion
     in module docstring above.
     """
-    
+
     # class variables
     single_parent_only: ClassVar[bool] = False
 
@@ -82,7 +82,7 @@ class TaxonNode:
 
     def __init__(self, level: str, name: str,
                  graph: Optional[nx.DiGraph] = None):
-        
+
         self.level = level
         self.name = name
         self.graph = graph
@@ -131,7 +131,7 @@ class TaxonNode:
         Args:
             parent: TaxonNode, must be higher in the taxonomical hierarchy
         """
-        
+
         assert self.graph is not None
         parents = self.parents
         if TaxonNode.single_parent_only and len(parents) > 0:
@@ -150,7 +150,7 @@ class TaxonNode:
         Args:
             child: TaxonNode, must be lower in the taxonomical hierarchy
         """
-        
+
         assert self.graph is not None
         self.graph.add_edge(self, child)
 
@@ -160,7 +160,7 @@ class TaxonNode:
             ds: str, name of dataset
             ds_label: str, name of label used by that dataset
         """
-        
+
         self.dataset_labels.add((ds, ds_label))
 
     def get_dataset_labels(self,
@@ -176,7 +176,7 @@ class TaxonNode:
 
         Returns: set of (ds, ds_label) tuples
         """
-        
+
         result = self.dataset_labels
         if include_datasets is not None:
             result = set(tup for tup in result if tup[0] in include_datasets)
@@ -199,7 +199,7 @@ class TaxonNode:
 
         Returns: TaxonNode, the LCA if it exists, or None if no LCA exists
         """
-        
+
         paths = []
         for node in nodes:
             # get path to root
@@ -242,7 +242,7 @@ def build_taxonomy_graph(taxonomy_df: pd.DataFrame
             TaxonNode node in the tree that contains the label,
             keys are all lowercase
     """
-    
+
     graph = nx.DiGraph()
     taxon_to_node = {}  # maps (taxon_level, taxon_name) to a TaxonNode
     label_to_node = {}  # maps (dataset_name, dataset_label) to a TaxonNode
@@ -303,12 +303,12 @@ def dag_to_tree(graph: nx.DiGraph,
     component separately.
 
     Args:
-        graph: nx.DiGraph, DAG representation of taxonomy hieararchy
+        graph: nx.DiGraph, DAG representation of taxonomy hierarchy
         taxon_to_node: dict, maps (taxon_level, taxon_name) to a TaxonNode
 
     Returns: nx.DiGraph, a tree-structured graph
     """
-    
+
     tree = nx.DiGraph()
     for node in graph.nodes:
         tree.add_node(node)

megadetector/taxonomy_mapping/validate_lila_category_mappings.py

@@ -17,9 +17,9 @@ from megadetector.data_management.lila.lila_common import read_lila_taxonomy_map
 #%% Prevent execution during infrastructural imports
 
 if False:
-        
+
     #%% Constants
-    
+
     lila_local_base = os.path.expanduser('~/lila')
 
     metadata_dir = os.path.join(lila_local_base,'metadata')
@@ -30,7 +30,7 @@ if False:
     lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
 
     assert os.path.isfile(lila_dataset_to_categories_file)
-        
+
 
     #%% Load category and taxonomy files
 
@@ -48,36 +48,36 @@ if False:
 
     # i_row = 1; row = taxonomy_df.iloc[i_row]; row
     for i_row,row in taxonomy_df.iterrows():
-        
+
         ds_query = row['dataset_name'] + ':' + row['query']
         ds_query = ds_query.lower()
-        
+
         if not isinstance(row['scientific_name'],str):
             unmapped_queries.add(ds_query)
             ds_query_to_scientific_name[ds_query] = 'unmapped'
             continue
-            
+
         ds_query_to_scientific_name[ds_query] = row['scientific_name']
-        
-        
+
+
     #%% For each dataset, make sure we can map every category to the taxonomy
 
     # dataset_name = list(lila_dataset_to_categories.keys())[0]
     for _dataset_name in lila_dataset_to_categories.keys():
-        
+
         if '_bbox' in _dataset_name:
             dataset_name = _dataset_name.replace('_bbox','')
         else:
             dataset_name = _dataset_name
-        
+
         categories = lila_dataset_to_categories[dataset_name]
-        
+
         # c = categories[0]
         for c in categories:
             ds_query = dataset_name + ':' + c['name']
             ds_query = ds_query.lower()
-            
+
             if ds_query not in ds_query_to_scientific_name:
-                print('Could not find mapping for {}'.format(ds_query))            
+                print('Could not find mapping for {}'.format(ds_query))
             else:
                 scientific_name = ds_query_to_scientific_name[ds_query]

megadetector/utils/azure_utils.py

@@ -12,8 +12,8 @@ Requires azure-storage-blob>=12.4.0
 
 import json
 
-from typing import Any, Iterable, List, Optional, Tuple, Union
-from azure.storage.blob import BlobPrefix, ContainerClient
+from typing import Any, Iterable, Optional, Union
+from azure.storage.blob import BlobPrefix, ContainerClient # type: ignore
 
 from megadetector.utils import path_utils
 from megadetector.utils import sas_blob_utils
@@ -26,20 +26,20 @@ def walk_container(container_client: ContainerClient,
                    prefix: str = '',
                    store_folders: bool = True,
                    store_blobs: bool = True,
-                   debug_max_items: int = -1) -> Tuple[List[str], List[str]]:
+                   debug_max_items: int = -1) -> tuple[list[str], list[str]]:
     """
     Recursively walk folders a Azure Blob Storage container.
 
     Based on:
     https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/storage/azure-storage-blob/samples/blob_samples_walk_blob_hierarchy.py
     """
-    
+
     depth = 1
 
     def walk_blob_hierarchy(prefix: str,
-                            folders: Optional[List[str]] = None,
-                            blobs: Optional[List[str]] = None
-                            ) -> Tuple[List[str], List[str]]:
+                            folders: Optional[list[str]] = None,
+                            blobs: Optional[list[str]] = None
+                            ) -> tuple[list[str], list[str]]:
         if folders is None:
             folders = []
         if blobs is None:
@@ -76,11 +76,11 @@ def walk_container(container_client: ContainerClient,
     return folders, blobs
 
 
-def list_top_level_blob_folders(container_client: ContainerClient) -> List[str]:
+def list_top_level_blob_folders(container_client: ContainerClient) -> list[str]:
     """
     List all top-level folders in a container.
     """
-    
+
     top_level_folders, _ = walk_container(
         container_client, max_depth=1, store_blobs=False)
     return top_level_folders
@@ -88,13 +88,13 @@ def list_top_level_blob_folders(container_client: ContainerClient) -> List[str]:
 
 def concatenate_json_lists(input_files: Iterable[str],
                            output_file: Optional[str] = None
-                           ) -> List[Any]:
+                           ) -> list[Any]:
     """
     Given a list of JSON files that contain lists (typically string
     filenames), concatenates the lists into a single list and optionally
     writes out this list to a new output JSON file.
     """
-    
+
     output_list = []
     for fn in input_files:
         with open(fn, 'r') as f:
@@ -116,12 +116,12 @@ def upload_file_to_blob(account_name: str,
     Uploads a local file to Azure Blob Storage and returns the uploaded
     blob URI with SAS token.
     """
-    
+
     container_uri = sas_blob_utils.build_azure_storage_uri(
         account=account_name, container=container_name, sas_token=sas_token)
     with open(local_path, 'rb') as data:
         return sas_blob_utils.upload_blob(
-            container_uri=container_uri, blob_name=blob_name, data=data, 
+            container_uri=container_uri, blob_name=blob_name, data=data,
             overwrite=overwrite)
 
 
@@ -131,11 +131,11 @@ def enumerate_blobs_to_file(
         container_name: str,
         sas_token: Optional[str] = None,
         blob_prefix: Optional[str] = None,
-        blob_suffix: Optional[Union[str, Tuple[str]]] = None,
+        blob_suffix: Optional[Union[str, tuple[str]]] = None,
         rsearch: Optional[str] = None,
         limit: Optional[int] = None,
         verbose: Optional[bool] = True
-        ) -> List[str]:
+        ) -> list[str]:
     """
     Enumerates blobs in a container, and writes the blob names to an output
     file.
@@ -143,7 +143,7 @@ def enumerate_blobs_to_file(
     Args:
         output_file: str, path to save list of files in container
             If ends in '.json', writes a JSON string. Otherwise, writes a
-            newline-delimited list. Can be None, in which case this is just a 
+            newline-delimited list. Can be None, in which case this is just a
             convenient wrapper for blob enumeration.
         account_name: str, Azure Storage account name
         container_name: str, Azure Blob Storage container name
@@ -155,24 +155,24 @@ def enumerate_blobs_to_file(
             be lowercased first before comparing with the suffix(es).
         rsearch: optional str, returned results will only contain blob names
             that match this regex. Can also be a list of regexes, in which case
-            blobs matching *any* of the regex's will be returned.            
+            blobs matching *any* of the regex's will be returned.
         limit: int, maximum # of blob names to list
             if None, then returns all blob names
 
     Returns: list of str, sorted blob names, of length limit or shorter.
     """
-    
+
     if sas_token is not None and len(sas_token) > 9 and sas_token[0] == '?':
         sas_token = sas_token[1:]
-        
+
     container_uri = sas_blob_utils.build_azure_storage_uri(
         account=account_name, container=container_name, sas_token=sas_token)
-    
+
     matched_blobs = sas_blob_utils.list_blobs_in_container(
         container_uri=container_uri, blob_prefix=blob_prefix,
         blob_suffix=blob_suffix, rsearch=rsearch, limit=limit, verbose=verbose)
-    
+
     if output_file is not None:
         path_utils.write_list_to_file(output_file, matched_blobs)
-        
+
     return matched_blobs