megadetector 5.0.28__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/taxonomy_mapping/taxonomy_csv_checker.py

@@ -36,7 +36,7 @@ def check_taxonomy_csv(csv_path: str) -> None:
     """
     See module docstring.
     """
-    
+
     taxonomy_df = pd.read_csv(csv_path)
 
     graph = nx.DiGraph()
@@ -46,12 +46,12 @@ def check_taxonomy_csv(csv_path: str) -> None:
     num_scientific_name_errors = 0
 
     for i_row, row in taxonomy_df.iterrows():
-        
+
         ds = row['dataset_name']
         ds_label = row['query']
         scientific_name = row['scientific_name']
         level = row['taxonomy_level']
-        
+
         # This used to represent the source of the mapping: iNat, gbif, or manual.  We've
         # stopped tracking this, so this is now vestigial.
         id_source = 0 # row['source']
@@ -95,8 +95,8 @@ def check_taxonomy_csv(csv_path: str) -> None:
                     num_scientific_name_errors += 1
 
             taxon_child = node
-    
-    # ...for each row in the taxonomy file        
+
+    # ...for each row in the taxonomy file
 
     assert nx.is_directed_acyclic_graph(graph)
 
@@ -124,36 +124,36 @@ def check_taxonomy_csv(csv_path: str) -> None:
         print(f'At least one node has unresolved ambiguous parents: {e}')
 
     print('Processed {} rows from {}'.format(len(taxonomy_df),csv_path))
-    
+
     print('num taxon level errors:', num_taxon_level_errors)
     print('num scientific name errors:', num_scientific_name_errors)
 
 
 #%% Command-line driver
-    
+
 if __name__ == '__main__':
-    
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'taxonomy_csv_path',
         help='path to taxonomy CSV file')
-    
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()
-    
+
     args = parser.parse_args()
 
     check_taxonomy_csv(args.taxonomy_csv_path)
 
 
 #%% Interactive driver
-    
+
 if False:
-    
+
     #%%
-    
+
     import os
     csv_path = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
     check_taxonomy_csv(csv_path)
-    
+

megadetector/taxonomy_mapping/taxonomy_graph.py

@@ -1,4 +1,4 @@
-"""
+r"""
 
 taxonomy_graph.py
 
@@ -69,7 +69,7 @@ class TaxonNode:
     By default, we support multiple parents for each TaxonNode. See discussion
     in module docstring above.
     """
-    
+
     # class variables
     single_parent_only: ClassVar[bool] = False
 
@@ -82,7 +82,7 @@ class TaxonNode:
 
     def __init__(self, level: str, name: str,
                  graph: Optional[nx.DiGraph] = None):
-        
+
         self.level = level
         self.name = name
         self.graph = graph
@@ -131,7 +131,7 @@ class TaxonNode:
         Args:
             parent: TaxonNode, must be higher in the taxonomical hierarchy
         """
-        
+
         assert self.graph is not None
         parents = self.parents
         if TaxonNode.single_parent_only and len(parents) > 0:
@@ -150,7 +150,7 @@ class TaxonNode:
         Args:
             child: TaxonNode, must be lower in the taxonomical hierarchy
         """
-        
+
         assert self.graph is not None
         self.graph.add_edge(self, child)
 
@@ -160,7 +160,7 @@ class TaxonNode:
             ds: str, name of dataset
             ds_label: str, name of label used by that dataset
         """
-        
+
         self.dataset_labels.add((ds, ds_label))
 
     def get_dataset_labels(self,
@@ -176,7 +176,7 @@ class TaxonNode:
 
         Returns: set of (ds, ds_label) tuples
         """
-        
+
         result = self.dataset_labels
         if include_datasets is not None:
             result = set(tup for tup in result if tup[0] in include_datasets)
@@ -199,7 +199,7 @@ class TaxonNode:
 
         Returns: TaxonNode, the LCA if it exists, or None if no LCA exists
         """
-        
+
         paths = []
         for node in nodes:
             # get path to root
@@ -242,7 +242,7 @@ def build_taxonomy_graph(taxonomy_df: pd.DataFrame
             TaxonNode node in the tree that contains the label,
             keys are all lowercase
     """
-    
+
     graph = nx.DiGraph()
     taxon_to_node = {}  # maps (taxon_level, taxon_name) to a TaxonNode
     label_to_node = {}  # maps (dataset_name, dataset_label) to a TaxonNode
@@ -308,7 +308,7 @@ def dag_to_tree(graph: nx.DiGraph,
 
     Returns: nx.DiGraph, a tree-structured graph
     """
-    
+
     tree = nx.DiGraph()
     for node in graph.nodes:
         tree.add_node(node)

megadetector/taxonomy_mapping/validate_lila_category_mappings.py

@@ -17,9 +17,9 @@ from megadetector.data_management.lila.lila_common import read_lila_taxonomy_map
 #%% Prevent execution during infrastructural imports
 
 if False:
-        
+
     #%% Constants
-    
+
     lila_local_base = os.path.expanduser('~/lila')
 
     metadata_dir = os.path.join(lila_local_base,'metadata')
@@ -30,7 +30,7 @@ if False:
     lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
 
     assert os.path.isfile(lila_dataset_to_categories_file)
-        
+
 
     #%% Load category and taxonomy files
 
@@ -48,36 +48,36 @@ if False:
 
     # i_row = 1; row = taxonomy_df.iloc[i_row]; row
     for i_row,row in taxonomy_df.iterrows():
-        
+
         ds_query = row['dataset_name'] + ':' + row['query']
         ds_query = ds_query.lower()
-        
+
         if not isinstance(row['scientific_name'],str):
             unmapped_queries.add(ds_query)
             ds_query_to_scientific_name[ds_query] = 'unmapped'
             continue
-            
+
         ds_query_to_scientific_name[ds_query] = row['scientific_name']
-        
-        
+
+
     #%% For each dataset, make sure we can map every category to the taxonomy
 
     # dataset_name = list(lila_dataset_to_categories.keys())[0]
     for _dataset_name in lila_dataset_to_categories.keys():
-        
+
         if '_bbox' in _dataset_name:
             dataset_name = _dataset_name.replace('_bbox','')
         else:
             dataset_name = _dataset_name
-        
+
         categories = lila_dataset_to_categories[dataset_name]
-        
+
         # c = categories[0]
         for c in categories:
             ds_query = dataset_name + ':' + c['name']
             ds_query = ds_query.lower()
-            
+
             if ds_query not in ds_query_to_scientific_name:
-                print('Could not find mapping for {}'.format(ds_query))            
+                print('Could not find mapping for {}'.format(ds_query))
             else:
                 scientific_name = ds_query_to_scientific_name[ds_query]

megadetector/utils/azure_utils.py

@@ -12,8 +12,8 @@ Requires azure-storage-blob>=12.4.0
 
 import json
 
-from typing import Any, Iterable, List, Optional, Tuple, Union
-from azure.storage.blob import BlobPrefix, ContainerClient
+from typing import Any, Iterable, Optional, Union
+from azure.storage.blob import BlobPrefix, ContainerClient # type: ignore
 
 from megadetector.utils import path_utils
 from megadetector.utils import sas_blob_utils
@@ -26,20 +26,20 @@ def walk_container(container_client: ContainerClient,
                    prefix: str = '',
                    store_folders: bool = True,
                    store_blobs: bool = True,
-                   debug_max_items: int = -1) -> Tuple[List[str], List[str]]:
+                   debug_max_items: int = -1) -> tuple[list[str], list[str]]:
     """
     Recursively walk folders a Azure Blob Storage container.
 
     Based on:
     https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/storage/azure-storage-blob/samples/blob_samples_walk_blob_hierarchy.py
     """
-    
+
     depth = 1
 
     def walk_blob_hierarchy(prefix: str,
-                            folders: Optional[List[str]] = None,
-                            blobs: Optional[List[str]] = None
-                            ) -> Tuple[List[str], List[str]]:
+                            folders: Optional[list[str]] = None,
+                            blobs: Optional[list[str]] = None
+                            ) -> tuple[list[str], list[str]]:
         if folders is None:
             folders = []
         if blobs is None:
@@ -76,11 +76,11 @@ def walk_container(container_client: ContainerClient,
     return folders, blobs
 
 
-def list_top_level_blob_folders(container_client: ContainerClient) -> List[str]:
+def list_top_level_blob_folders(container_client: ContainerClient) -> list[str]:
     """
     List all top-level folders in a container.
     """
-    
+
     top_level_folders, _ = walk_container(
         container_client, max_depth=1, store_blobs=False)
     return top_level_folders
@@ -88,13 +88,13 @@ def list_top_level_blob_folders(container_client: ContainerClient) -> List[str]:
 
 def concatenate_json_lists(input_files: Iterable[str],
                            output_file: Optional[str] = None
-                           ) -> List[Any]:
+                           ) -> list[Any]:
     """
     Given a list of JSON files that contain lists (typically string
     filenames), concatenates the lists into a single list and optionally
     writes out this list to a new output JSON file.
     """
-    
+
     output_list = []
     for fn in input_files:
         with open(fn, 'r') as f:
@@ -116,12 +116,12 @@ def upload_file_to_blob(account_name: str,
     Uploads a local file to Azure Blob Storage and returns the uploaded
     blob URI with SAS token.
     """
-    
+
     container_uri = sas_blob_utils.build_azure_storage_uri(
         account=account_name, container=container_name, sas_token=sas_token)
     with open(local_path, 'rb') as data:
         return sas_blob_utils.upload_blob(
-            container_uri=container_uri, blob_name=blob_name, data=data, 
+            container_uri=container_uri, blob_name=blob_name, data=data,
             overwrite=overwrite)
 
 
@@ -131,11 +131,11 @@ def enumerate_blobs_to_file(
         container_name: str,
         sas_token: Optional[str] = None,
         blob_prefix: Optional[str] = None,
-        blob_suffix: Optional[Union[str, Tuple[str]]] = None,
+        blob_suffix: Optional[Union[str, tuple[str]]] = None,
         rsearch: Optional[str] = None,
         limit: Optional[int] = None,
         verbose: Optional[bool] = True
-        ) -> List[str]:
+        ) -> list[str]:
     """
     Enumerates blobs in a container, and writes the blob names to an output
     file.
@@ -143,7 +143,7 @@ def enumerate_blobs_to_file(
     Args:
         output_file: str, path to save list of files in container
             If ends in '.json', writes a JSON string. Otherwise, writes a
-            newline-delimited list. Can be None, in which case this is just a 
+            newline-delimited list. Can be None, in which case this is just a
             convenient wrapper for blob enumeration.
         account_name: str, Azure Storage account name
         container_name: str, Azure Blob Storage container name
@@ -155,24 +155,24 @@ def enumerate_blobs_to_file(
             be lowercased first before comparing with the suffix(es).
         rsearch: optional str, returned results will only contain blob names
             that match this regex. Can also be a list of regexes, in which case
-            blobs matching *any* of the regex's will be returned.            
+            blobs matching *any* of the regex's will be returned.
         limit: int, maximum # of blob names to list
             if None, then returns all blob names
 
     Returns: list of str, sorted blob names, of length limit or shorter.
     """
-    
+
     if sas_token is not None and len(sas_token) > 9 and sas_token[0] == '?':
         sas_token = sas_token[1:]
-        
+
     container_uri = sas_blob_utils.build_azure_storage_uri(
         account=account_name, container=container_name, sas_token=sas_token)
-    
+
     matched_blobs = sas_blob_utils.list_blobs_in_container(
         container_uri=container_uri, blob_prefix=blob_prefix,
         blob_suffix=blob_suffix, rsearch=rsearch, limit=limit, verbose=verbose)
-    
+
     if output_file is not None:
         path_utils.write_list_to_file(output_file, matched_blobs)
-        
+
     return matched_blobs