megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

taxonomy_mapping/taxonomy_csv_checker.py

@@ -1,159 +0,0 @@
-"""
-
-taxonomy_csv_checker.py
-
-Checks the taxonomy CSV file to make sure that for each row:
-
-1) The 'taxonomy_level' column matches the lowest-level taxon level in the
-    'taxonomy_string' column.
-
-2) The 'scientific_name' column matches the scientific name from the
-    lowest-level taxon level in the 'taxonomy_string' column.
-
-Prints out any mismatches.
-
-Also prints out nodes that have 2 ambiguous parents. See "CASE 2" from the
-module docstring of taxonomy_graph.py.
-
-"""
-
-#%% Imports
-
-import sys
-import argparse
-
-import networkx as nx
-import pandas as pd
-
-from typing import Optional
-
-from taxonomy_mapping.taxonomy_graph import TaxonNode, dag_to_tree
-
-
-#%% Taxonomy checking
-
-def check_taxonomy_csv(csv_path: str) -> None:
-    """
-    See module docstring.
-    """
-    
-    taxonomy_df = pd.read_csv(csv_path)
-
-    graph = nx.DiGraph()
-    taxon_to_node = {}  # maps (taxon_level, taxon_name) to a TaxonNode
-
-    num_taxon_level_errors = 0
-    num_scientific_name_errors = 0
-
-    for i_row, row in taxonomy_df.iterrows():
-        
-        ds = row['dataset_name']
-        ds_label = row['query']
-        scientific_name = row['scientific_name']
-        level = row['taxonomy_level']
-        
-        # This used to represent the source of the mapping: iNat, gbif, or manual.  We've
-        # stopped tracking this, so this is now vestigial.
-        id_source = 0 # row['source']
-
-        taxa_ancestry = row['taxonomy_string']
-        if pd.isna(taxa_ancestry):
-            # taxonomy CSV rows without 'taxonomy_string' entries are excluded
-            # from the taxonomy graph, but can be included in a classification
-            # label specification JSON via the 'dataset_labels' key
-            continue
-        else:
-            taxa_ancestry = eval(taxa_ancestry)  # pylint: disable=eval-used
-
-        taxon_child: Optional[TaxonNode] = None
-        for j, taxon in enumerate(taxa_ancestry):
-            taxon_id, taxon_level, taxon_name, _ = taxon
-
-            key = (taxon_level, taxon_name)
-            if key not in taxon_to_node:
-                taxon_to_node[key] = TaxonNode(level=taxon_level,
-                                               name=taxon_name, graph=graph)
-            node = taxon_to_node[key]
-
-            if taxon_child is not None:
-                node.add_child(taxon_child)
-
-            node.add_id(id_source, int(taxon_id))  # np.int64 -> int
-            if j == 0:
-                if level != taxon_level:
-                    print(f'row: {i_row}, {ds}, {ds_label}')
-                    print(f'- taxonomy_level column: {level}, '
-                          f'level from taxonomy_string: {taxon_level}')
-                    print()
-                    num_taxon_level_errors += 1
-
-                if scientific_name != taxon_name:
-                    print(f'row: {i_row}, {ds}, {ds_label}')
-                    print(f'- scientific_name column: {scientific_name}, '
-                          f'name from taxonomy_string: {taxon_name}')
-                    print()
-                    num_scientific_name_errors += 1
-
-            taxon_child = node
-    
-    # ...for each row in the taxonomy file        
-
-    assert nx.is_directed_acyclic_graph(graph)
-
-    for node in graph.nodes:
-        assert len(node.parents) <= 2
-        if len(node.parents) == 2:
-            p0 = node.parents[0]
-            p1 = node.parents[1]
-            assert p0 is not p1
-
-            p0_is_ancestor_of_p1 = p1 in nx.descendants(graph, p0)
-            p1_is_ancestor_of_p0 = p0 in nx.descendants(graph, p1)
-            if not p0_is_ancestor_of_p1 and not p1_is_ancestor_of_p0:
-                print('Node with two ambiguous parents:', node)
-                print('\t', p0)
-                print('\t\t', p0.parents)
-                print('\t', p1)
-                print('\t\t', p1.parents)
-
-    try:
-        dag_to_tree(graph, taxon_to_node)
-        print('All ambiguous parents have hard-coded resolution in '
-              'dag_to_tree().')
-    except AssertionError as e:
-        print(f'At least one node has unresolved ambiguous parents: {e}')
-
-    print('Processed {} rows from {}'.format(len(taxonomy_df),csv_path))
-    
-    print('num taxon level errors:', num_taxon_level_errors)
-    print('num scientific name errors:', num_scientific_name_errors)
-
-
-#%% Command-line driver
-    
-if __name__ == '__main__':
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'taxonomy_csv_path',
-        help='path to taxonomy CSV file')
-    
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-    
-    args = parser.parse_args()
-
-    check_taxonomy_csv(args.taxonomy_csv_path)
-
-
-#%% Interactive driver
-    
-if False:
-    
-    #%%
-    
-    import os
-    csv_path = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
-    check_taxonomy_csv(csv_path)
-    

taxonomy_mapping/taxonomy_graph.py

@@ -1,346 +0,0 @@
-"""
-
-taxonomy_graph.py
-
-Methods for transforming taxonomy CSV into a graph structure backed by
-NetworkX.
-
-We treat each taxon in the taxonomy as a node in a graph, represented by the
-TaxonNode class. We use a NetworkX directed graph (nx.DiGraph) to keep track of
-the edges (parent-child relationships) between the nodes.
-
-In theory, the true biological taxonomy graph should be a tree, where every
-taxon node has exactly 1 parent. However, because we use both GBIF and INAT
-taxonomies, there are 2 situations where a taxon node ends up with two parents.
-Thus, the graph is actually a "directed acyclic graph" (DAG) instead of a tree.
-
-The two situations are explained in detail below. This module includes a
-function dag_to_tree() which converts a DAG to a tree by heuristically removing
-edges from the DAG so that each node only has 1 parent.
-
-CASE 1: INAT and GBIF have different granularity in their taxonomy levels
-======
-An example is shown below. In dag_to_tree(), the lower parent is kept, while
-the higher-up parent is discarded. In this example, the "sciurini -> sciurus"
-edge would be kept, while "sciuridae -> sciurus" would be removed.
-
-        "eastern gray squirrel" (inat)     "squirrel" (gbif)
-        ------------------------------     -----------------
-    family:                        sciuridae
-                                  /          \
-    subfamily:          sciurinae             |  # skips subfamily
-                                |             |
-    tribe:               sciurini             |  # skips tribe
-                                  \          /
-    genus:                          sciurus
-
-
-CASE 2: INAT and GBIF have different taxonomies
-======
-An example is shown below. In dag_to_tree(), the resolution to these
-discrepancies are hard-coded.
-
-    order:    cathartiformes (inat)     accipitriformes (gbif)
-                           \           /
-    family:                 cathartidae
-
-"""
-
-#%% Imports and constants
-
-# allow forward references in typing annotations
-from __future__ import annotations
-
-from typing import (ClassVar, Container, Dict, Iterable, List, Optional, Set,
-                    Tuple)
-
-import networkx as nx
-import pandas as pd
-
-default_source = 'inat'
-
-
-#%% Classes
-
-class TaxonNode:
-    """
-    A node in a taxonomy graph (DAG), associated with a set of dataset labels.
-
-    By default, we support multiple parents for each TaxonNode. See discussion
-    in module docstring above.
-    """
-    
-    # class variables
-    single_parent_only: ClassVar[bool] = False
-
-    # instance variables
-    level: str
-    name: str
-    ids: Set[Tuple[str, int]]
-    graph: Optional[nx.DiGraph]
-    dataset_labels: Set[Tuple[str, str]]
-
-    def __init__(self, level: str, name: str,
-                 graph: Optional[nx.DiGraph] = None):
-        
-        self.level = level
-        self.name = name
-        self.graph = graph
-        self.ids = set()
-        self.dataset_labels = set()
-
-    def __repr__(self):
-        id_str = ', '.join(f'{source}={id}' for source, id in self.ids)
-        return f'TaxonNode({id_str}, level={self.level}, name={self.name})'
-
-    @property  # read-only getter
-    def parents(self) -> List[TaxonNode]:
-        assert self.graph is not None
-        return list(self.graph.predecessors(self))
-
-    @parents.setter
-    def parents(self, parents: Iterable[TaxonNode]) -> None:
-        assert self.graph is not None
-        for p in self.parents:
-            self.graph.remove_edge(p, self)
-        for p in parents:
-            self.graph.add_edge(p, self)
-
-    @property  # read-only getter
-    def children(self) -> List[TaxonNode]:
-        assert self.graph is not None
-        return list(self.graph.successors(self))
-
-    @children.setter
-    def children(self, children: Iterable[TaxonNode]) -> None:
-        assert self.graph is not None
-        for c in self.children:
-            self.graph.remove_edge(self, c)
-        for c in children:
-            self.graph.add_edge(self, c)
-
-    def add_id(self, source: str, taxon_id: int) -> None:
-        # assert source in ['gbif', 'inat', 'manual']
-        self.ids.add((source, taxon_id))
-
-    def add_parent(self, parent: TaxonNode) -> None:
-        """
-        Adds a TaxonNode to the list of parents of the current TaxonNode.
-        Requires this TaxonNode to be associated with a Graph.
-
-        Args:
-            parent: TaxonNode, must be higher in the taxonomical hierarchy
-        """
-        
-        assert self.graph is not None
-        parents = self.parents
-        if TaxonNode.single_parent_only and len(parents) > 0:
-            assert len(parents) == 1
-            assert parents[0] is parent, (
-                f'self.parents: {parents}, new parent: {parent}')
-            return
-        if parent not in parents:
-            self.graph.add_edge(parent, self)
-
-    def add_child(self, child: TaxonNode) -> None:
-        """
-        Adds a TaxonNode to the list of children of the current TaxonNode.
-        Requires this TaxonNode to be associated with a Graph.
-
-        Args:
-            child: TaxonNode, must be lower in the taxonomical hierarchy
-        """
-        
-        assert self.graph is not None
-        self.graph.add_edge(self, child)
-
-    def add_dataset_label(self, ds: str, ds_label: str) -> None:
-        """
-        Args:
-            ds: str, name of dataset
-            ds_label: str, name of label used by that dataset
-        """
-        
-        self.dataset_labels.add((ds, ds_label))
-
-    def get_dataset_labels(self,
-                           include_datasets: Optional[Container[str]] = None
-                           ) -> Set[Tuple[str, str]]:
-        """
-        Returns a set of all (ds, ds_label) tuples that belong to this taxon
-        node or its descendants.
-
-        Args:
-            include_datasets: list of str, names of datasets to include
-                if None, then all datasets are included
-
-        Returns: set of (ds, ds_label) tuples
-        """
-        
-        result = self.dataset_labels
-        if include_datasets is not None:
-            result = set(tup for tup in result if tup[0] in include_datasets)
-
-        for child in self.children:
-            result |= child.get_dataset_labels(include_datasets)
-        return result
-
-    @classmethod
-    def lowest_common_ancestor(cls, nodes: Iterable[TaxonNode]
-                               ) -> Optional[TaxonNode]:
-        """
-        Returns the lowest common ancestor (LCA) of a list or set of nodes.
-
-        For each node in <nodes>, get the set of nodes on the path to the root.
-        The LCA of <nodes> is certainly in the intersection of these sets.
-        Iterate through the nodes in this set intersection, looking for a node
-        such that none of its children is in this intersection. Given n nodes
-        from a k-ary tree of height h, the algorithm runs in O((n + k)h).
-
-        Returns: TaxonNode, the LCA if it exists, or None if no LCA exists
-        """
-        
-        paths = []
-        for node in nodes:
-            # get path to root
-            path = {node}
-            remaining = node.parents.copy()  # make a shallow copy
-            while len(remaining) > 0:
-                x = remaining.pop()
-                if x not in path:
-                    path.add(x)
-                    remaining += x.parents
-            paths.append(path)
-        intersect = set.intersection(*paths)
-
-        for node in intersect:
-            if intersect.isdisjoint(node.children):
-                return node
-        return None
-
-
-#%% Module functions
-
-def build_taxonomy_graph(taxonomy_df: pd.DataFrame
-                         ) -> Tuple[
-                             nx.DiGraph,
-                             Dict[Tuple[str, str], TaxonNode],
-                             Dict[Tuple[str, str], TaxonNode]
-                         ]:
-    """
-    Creates a mapping from (taxon_level, taxon_name) to TaxonNodes, used for
-    gathering all dataset labels associated with a given taxon.
-
-    Args:
-        taxonomy_df: pd.DataFrame, the taxonomy CSV
-
-    Returns:
-        graph: nx.DiGraph
-        taxon_to_node: dict, maps (taxon_level, taxon_name) to a TaxonNode,
-            keys are all lowercase
-        label_to_node: dict, maps (dataset_name, dataset_label) to the lowest
-            TaxonNode node in the tree that contains the label,
-            keys are all lowercase
-    """
-    
-    graph = nx.DiGraph()
-    taxon_to_node = {}  # maps (taxon_level, taxon_name) to a TaxonNode
-    label_to_node = {}  # maps (dataset_name, dataset_label) to a TaxonNode
-    for _, row in taxonomy_df.iterrows():
-        ds = row['dataset_name'].lower()
-        ds_label = row['query'].lower()
-        if 'source' in row:
-            id_source = row['source']
-        else:
-            id_source = default_source
-        taxa_ancestry = row['taxonomy_string']
-        if pd.isna(taxa_ancestry):
-            # taxonomy CSV rows without 'taxonomy_string' entries are excluded
-            # from the taxonomy graph, but can be included in a classification
-            # label specification JSON via the 'dataset_labels' key
-            continue
-        else:
-            taxa_ancestry = eval(taxa_ancestry)  # pylint: disable=eval-used
-
-        taxon_child: Optional[TaxonNode] = None
-        for i, taxon in enumerate(taxa_ancestry):
-            taxon_id, taxon_level, taxon_name, _ = taxon
-            taxon_level = taxon_level.lower()
-            taxon_name = taxon_name.lower()
-
-            key = (taxon_level, taxon_name)
-            if key not in taxon_to_node:
-                taxon_to_node[key] = TaxonNode(level=taxon_level,
-                                               name=taxon_name, graph=graph)
-            node = taxon_to_node[key]
-
-            if taxon_child is not None:
-                node.add_child(taxon_child)
-
-            node.add_id(id_source, int(taxon_id))  # np.int64 -> int
-            if i == 0:
-                assert row['taxonomy_level'] == taxon_level, (
-                    f'taxonomy CSV level: {row["taxonomy_level"]}, '
-                    f'level from taxonomy_string: {taxon_level}')
-                assert row['scientific_name'] == taxon_name
-                node.add_dataset_label(ds, ds_label)
-                label_to_node[(ds, ds_label)] = node
-
-            taxon_child = node
-
-    assert nx.is_directed_acyclic_graph(graph)
-    return graph, taxon_to_node, label_to_node
-
-
-def dag_to_tree(graph: nx.DiGraph,
-                taxon_to_node: Dict[Tuple[str, str], TaxonNode]) -> nx.DiGraph:
-    """
-    Converts the taxonomy graph from a DAG to a tree. See module docstring
-    for more information.
-
-    NOTE: nx.is_tree() on the output of this function might fail because the
-    tree may have disconnected components. Instead, check nx.is_tree() on each
-    component separately.
-
-    Args:
-        graph: nx.DiGraph, DAG representation of taxonomy hieararchy
-        taxon_to_node: dict, maps (taxon_level, taxon_name) to a TaxonNode
-
-    Returns: nx.DiGraph, a tree-structured graph
-    """
-    
-    tree = nx.DiGraph()
-    for node in graph.nodes:
-        tree.add_node(node)
-
-        if len(node.parents) == 1:
-            tree.add_edge(node.parents[0], node)
-
-        elif len(node.parents) == 2:
-            p0 = node.parents[0]
-            p1 = node.parents[1]
-
-            # use the lower parent
-            if p1 in nx.descendants(graph, p0):
-                tree.add_edge(p1, node)
-            elif p0 in nx.descendants(graph, p1):
-                tree.add_edge(p0, node)
-            else:
-                # special cases
-                if node.name == 'cathartidae':
-                    p = taxon_to_node[('order', 'accipitriformes')]
-                elif node.name == 'soricidae':
-                    p = taxon_to_node[('order', 'eulipotyphla')]
-                elif node.name == 'nyctanassa violacea':
-                    p = taxon_to_node[('genus', 'nyctanassa')]
-                elif node.name == 'trochilidae':  # this one is controversial
-                    p = taxon_to_node[('order', 'caprimulgiformes')]
-                else:
-                    assert False
-
-                assert (p is p0) or (p is p1)
-                tree.add_edge(p, node)
-
-    for node in tree.nodes:
-        node.graph = tree
-    return tree

taxonomy_mapping/validate_lila_category_mappings.py

@@ -1,83 +0,0 @@
-"""
-
-validate_lila_category_mappings.py
-
-Confirm that all category names on LILA have mappings in the taxonomy file.
-
-"""
-
-#%% Constants and imports
-
-import json
-import os
-
-from data_management.lila.lila_common import read_lila_taxonomy_mapping
-
-
-#%% Prevent execution during infrastructural imports
-
-if False:
-        
-    #%% Constants
-    
-    lila_local_base = os.path.expanduser('~/lila')
-
-    metadata_dir = os.path.join(lila_local_base,'metadata')
-    os.makedirs(metadata_dir,exist_ok=True)
-
-    # Created by get_lila_category_list.py... contains counts for each category
-    category_list_dir = os.path.join(lila_local_base,'lila_categories_list')
-    lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
-
-    assert os.path.isfile(lila_dataset_to_categories_file)
-        
-
-    #%% Load category and taxonomy files
-
-    with open(lila_dataset_to_categories_file,'r') as f:
-        lila_dataset_to_categories = json.load(f)
-
-    taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
-
-
-    #%% Map dataset names and category names to scientific names
-
-    ds_query_to_scientific_name = {}
-
-    unmapped_queries = set()
-
-    # i_row = 1; row = taxonomy_df.iloc[i_row]; row
-    for i_row,row in taxonomy_df.iterrows():
-        
-        ds_query = row['dataset_name'] + ':' + row['query']
-        ds_query = ds_query.lower()
-        
-        if not isinstance(row['scientific_name'],str):
-            unmapped_queries.add(ds_query)
-            ds_query_to_scientific_name[ds_query] = 'unmapped'
-            continue
-            
-        ds_query_to_scientific_name[ds_query] = row['scientific_name']
-        
-        
-    #%% For each dataset, make sure we can map every category to the taxonomy
-
-    # dataset_name = list(lila_dataset_to_categories.keys())[0]
-    for _dataset_name in lila_dataset_to_categories.keys():
-        
-        if '_bbox' in _dataset_name:
-            dataset_name = _dataset_name.replace('_bbox','')
-        else:
-            dataset_name = _dataset_name
-        
-        categories = lila_dataset_to_categories[dataset_name]
-        
-        # c = categories[0]
-        for c in categories:
-            ds_query = dataset_name + ':' + c['name']
-            ds_query = ds_query.lower()
-            
-            if ds_query not in ds_query_to_scientific_name:
-                print('Could not find mapping for {}'.format(ds_query))            
-            else:
-                scientific_name = ds_query_to_scientific_name[ds_query]