refscan 0.0.0__py3-none-any.whl

refscan/lib/Finder.py

@@ -0,0 +1,229 @@
+from typing import List, Optional, Dict, Tuple
+
+from pymongo.database import Database
+from pymongo.client_session import ClientSession
+
+
+class Finder:
+    r"""
+    A class that can be used to find a document in a MongoDB database, in a way that uses past searches to try to
+    speed up future searches.
+    """
+
+    def __init__(self, database: Database):
+        r"""
+        Initializes an instance of the `Finder` class. The instance is bound to the specified MongoDB database.
+        """
+        self.db = database
+
+        # Initialize our cache of names of collections we most recently found referenced document `id`s in.
+        #
+        # Note: This is a queue we will use to keep track of which collections we most recently found referenced
+        #       document `id`s in. This will enable us to _begin_ searching in those same collections for subsequent
+        #       searches. The idea that this will speed things up is based on the assumption that references emanating
+        #       from source documents that we process _temporally close_ to one another, likely point to target
+        #       documents residing _in the same collections_ as one another.
+        #
+        self.cached_collection_names_where_recently_found: List[str] = []
+        self.cache_size: int = 2  # we'll cache up to 2 collection names
+
+        # Initialize our cache of document `id` presences/absences by collection.
+        #
+        # Note: This is a dictionary we will use to keep track of the referenced `id`s we find in each collection,
+        #       and the referenced `id`s we fail to find (i.e. confirm do _not_ exist) in each collection. This will
+        #       enable us to avoid querying the database for the same information that we have previously obtained
+        #       by querying the database.
+        #
+        # Note: The top-level items in the dictionary have keys that are collection names and values that are
+        #       dictionaries. Each of those dictionaries has keys that are document `id`s and values that are boolean
+        #       flags, which indicate whether we know that a document having that `id` exists in that collection
+        #       (`True`) or we know that a document having that `id` does _not_ exist in that collection (`False`).
+        #       Here's an example value of this attribute:
+        #       ```
+        #       {
+        #           "biosample_set": {"nmdc:bsm-00-000001": True, "nmdc:bsm-00-000002": False},
+        #           "study_set": {"nmdc:sty-00-000001": True},
+        #       }
+        #       ```
+        #
+        # Note: If this program ever leads to memory consumption (RAM usage) issues in practice, consider
+        #       redesigning this cache and/or allowing the user to disable its use.
+        #
+        self.cached_id_presence_by_collection: Dict[str, Dict[str, bool]] = {}
+
+    def _set_name_of_collection_most_recently_found_in(self, collection_name: str):
+        r"""
+        Helper function that updates our cache of collection names in which target documents were most recently found,
+        so that the specified collection is at the front/start of the list. That will make it so it gets searched first
+        next time.
+        """
+        # Make a concise alias for the instance attribute (both will refer to the same data structure).
+        queue = self.cached_collection_names_where_recently_found
+
+        # Check whether either (a) the queue is empty or (b) this collection name is not at the front of the queue.
+        # Example: [] is empty, or "b_set" is not the first item in ["a_set", "b_set", "c_set"]
+        if len(queue) == 0 or queue[0] != collection_name:
+
+            # If this collection name is already (elsewhere) in the queue, remove it from that position, automatically
+            # updating the list indexes of subsequent list items (so there is no "gap" in the list).
+            # Example: Removing "b_set" from ["a_set", "b_set", "c_set"] → ["a_set", "c_set"]
+            if collection_name in queue:
+                queue.remove(collection_name)
+
+            # Prepend this collection name to the beginning of the queue, dropping any excess collection names.
+            # Example: Prepending ["b_set"] to ["a_set", "c_set"] → ["b_set", "a_set", "c_set"]
+            queue = [collection_name] + queue[0 : self.cache_size - 1]
+
+            # Note: This unnecessary assignment is here only to tell my IDE that `queue` isn't an unused variable.
+            self.cached_collection_names_where_recently_found = queue
+
+    def _optimize_collection_search_order(self, names_of_eligible_collections: List[str]) -> List[str]:
+        r"""
+        Helper function that returns an optimized order in which the specified collections could be searched, in order
+        to take advantage of things the instance "remembers" from recent searches.
+        """
+        ordered_collection_names = names_of_eligible_collections.copy()
+
+        # Iterate over the collection names in the queue, from back (oldest) to front (newest).
+        for cached_collection_name in reversed(self.cached_collection_names_where_recently_found):
+
+            # If this cached collection name is among the eligible ones, move it to the front of the list.
+            # Example: Since "b_set" is in ["a_set", "b_set", "c_set"] → ["b_set, "a_set", "c_set"]
+            if cached_collection_name in names_of_eligible_collections:
+                ordered_collection_names.remove(cached_collection_name)
+                ordered_collection_names.insert(0, cached_collection_name)
+
+        return ordered_collection_names
+
+    def _set_cached_id_presence_in_collection(self, collection_name: str, document_id: str, is_present: bool) -> None:
+        r"""
+        Helper function that updates our cache of document `id` presences/absences, setting the presence flag for the
+        specified document `id` in the specified collection.
+        """
+        if collection_name not in self.cached_id_presence_by_collection:
+            self.cached_id_presence_by_collection[collection_name] = {}  # creates key if it does not exist
+        self.cached_id_presence_by_collection[collection_name][document_id] = is_present
+
+    def _get_cached_id_presence_in_collection(self, collection_name: str, document_id: str) -> Optional[bool]:
+        r"""
+        Helper function that checks our cache of document `id` presences/absences, returning the presence/absence flag,
+        if any, for the specified document `id` in the specified collection.
+        """
+        if (
+            collection_name not in self.cached_id_presence_by_collection
+            or document_id not in self.cached_id_presence_by_collection[collection_name]
+        ):
+            return None
+        else:
+            return self.cached_id_presence_by_collection[collection_name][document_id]
+
+    def check_whether_document_having_id_exists_among_collections(
+        self, document_id: str, collection_names: List[str], client_session: Optional[ClientSession] = None
+    ) -> Optional[str]:
+        r"""
+        Checks whether any document in any of the specified collections has the specified value in its `id` field.
+        Returns the name of the first collection, if any, containing such a document. If none of the collections
+        contain such a document, the function returns `None`.
+
+        If a `pymongo.client_session.ClientSession` is specified, this function will access the database within
+        the context of that session. If a transaction is pending on that session, this feature can be used
+        to check whether the specified document _would_ exist if that transaction were to be committed.
+
+        References:
+        - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.find_one
+        - https://pymongo.readthedocs.io/en/stable/api/pymongo/client_session.html#transactions
+        """
+
+        # Initialize the name of the containing collection to `None`.
+        name_of_collection_containing_target_document = None
+
+        # Optimize the search order of the collections, based upon the outcomes of recent searches.
+        ordered_collection_names = self._optimize_collection_search_order(collection_names)
+
+        # Search the collections.
+        query_filter = dict(id=document_id)
+        for collection_name in ordered_collection_names:
+
+            # Before querying the database, check our cache of document `id` presences/absences per collection.
+            is_id_present: Optional[bool] = self._get_cached_id_presence_in_collection(
+                collection_name=collection_name, document_id=document_id
+            )
+
+            # If we already know it's _not_ in this collection, stop evaluating this collection and, instead,
+            # proceed to evaluate the next collection.
+            if is_id_present is False:
+                continue  # stop evaluating this collection
+
+            # If we already know it _is_ in this collection, record the collection name, update the cache of names
+            # of collections where a referenced document was recently found, and skip searching additional collections.
+            if is_id_present is True:
+                name_of_collection_containing_target_document = collection_name
+                self._set_name_of_collection_most_recently_found_in(collection_name=collection_name)
+                break
+
+            # If we find it in the collection in the database, record the collection name, update both (a) the cache
+            # of which documents we have found in which collections, and (b) the cache of names of collections where
+            # a referenced document was recently found, and skip searching additional collections.
+            collection = self.db.get_collection(collection_name)
+            if collection.find_one(query_filter, projection=["_id"], session=client_session) is not None:
+                name_of_collection_containing_target_document = collection_name
+
+                # Record that a document having this `id` is _present_ in this collection.
+                self._set_cached_id_presence_in_collection(
+                    collection_name=collection_name, document_id=document_id, is_present=True
+                )
+
+                self._set_name_of_collection_most_recently_found_in(collection_name=collection_name)
+                break
+
+            # If we made it this far, record that a document having this `id` does _not_ exist in this collection.
+            self._set_cached_id_presence_in_collection(
+                collection_name=collection_name, document_id=document_id, is_present=False
+            )
+
+        return name_of_collection_containing_target_document
+
+    def find_documents_having_type_and_value_in_field(
+        self,
+        collection_name: str,
+        type_and_field_name_tuples: List[Tuple[str, str]],
+        value: str,
+        client_session: Optional[ClientSession] = None,
+    ) -> List[dict]:
+        r"""
+        Finds all documents residing in the specified collection that—for any of the specified
+        `type`-and-`field_name` combinations—have both (a) that `type` value and (b) the specified
+        `value` value in their `field_name` field. Returns a list of descriptors of those
+        documents (each descriptor consists of the document's `_id`, `id`, and `type`).
+
+        For example, if the `type_and_field_name_tuples` parameter contains the tuple
+        `("nmdc:Study", "part_of")`, and the `value` parameter is `"nmdc:sty-00-000001"`,
+        then this function will return all documents in the specified collection where both
+        (a) the document's `type` field contains the value `"nmdc:Study"` and
+        (b) the document's `part_of` field contains the value `"nmdc:sty-00-000001"`.
+
+        Note: `type_and_field_name_tuples` is a list of tuples, where the first item in each tuple is a
+              `type` value (e.g. "nmdc:Study") and the second item is a field name (e.g. "part_of").
+
+        If a `pymongo.client_session.ClientSession` is specified, this function will access the database within
+        the context of that session.
+        """
+
+        # Build the query filter terms based on the `type`-and-`field_name` tuples.
+        query_filter_terms = []
+        for type_and_field_name_tuple in type_and_field_name_tuples:
+            type_value, field_name = type_and_field_name_tuple
+            query_filter_term = {"type": type_value, field_name: value}
+            query_filter_terms.append(query_filter_term)
+
+        # Define the overall query filter as a logical "OR" of the individual terms.
+        query_filter = {"$or": query_filter_terms}
+
+        # Only request essential data from the database (as a performance optimization).
+        projection = ["_id", "id", "type"]
+
+        documents = []
+        collection = self.db.get_collection(collection_name)
+        for document in collection.find(query_filter, projection=projection, session=client_session):
+            documents.append(document)
+        return documents

refscan/lib/Reference.py

@@ -0,0 +1,37 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, order=True)
+class Reference:
+    """
+    A generic reference to a document in a collection.
+
+    Note: `frozen` means the instances are immutable.
+    Note: `order` means the instances have methods that help with sorting. For example, an `__eq__` method that
+          can be used to compare instances of the class as though they were tuples of those instances' fields.
+    """
+
+    source_collection_name: str = field()  # e.g. "study_set"
+    source_class_name: str = field()  # e.g. "Study"
+    source_field_name: str = field()  # e.g. "part_of"
+    target_collection_name: str = field()  # e.g. "study_set" (reminder: a study can be part of another study)
+    target_class_name: str = field()  # e.g. "Study"
+
+    def __eq__(self, other):
+        r"""
+        Determines whether an instance of this class is equal to the specified "other" value.
+
+        Note: This method dictates what will happen under the hood when the `==` operator is used.
+              Reference: https://docs.python.org/3/reference/datamodel.html#object.__eq__
+        """
+
+        if not isinstance(other, Reference):
+            return False
+        else:
+            return (
+                self.source_collection_name == other.source_collection_name
+                and self.source_class_name == other.source_class_name
+                and self.source_field_name == other.source_field_name
+                and self.target_collection_name == other.target_collection_name
+                and self.target_class_name == other.target_class_name
+            )

refscan/lib/ReferenceList.py

@@ -0,0 +1,206 @@
+from typing import List, Dict, Union, Optional
+from pathlib import Path
+from dataclasses import fields, astuple
+from collections import UserList
+from itertools import groupby
+import csv
+
+from rich.table import Table, Column
+
+from refscan.lib.Reference import Reference
+
+
+class ReferenceList(UserList):
+    """
+    A list of references.
+
+    Note: `UserList` is a base class that facilitates the implementation of custom list classes.
+          One thing it does is enable sorting via `sorted(the_list)`.
+    """
+
+    # A cached dictionary that maps source class names to a list of the names of
+    # the fields of that class that contain references. Initialized to `None` to
+    # indicate that it hasn't been computed/cached yet.
+    #
+    # Example: {"Study": ["part_of"]}
+    #
+    # TODO: Consider implementing caching via functools's `@cached_property` decorator instead.
+    #       Reference: https://docs.python.org/3/library/functools.html#functools.cached_property
+    #
+    _reference_field_names_by_source_class_name: Optional[Dict[str, List[str]]] = None
+
+    def get_source_collection_names(self) -> list[str]:
+        """
+        Returns the distinct `source_collection_names` values among all references in the list.
+        """
+        distinct_source_collection_names = []
+        for reference in self.data:
+            if reference.source_collection_name not in distinct_source_collection_names:
+                distinct_source_collection_names.append(reference.source_collection_name)
+        return distinct_source_collection_names
+
+    def count_source_collections(self) -> int:
+        r"""
+        Returns the number of distinct source collection names among all references in the list.
+        """
+        return len(self.get_source_collection_names())
+
+    def get_source_field_names_of_source_collection(self, collection_name: str) -> list[str]:
+        """
+        Returns the distinct source field names of the specified source collection.
+        """
+        distinct_source_field_names = []
+        for reference in self.data:
+            if reference.source_collection_name == collection_name:
+                if reference.source_field_name not in distinct_source_field_names:
+                    distinct_source_field_names.append(reference.source_field_name)
+        return distinct_source_field_names
+
+    def get_target_collection_names(
+        self,
+        source_class_name: str,
+        source_field_name: str,
+    ) -> list[str]:
+        """
+        Returns a list of the names of the collections in which a [target] document referenced by the specified field
+        of a [source] document representing an instance of the specified schema class, might exist.
+
+        TODO: Execution spends a lot of time in this function. Consider replacing it with something faster (e.g. a LUT).
+        """
+        target_collection_names = []
+        for reference in self.data:  # note: in a `UserList`, `self.data` refers to the underlying list data structure
+
+            # If this reference's source describes the specified source, record the reference's target collection name.
+            if reference.source_class_name == source_class_name and reference.source_field_name == source_field_name:
+                target_collection_names.append(reference.target_collection_name)
+
+        distinct_target_collection_names = list(set(target_collection_names))
+        return distinct_target_collection_names
+
+    def get_groups(self, field_names: list[str]):
+        r"""
+        Returns an iterable of groups, where each group has a distinct combination of values in the specified fields.
+
+        TODO: Add a type hint that indicates the return type. It's an instance of a class named `groupby` (lowercase).
+
+        Note: This method can be used to "consolidate" references that have the same source collection name,
+              source field name, and target collection name (i.e. ones that only differ by target class name).
+        """
+
+        def make_group_key(reference: Reference) -> tuple:
+            """Helper function that returns a key that can be used to group references."""
+            values = []
+            for field_name in field_names:
+                if not hasattr(reference, field_name):
+                    raise ValueError(f"No such field: {field_name}")
+                values.append(getattr(reference, field_name))
+            return tuple(values)
+
+        groups = groupby(sorted(self.data), key=make_group_key)
+        return groups
+
+    def dump_to_tsv_file(self, file_path: Union[str, Path]) -> None:
+        r"""
+        Helper function that dumps the references to a TSV file at the specified path.
+        """
+        column_names = [field_.name for field_ in fields(Reference)]
+        with open(file_path, "w", newline="") as tsv_file:
+            writer = csv.writer(tsv_file, delimiter="\t")
+            writer.writerow(column_names)  # header row
+            for reference in self.data:
+                writer.writerow(astuple(reference))  # data row
+
+    def get_reference_field_names_by_source_class_name(self) -> Dict[str, List[str]]:
+        r"""
+        Returns a dictionary that maps source class names to a list of the names of
+        the fields of that class that contain references.
+
+        Example: {"Study": ["part_of"]}
+        """
+
+        # If this dictionary has already been computed by this class, return that cached dictionary.
+        if isinstance(self._reference_field_names_by_source_class_name, dict):
+            return self._reference_field_names_by_source_class_name
+
+        reference_field_names_by_source_class_name: Dict[str, List[str]] = {}
+        for reference in self.data:
+            source_class_name = reference.source_class_name
+            source_field_name = reference.source_field_name
+
+            # Initialize a dictionary item for this source class, if one doesn't already exist.
+            if source_class_name not in reference_field_names_by_source_class_name.keys():
+                reference_field_names_by_source_class_name[source_class_name] = []
+
+            # If the source field name isn't already in this class's list, append it.
+            if source_field_name not in reference_field_names_by_source_class_name[source_class_name]:
+                reference_field_names_by_source_class_name[source_class_name].append(source_field_name)
+
+        # Cache the dictionary for subsequent invocations of this method.
+        self._reference_field_names_by_source_class_name = reference_field_names_by_source_class_name
+
+        return reference_field_names_by_source_class_name
+
+    def get_by_target_class_name(self, target_class_name: str) -> "ReferenceList":
+        r"""
+        Returns a new `ReferenceList` that contains only the references whose target class
+        has the specified name.
+
+        Maintainer note: The return type is written as a string to avoid the error:
+                         > NameError: name 'ReferenceList' is not defined
+        """
+
+        filtered_references = ReferenceList()
+        for reference in self.data:
+            if reference.target_class_name == target_class_name:
+                filtered_references.append(reference)
+        return filtered_references
+
+    def group_by_source_collection_name(self) -> Dict[str, "ReferenceList"]:
+        r"""
+        Returns a dictionary that maps source collection names to `ReferenceList` instances
+        containing the references that have that source collection name.
+        """
+
+        grouped_references: Dict[str, ReferenceList] = {}
+        for reference in self.data:
+            if reference.source_collection_name not in grouped_references:
+                grouped_references[reference.source_collection_name] = ReferenceList()
+            grouped_references[reference.source_collection_name].append(reference)
+
+        return grouped_references
+
+    def as_table(self) -> Table:
+        r"""
+        Returns the references as a `rich.Table` instance.
+        """
+
+        # Make the data rows for the table. Data rows that would have had the same
+        # (combination of) the following fields, get consolidated into a single row;
+        # and all the target class names are displayed as a list on that row.
+        fields_to_group_rows_by = [
+            "source_collection_name",
+            "source_class_name",
+            "source_field_name",
+            "target_collection_name",
+        ]
+        groups = self.get_groups(fields_to_group_rows_by)
+        data_rows: list[tuple[str, str, str, str, str]] = []
+        for key, group in groups:
+            target_class_names = list(set([ref.target_class_name for ref in group]))  # omit duplicate class names
+            row = (key[0], key[1], key[2], key[3], ", ".join(target_class_names))
+            data_rows.append(row)
+
+        # Initialize the table, then add the data rows to it.
+        table = Table(
+            Column(header="Source collection", footer=f"{len(data_rows)} rows"),
+            Column(header="Source class"),
+            Column(header="Source field"),
+            Column(header="Target collection"),
+            Column(header="Target class(es)"),
+            title="References",
+            show_footer=True,
+        )
+        for row in data_rows:
+            table.add_row(*row)
+
+        return table

refscan/lib/Violation.py

@@ -0,0 +1,23 @@
+from typing import Optional
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, order=True)
+class Violation:
+    """
+    A specific reference that lacks integrity.
+    """
+
+    source_collection_name: str = field()
+    source_class_name: str = field()
+    source_field_name: str = field()
+    source_document_object_id: str = field()
+    source_document_id: str = field()
+    target_id: str = field()
+
+    # Note: This field contains the name of the collection, if any, in which the targeted document was found.
+    #       Its name was designed to emphasize that it is _not_ a collection that the schema allows the document
+    #       to exist in, but is instead the collection the document _actually_ exists in. The fact that the latter
+    #       is not among the former is why there is a `Violation` instance describing this reference.
+    #
+    name_of_collection_containing_target: Optional[str] = field()

refscan/lib/ViolationList.py

@@ -0,0 +1,28 @@
+from typing import Union
+from pathlib import Path
+from dataclasses import fields, astuple
+from collections import UserList
+import csv
+
+from refscan.lib.Violation import Violation
+
+
+class ViolationList(UserList):
+    """
+    A list of violations.
+    """
+
+    def dump_to_tsv_file(
+        self, file_path: Union[str, Path], include_name_of_collection_containing_target: bool = True
+    ) -> None:
+        """
+        Helper function that dumps the violations to a TSV file at the specified path.
+        """
+        column_names = [field_.name for field_ in fields(Violation)]
+        if not include_name_of_collection_containing_target:
+            column_names = [cn for cn in column_names if cn != "name_of_collection_containing_target"]
+        with open(file_path, "w", newline="") as tsv_file:
+            writer = csv.writer(tsv_file, delimiter="\t")
+            writer.writerow(column_names)  # header row
+            for violation in self.data:
+                writer.writerow(astuple(violation))  # data row

refscan/lib/constants.py

@@ -0,0 +1,8 @@
+from rich.console import Console
+
+# Instantiate a Rich console for fancy console output.
+# Reference: https://rich.readthedocs.io/en/stable/console.html
+console = Console()
+
+# Note: This is the only schema class name hard-coded into this script.
+DATABASE_CLASS_NAME = "Database"