refscan 0.2.3__tar.gz → 0.2.5__tar.gz

{refscan-0.2.3 → refscan-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: refscan
-Version: 0.2.3
+Version: 0.2.5
 Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
 Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
 Requires-Python: >=3.9,<4.0

{refscan-0.2.3 → refscan-0.2.5}/pyproject.toml

@@ -2,7 +2,7 @@
 
 [tool.poetry]
 name = "refscan"
-version = "v0.2.3"
+version = "v0.2.5"
 description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations"
 authors = []
 readme = "README.md"

{refscan-0.2.3 → refscan-0.2.5}/refscan/cli/scan.py

@@ -8,10 +8,10 @@ import linkml_runtime
 from refscan.lib.constants import console
 from refscan.lib.helpers import (
     connect_to_database,
+    get_collection_name_to_class_names_map,
     get_collection_names_from_schema,
     get_lowercase_key,
     print_section_header,
-    get_names_of_classes_eligible_for_collection,
     identify_references,
 )
 from refscan.lib.ViolationList import ViolationList
@@ -134,12 +134,7 @@ def scan(
     console.print(f"Collections described by schema: {len(collection_names)}")
 
     # For each collection, determine the names of the classes whose instances can be stored in that collection.
-    collection_name_to_class_names = {}  # example: { "study_set": ["Study"] }
-    for collection_name in sorted(collection_names):
-        collection_name_to_class_names[collection_name] = get_names_of_classes_eligible_for_collection(
-            schema_view=schema_view,
-            collection_name=collection_name,
-        )
+    collection_name_to_class_names = get_collection_name_to_class_names_map(schema_view=schema_view)
 
     # Identify the inter-document references that the schema allows to exist.
     references = identify_references(

{refscan-0.2.3 → refscan-0.2.5}/refscan/grapher.py

@@ -1,5 +1,4 @@
 from enum import Enum
-from pathlib import Path
 from typing import Union
 import json
 import base64
@@ -10,9 +9,9 @@ import linkml_runtime
 from refscan import get_package_metadata
 from refscan.lib.constants import console
 from refscan.lib.helpers import (
+    get_collection_name_to_class_names_map,
     print_section_header,
     get_collection_names_from_schema,
-    get_names_of_classes_eligible_for_collection,
     identify_references,
 )
 
@@ -78,12 +77,7 @@ def graph(
     console.print(f"Collections described by schema: {len(collection_names)}")
 
     # For each collection, determine the names of the classes whose instances can be stored in that collection.
-    collection_name_to_class_names = {}  # example: { "study_set": ["Study"] }
-    for collection_name in sorted(collection_names):
-        collection_name_to_class_names[collection_name] = get_names_of_classes_eligible_for_collection(
-            schema_view=schema_view,
-            collection_name=collection_name,
-        )
+    collection_name_to_class_names = get_collection_name_to_class_names_map(schema_view=schema_view)
 
     # Identify the inter-document references that the schema allows to exist.
     references = identify_references(

{refscan-0.2.3 → refscan-0.2.5}/refscan/lib/Finder.py

@@ -1,6 +1,7 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Tuple
 
 from pymongo.database import Database
+from pymongo.client_session import ClientSession
 
 
 class Finder:
@@ -10,6 +11,9 @@ class Finder:
     """
 
     def __init__(self, database: Database):
+        r"""
+        Initializes an instance of the `Finder` class. The instance is bound to the specified MongoDB database.
+        """
         self.db = database
 
         # Initialize our cache of names of collections we most recently found referenced document `id`s in.
@@ -114,15 +118,20 @@ class Finder:
             return self.cached_id_presence_by_collection[collection_name][document_id]
 
     def check_whether_document_having_id_exists_among_collections(
-        self, document_id: str, collection_names: List[str]
+        self, document_id: str, collection_names: List[str], client_session: Optional[ClientSession] = None
     ) -> Optional[str]:
         r"""
         Checks whether any document in any of the specified collections has the specified value in its `id` field.
         Returns the name of the first collection, if any, containing such a document. If none of the collections
         contain such a document, the function returns `None`.
 
+        If a `pymongo.client_session.ClientSession` is specified, this function will access the database within
+        the context of that session. If a transaction is pending on that session, this feature can be used
+        to check whether the specified document _would_ exist if that transaction were to be committed.
+
         References:
         - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.find_one
+        - https://pymongo.readthedocs.io/en/stable/api/pymongo/client_session.html#transactions
         """
 
         # Initialize the name of the containing collection to `None`.
@@ -155,7 +164,8 @@ class Finder:
             # If we find it in the collection in the database, record the collection name, update both (a) the cache
             # of which documents we have found in which collections, and (b) the cache of names of collections where
             # a referenced document was recently found, and skip searching additional collections.
-            if self.db.get_collection(collection_name).find_one(query_filter, projection=["_id"]) is not None:
+            collection = self.db.get_collection(collection_name)
+            if collection.find_one(query_filter, projection=["_id"], session=client_session) is not None:
                 name_of_collection_containing_target_document = collection_name
 
                 # Record that a document having this `id` is _present_ in this collection.
@@ -172,3 +182,48 @@ class Finder:
             )
 
         return name_of_collection_containing_target_document
+
+    def find_documents_having_type_and_value_in_field(
+        self,
+        collection_name: str,
+        type_and_field_name_tuples: List[Tuple[str, str]],
+        value: str,
+        client_session: Optional[ClientSession] = None,
+    ) -> List[dict]:
+        r"""
+        Finds all documents residing in the specified collection that—for any of the specified
+        `type`-and-`field_name` combinations—have both (a) that `type` value and (b) the specified
+        `value` value in their `field_name` field. Returns a list of descriptors of those
+        documents (each descriptor consists of the document's `_id`, `id`, and `type`).
+
+        For example, if the `type_and_field_name_tuples` parameter contains the tuple
+        `("nmdc:Study", "part_of")`, and the `value` parameter is `"nmdc:sty-00-000001"`,
+        then this function will return all documents in the specified collection where both
+        (a) the document's `type` field contains the value `"nmdc:Study"` and
+        (b) the document's `part_of` field contains the value `"nmdc:sty-00-000001"`.
+
+        Note: `type_and_field_name_tuples` is a list of tuples, where the first item in each tuple is a
+              `type` value (e.g. "nmdc:Study") and the second item is a field name (e.g. "part_of").
+
+        If a `pymongo.client_session.ClientSession` is specified, this function will access the database within
+        the context of that session.
+        """
+
+        # Build the query filter terms based on the `type`-and-`field_name` tuples.
+        query_filter_terms = []
+        for type_and_field_name_tuple in type_and_field_name_tuples:
+            type_value, field_name = type_and_field_name_tuple
+            query_filter_term = {"type": type_value, field_name: value}
+            query_filter_terms.append(query_filter_term)
+
+        # Define the overall query filter as a logical "OR" of the individual terms.
+        query_filter = {"$or": query_filter_terms}
+
+        # Only request essential data from the database (as a performance optimization).
+        projection = ["_id", "id", "type"]
+
+        documents = []
+        collection = self.db.get_collection(collection_name)
+        for document in collection.find(query_filter, projection=projection, session=client_session):
+            documents.append(document)
+        return documents

{refscan-0.2.3 → refscan-0.2.5}/refscan/lib/ReferenceList.py

@@ -121,6 +121,35 @@ class ReferenceList(UserList):
 
         return reference_field_names_by_source_class_name
 
+    def get_by_target_class_name(self, target_class_name: str) -> "ReferenceList":
+        r"""
+        Returns a new `ReferenceList` that contains only the references whose target class
+        has the specified name.
+
+        Maintainer note: The return type is written as a string to avoid the error:
+                         > NameError: name 'ReferenceList' is not defined
+        """
+
+        filtered_references = ReferenceList()
+        for reference in self.data:
+            if reference.target_class_name == target_class_name:
+                filtered_references.append(reference)
+        return filtered_references
+
+    def group_by_source_collection_name(self) -> Dict[str, "ReferenceList"]:
+        r"""
+        Returns a dictionary that maps source collection names to `ReferenceList` instances
+        containing the references that have that source collection name.
+        """
+
+        grouped_references: Dict[str, ReferenceList] = {}
+        for reference in self.data:
+            if reference.source_collection_name not in grouped_references:
+                grouped_references[reference.source_collection_name] = ReferenceList()
+            grouped_references[reference.source_collection_name].append(reference)
+
+        return grouped_references
+
     def as_table(self) -> Table:
         r"""
         Returns the references as a `rich.Table` instance.

{refscan-0.2.3 → refscan-0.2.5}/refscan/lib/helpers.py

@@ -1,4 +1,4 @@
-from typing import Optional, List
+from typing import Optional, List, Dict
 from functools import cache
 
 from pymongo import MongoClient, timeout
@@ -30,6 +30,7 @@ def connect_to_database(mongo_uri: str, database_name: str, verbose: bool = True
     return mongo_client
 
 
+@cache  # memoizes the decorated function
 def get_collection_names_from_schema(schema_view: SchemaView) -> list[str]:
     """
     Returns the names of the slots of the `Database` class that describe database collections.
@@ -53,6 +54,7 @@ def get_collection_names_from_schema(schema_view: SchemaView) -> list[str]:
     return collection_names
 
 
+@cache  # memoizes the decorated function
 def get_names_of_classes_eligible_for_collection(schema_view: SchemaView, collection_name: str) -> list[str]:
     r"""
     Returns a list of the names of the classes whose instances can be stored in the specified collection,
@@ -65,6 +67,30 @@ def get_names_of_classes_eligible_for_collection(schema_view: SchemaView, collec
     return names_of_eligible_classes
 
 
+@cache  # memoizes the decorated function
+def get_collection_name_to_class_names_map(
+    schema_view: SchemaView,
+) -> Dict[str, List[str]]:
+    r"""
+    Returns a mapping of collection names to the names of the classes whose instances can be stored
+    in those collections, according to the specified `SchemaView`.
+
+    Example output:
+    ```
+    {
+        "study_set": ["Study"],
+        "biosample_set": ["Biosample"],
+        ...
+    }
+    ```
+    """
+    collection_name_to_class_names = {}
+    for collection_name in get_collection_names_from_schema(schema_view):
+        class_names = get_names_of_classes_eligible_for_collection(schema_view, collection_name)
+        collection_name_to_class_names[collection_name] = class_names
+    return collection_name_to_class_names
+
+
 @cache  # memoizes the decorated function
 def translate_class_uri_into_schema_class_name(schema_view: SchemaView, class_uri: str) -> Optional[str]:
     r"""
@@ -85,6 +111,22 @@ def translate_class_uri_into_schema_class_name(schema_view: SchemaView, class_ur
     return schema_class_name
 
 
+@cache  # memoizes the decorated function
+def translate_schema_class_name_into_class_uri(schema_view: SchemaView, schema_class_name: str) -> Optional[str]:
+    r"""
+    Returns the `class_uri` of the schema class having the specified name.
+
+    Example: "Biosample" (a schema class name) -> "nmdc:Biosample" (a `class_uri` value)
+    """
+    class_uri = None
+    all_class_definitions_in_schema = schema_view.all_classes()
+    for class_name, class_definition in all_class_definitions_in_schema.items():
+        if class_name == schema_class_name:
+            class_uri = class_definition.class_uri
+            break
+    return class_uri
+
+
 def derive_schema_class_name_from_document(schema_view: SchemaView, document: dict) -> Optional[str]:
     r"""
     Returns the name of the schema class, if any, of which the specified document claims to represent an instance.

{refscan-0.2.3 → refscan-0.2.5}/refscan/scanner.py

@@ -1,12 +1,15 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Set, Tuple
 
 from pymongo.database import Database
+from pymongo.client_session import ClientSession
 from rich.console import Console
 from linkml_runtime import SchemaView
 
 from refscan.lib.Finder import Finder
 from refscan.lib.helpers import (
     derive_schema_class_name_from_document,
+    translate_class_uri_into_schema_class_name,
+    translate_schema_class_name_into_class_uri,
     init_progress_bar,
 )
 from refscan.lib.ReferenceList import ReferenceList
@@ -15,6 +18,95 @@ from refscan.lib.ViolationList import ViolationList
 from refscan.lib.constants import console as default_console
 
 
+def identify_referring_documents(
+    document: dict,
+    schema_view: SchemaView,
+    references: ReferenceList,
+    finder: Finder,
+    client_session: Optional[ClientSession] = None,
+) -> List[dict]:
+    r"""
+    Identifies documents that reference the specified one.
+
+    Note: This function only searches collections and fields that the _schema_ says
+          can reference the specified document. If an arbitrary collection contains
+          a document that has a field that contains the `id` of the specified document
+          (e.g. the `foo_set` collection contains a document whose `nickname` field
+          contains `someIdentifier`), it is not necessarily the case that this function
+          will consider that to be a reference. It depends upon what the _schema_ says.
+
+    Note: This can be useful for determining whether the specified document can be safely deleted.
+
+    :param document: The document whose incoming references you want to identify
+    :param schema_view: A `SchemaView` bound to the schema
+    :param references: A `ReferenceList` derived from the schema
+    :param finder: A `Finder` bound to the database being scanned
+    :param client_session: A `pymongo.client_session.ClientSession` instance that, if specified, will be used when
+                           searching for referring documents. If a transaction happens to be pending on that session,
+                           the scan will effectively happen on the database as it _would_ exist if that transaction
+                           were to be committed.
+    :return: A list of descriptors of documents that reference the specified document
+    """
+
+    # Initialize a list of descriptors of documents that reference the specified document.
+    referring_document_descriptors = []
+
+    # Get the specified document's `id` and derive its schema class name.
+    document_id = document["id"]
+    document_class_name = derive_schema_class_name_from_document(schema_view, document)
+    if document_class_name is None:
+        raise ValueError(f"Failed to identify schema class of document having `id`: {document_id}")
+
+    # Identify potential references to the specified document, then group them by their source collection names.
+    potential_references_to_document = references.get_by_target_class_name(document_class_name)
+    potential_references_by_source_collection = potential_references_to_document.group_by_source_collection_name()
+
+    # For each source collection name, check each of its potential references.
+    for collection_name, potential_references_in_collection in potential_references_by_source_collection.items():
+        # Make a list of all the distinct `class_uri`-and-`field_name` tuples.
+        #
+        # Note: Since a `Reference` doesn't have a `source_class_uri` attribute (but it does have a
+        #       `source_class_name` attribute), we derive its `class_uri` from that `source_class_name` attribute.
+        #
+        class_uri_and_field_name_tuples: Set[Tuple[str, str]] = set()
+        for potential_reference in potential_references_in_collection:
+            class_uri = translate_schema_class_name_into_class_uri(
+                schema_view=schema_view, schema_class_name=potential_reference.source_class_name
+            )
+            if class_uri is None:
+                raise ValueError(
+                    f"Failed to translate schema class name '{potential_reference.source_class_name}' into class_uri."
+                )
+            class_uri_and_field_name_tuple = (class_uri, potential_reference.source_field_name)
+            class_uri_and_field_name_tuples.add(class_uri_and_field_name_tuple)
+
+        # For each distinct combination of `class_uri` and `field_name`, check whether any documents in this
+        # collection both (a) have a `type` value matching that `class_uri` and (b) have the subject document's
+        # `id` value in the specified field.
+        referring_documents = finder.find_documents_having_type_and_value_in_field(
+            collection_name=collection_name,
+            type_and_field_name_tuples=list(class_uri_and_field_name_tuples),
+            value=document_id,
+            client_session=client_session,
+        )
+
+        # For each referring document, store a descriptor of it.
+        for referring_document in referring_documents:
+            source_class_name = translate_class_uri_into_schema_class_name(
+                schema_view=schema_view,
+                class_uri=referring_document["type"],
+            )
+            document_descriptor = dict(
+                source_collection_name=collection_name,
+                source_class_name=source_class_name,
+                source_document_object_id=referring_document["_id"],
+                source_document_id=referring_document["id"],
+            )
+            referring_document_descriptors.append(document_descriptor)
+
+    return referring_document_descriptors
+
+
 def scan_outgoing_references(
     document: dict,
     schema_view: SchemaView,
@@ -24,6 +116,7 @@ def scan_outgoing_references(
     finder: Finder,
     collection_names: List[str],
     source_collection_name: str,
+    client_session: Optional[ClientSession] = None,
     user_wants_to_locate_misplaced_documents: bool = False,
 ) -> ViolationList:
     r"""
@@ -35,12 +128,16 @@ def scan_outgoing_references(
     :param reference_field_names_by_source_class_name: Dictionary mapping class names to lists of reference field names
     :param references: A `ReferenceList` derived from the schema
     :param finder: A `Finder` bound to the database being scanned
+    :param collection_names: List of collection names that are described by the schema
+    :param source_collection_name: Name of collection in which source document resides (this is included in the
+                                   violation report in an attempt to facilitate investigation of violations)
+    :param client_session: A `pymongo.client_session.ClientSession` instance that, if specified, will be used when
+                           searching for referenced documents. If a transaction happens to be pending on that session,
+                           the scan will effectively happen on the database as it _would_ exist if that transaction
+                           were to be committed.
     :param user_wants_to_locate_misplaced_documents: Whether the user wants the function to proceed to search illegal
                                                      collections after failing to find the referenced document among
                                                      all the legal collections
-    :param source_collection_name: Name of collection in which source document resides (this is included in the
-                                   violation report in an attempt to facilitate investigation of violations)
-    :param collection_names: List of collection names that are described by the schema
     """
 
     # Initialize a list of violations.
@@ -77,7 +174,7 @@ def scan_outgoing_references(
             for target_id in target_ids:
                 name_of_collection_containing_target_document = (
                     finder.check_whether_document_having_id_exists_among_collections(
-                        collection_names=target_collection_names, document_id=target_id
+                        collection_names=target_collection_names, document_id=target_id, client_session=client_session
                     )
                 )
                 if name_of_collection_containing_target_document is None:
@@ -88,7 +185,9 @@ def scan_outgoing_references(
                         names_of_ineligible_collections = list(set(collection_names) - set(target_collection_names))
                         name_of_collection_containing_target_document = (
                             finder.check_whether_document_having_id_exists_among_collections(
-                                collection_names=names_of_ineligible_collections, document_id=target_id
+                                collection_names=names_of_ineligible_collections,
+                                document_id=target_id,
+                                client_session=client_session,
                             )
                         )