refscan 0.2.5__tar.gz → 0.3.2__tar.gz

{refscan-0.2.5 → refscan-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: refscan
-Version: 0.2.5
+Version: 0.3.2
 Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
 Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
 Requires-Python: >=3.9,<4.0
@@ -313,6 +313,16 @@ While editing the tool's source code, you can run the tool as you normally would
 poetry run refscan --help
 ```
 
+### Check types
+
+We use [mypy](https://mypy.readthedocs.io/en/stable/) as the static type checker for `refscan`.
+
+You can perform static type checking by running the following command from the root directory of the repository:
+
+```shell
+poetry run mypy
+```
+
 ### Run tests
 
 We use [pytest](https://docs.pytest.org/en/8.2.x/) as the testing framework for `refscan`.

{refscan-0.2.5 → refscan-0.3.2}/README.md

@@ -292,6 +292,16 @@ While editing the tool's source code, you can run the tool as you normally would
 poetry run refscan --help
 ```
 
+### Check types
+
+We use [mypy](https://mypy.readthedocs.io/en/stable/) as the static type checker for `refscan`.
+
+You can perform static type checking by running the following command from the root directory of the repository:
+
+```shell
+poetry run mypy
+```
+
 ### Run tests
 
 We use [pytest](https://docs.pytest.org/en/8.2.x/) as the testing framework for `refscan`.

{refscan-0.2.5 → refscan-0.3.2}/pyproject.toml

@@ -2,7 +2,7 @@
 
 [tool.poetry]
 name = "refscan"
-version = "v0.2.5"
+version = "v0.3.2"
 description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations"
 authors = []
 readme = "README.md"
@@ -43,6 +43,10 @@ pytest-cov = "^5.0.0"
 # We use `mongomock` to test MongoDB interaction without involving a real MongoDB instance.
 # Docs: https://github.com/mongomock/mongomock
 mongomock = "^4.3.0"
+# We use `mypy` to perform static type checking.
+# Usage: $ poetry run mypy
+# Docs: https://mypy.readthedocs.io/en/stable/
+mypy = "^1.16.0"
 
 [tool.poetry.scripts]
 # Reference: https://python-poetry.org/docs/pyproject#scripts
@@ -61,6 +65,25 @@ addopts = "--doctest-modules --cov=refscan"
 # Reference: https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-format
 line-length = 120
 
+[tool.mypy]
+# Make it so, when someone runs `$ poetry run mypy`, these default CLI options are used.
+# Reference: https://mypy.readthedocs.io/en/stable/config_file.html
+files = "."
+
+[[tool.mypy.overrides]]
+# Silence the following error:
+# > Skipping analyzing "linkml_runtime": module is installed, but missing library stubs or py.typed marker  [import-untyped] 
+#
+# Note: We first tried using the following, but it seemed to have no effect on the behavior of mypy when run via `$ poetry run mypy`.
+#       ```
+#       [mypy-linkml_runtime]
+#       ignore_missing_imports = true
+#       ```
+#       So, we—instead—used the syntax shown on: https://stackoverflow.com/a/74198016
+#
+module = "linkml_runtime"
+ignore_missing_imports = true
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

{refscan-0.2.5 → refscan-0.3.2}/refscan/__init__.py

@@ -12,7 +12,7 @@ def get_package_metadata(key: str) -> str:
     metadata_value = ""
     try:
         package_metadata = metadata("refscan")
-        metadata_value = package_metadata.get(key)
+        metadata_value = package_metadata.get(key, "")
     except PackageNotFoundError:
         pass
     return metadata_value

{refscan-0.2.5 → refscan-0.3.2}/refscan/cli/scan.py

@@ -172,7 +172,6 @@ def scan(
         db=db,
         schema_view=schema_view,
         references=references,
-        collection_names=collection_names,
         names_of_source_collections_to_skip=names_of_source_collections_to_skip,
         user_wants_to_locate_misplaced_documents=user_wants_to_locate_misplaced_documents,
         verbose=verbose,

{refscan-0.2.5 → refscan-0.3.2}/refscan/grapher.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Union
+from typing import Union, List, Dict, Any
 import json
 import base64
 from importlib import resources
@@ -15,6 +15,11 @@ from refscan.lib.helpers import (
     identify_references,
 )
 
+# Define some type aliases that we can reference later, to simplify the latter type hints.
+# TODO: Update these type hints to not use `Any`.
+NodeType = Dict[str, Dict[str, Any]]
+EdgeType = Dict[str, Dict[str, Any]]
+
 
 class Subject(str, Enum):
     r"""The subject (i.e. focal point) of the graph."""
@@ -105,8 +110,8 @@ def graph(
     #
     # Reference: https://js.cytoscape.org/#notation/elements-json
     #
-    nodes = []
-    edges = []
+    nodes: List[NodeType] = []
+    edges: List[EdgeType] = []
     for r in references:
         # Get the source and target names depending upon the subject of the graph.
         source_name = r.source_collection_name if subject == Subject.collection else r.source_class_name

{refscan-0.2.5 → refscan-0.3.2}/refscan/lib/Finder.py

@@ -24,8 +24,8 @@ class Finder:
         #       from source documents that we process _temporally close_ to one another, likely point to target
         #       documents residing _in the same collections_ as one another.
         #
-        self.cached_collection_names_where_recently_found = []
-        self.cache_size = 2  # we'll cache up to 2 collection names
+        self.cached_collection_names_where_recently_found: List[str] = []
+        self.cache_size: int = 2  # we'll cache up to 2 collection names
 
         # Initialize our cache of document `id` presences/absences by collection.
         #

{refscan-0.2.5 → refscan-0.3.2}/refscan/lib/ReferenceList.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Optional
 from pathlib import Path
 from dataclasses import fields, astuple
 from collections import UserList
@@ -18,6 +18,17 @@ class ReferenceList(UserList):
           One thing it does is enable sorting via `sorted(the_list)`.
     """
 
+    # A cached dictionary that maps source class names to a list of the names of
+    # the fields of that class that contain references. Initialized to `None` to
+    # indicate that it hasn't been computed/cached yet.
+    #
+    # Example: {"Study": ["part_of"]}
+    #
+    # TODO: Consider implementing caching via functools's `@cached_property` decorator instead.
+    #       Reference: https://docs.python.org/3/library/functools.html#functools.cached_property
+    #
+    _reference_field_names_by_source_class_name: Optional[Dict[str, List[str]]] = None
+
     def get_source_collection_names(self) -> list[str]:
         """
         Returns the distinct `source_collection_names` values among all references in the list.
@@ -101,11 +112,16 @@ class ReferenceList(UserList):
 
     def get_reference_field_names_by_source_class_name(self) -> Dict[str, List[str]]:
         r"""
-        Returns a dictionary that maps source class names to a list of the
-        names of the fields of that class that contain references.
+        Returns a dictionary that maps source class names to a list of the names of
+        the fields of that class that contain references.
 
         Example: {"Study": ["part_of"]}
         """
+
+        # If this dictionary has already been computed by this class, return that cached dictionary.
+        if isinstance(self._reference_field_names_by_source_class_name, dict):
+            return self._reference_field_names_by_source_class_name
+
         reference_field_names_by_source_class_name: Dict[str, List[str]] = {}
         for reference in self.data:
             source_class_name = reference.source_class_name
@@ -119,6 +135,9 @@ class ReferenceList(UserList):
             if source_field_name not in reference_field_names_by_source_class_name[source_class_name]:
                 reference_field_names_by_source_class_name[source_class_name].append(source_field_name)
 
+        # Cache the dictionary for subsequent invocations of this method.
+        self._reference_field_names_by_source_class_name = reference_field_names_by_source_class_name
+
         return reference_field_names_by_source_class_name
 
     def get_by_target_class_name(self, target_class_name: str) -> "ReferenceList":

{refscan-0.2.5 → refscan-0.3.2}/refscan/lib/helpers.py

@@ -18,6 +18,8 @@ def connect_to_database(mongo_uri: str, database_name: str, verbose: bool = True
     mongo_client: MongoClient = MongoClient(host=mongo_uri, directConnection=True)
 
     with timeout(5):  # if any message exchange takes > 5 seconds, this will raise an exception
+        if mongo_client.address is None:
+            raise ValueError("Failed to determine MongoDB server hostname or port number.")
         (host, port_number) = mongo_client.address
 
         if verbose:

{refscan-0.2.5 → refscan-0.3.2}/refscan/scanner.py

@@ -8,6 +8,7 @@ from linkml_runtime import SchemaView
 from refscan.lib.Finder import Finder
 from refscan.lib.helpers import (
     derive_schema_class_name_from_document,
+    get_collection_names_from_schema,
     translate_class_uri_into_schema_class_name,
     translate_schema_class_name_into_class_uri,
     init_progress_bar,
@@ -51,6 +52,10 @@ def identify_referring_documents(
     # Initialize a list of descriptors of documents that reference the specified document.
     referring_document_descriptors = []
 
+    # If the specified document does not have an `id` field, we know it cannot be referenced.
+    if "id" not in document:
+        return referring_document_descriptors
+
     # Get the specified document's `id` and derive its schema class name.
     document_id = document["id"]
     document_class_name = derive_schema_class_name_from_document(schema_view, document)
@@ -92,6 +97,17 @@ def identify_referring_documents(
 
         # For each referring document, store a descriptor of it.
         for referring_document in referring_documents:
+
+            # Handle the case where the source document does not have an `id`.
+            #
+            # Note: As a reminder, according to the NMDC Schema (as of version v11.8.0), documents in the
+            #       `functional_annotation_agg` collection do not have an `id` field.
+            #       Reference: https://microbiomedata.github.io/nmdc-schema/FunctionalAnnotationAggMember/
+            #
+            source_document_id = referring_document.get("id", None)
+            if not isinstance(source_document_id, str):
+                source_document_id = ""
+
             source_class_name = translate_class_uri_into_schema_class_name(
                 schema_view=schema_view,
                 class_uri=referring_document["type"],
@@ -100,7 +116,7 @@ def identify_referring_documents(
                 source_collection_name=collection_name,
                 source_class_name=source_class_name,
                 source_document_object_id=referring_document["_id"],
-                source_document_id=referring_document["id"],
+                source_document_id=source_document_id,
             )
             referring_document_descriptors.append(document_descriptor)
 
@@ -110,11 +126,8 @@ def identify_referring_documents(
 def scan_outgoing_references(
     document: dict,
     schema_view: SchemaView,
-    # TODO: Cache this dictionary, since it is derived from the schema alone.
-    reference_field_names_by_source_class_name: Dict[str, List[str]],
     references: ReferenceList,
     finder: Finder,
-    collection_names: List[str],
     source_collection_name: str,
     client_session: Optional[ClientSession] = None,
     user_wants_to_locate_misplaced_documents: bool = False,
@@ -125,10 +138,8 @@ def scan_outgoing_references(
 
     :param document: The source document from which the references emanate
     :param schema_view: A SchemaView bound to the schema
-    :param reference_field_names_by_source_class_name: Dictionary mapping class names to lists of reference field names
     :param references: A `ReferenceList` derived from the schema
     :param finder: A `Finder` bound to the database being scanned
-    :param collection_names: List of collection names that are described by the schema
     :param source_collection_name: Name of collection in which source document resides (this is included in the
                                    violation report in an attempt to facilitate investigation of violations)
     :param client_session: A `pymongo.client_session.ClientSession` instance that, if specified, will be used when
@@ -145,14 +156,20 @@ def scan_outgoing_references(
 
     # Get the document's `id` so that we can include it in this script's output.
     source_document_object_id = document["_id"]
-    source_document_id = document["id"] if "id" in document else None
+    source_document_id: Optional[str] = document["id"] if "id" in document else None
 
     # Get the document's schema class name so that we can interpret its fields accordingly.
     source_class_name = derive_schema_class_name_from_document(schema_view, document)
+    if source_class_name is None:
+        raise ValueError(f"Failed to derive schema class name from document having `id`: {source_document_id}")
 
     # Get the names of that class's fields that can contain references.
+    reference_field_names_by_source_class_name = references.get_reference_field_names_by_source_class_name()
     names_of_reference_fields = reference_field_names_by_source_class_name.get(source_class_name, [])
 
+    # Get the names of collections that are described by the schema.
+    collection_names = get_collection_names_from_schema(schema_view=schema_view)
+
     # Check each field that both (a) exists in the document and (b) can contain a reference.
     for field_name in names_of_reference_fields:
         if field_name in document:
@@ -191,6 +208,15 @@ def scan_outgoing_references(
                             )
                         )
 
+                    # Handle the case where the source document does not have an `id`.
+                    #
+                    # Note: As a reminder, according to the NMDC Schema (as of version v11.8.0), documents in the
+                    #       `functional_annotation_agg` collection do not have an `id` field.
+                    #       Reference: https://microbiomedata.github.io/nmdc-schema/FunctionalAnnotationAggMember/
+                    #
+                    if not isinstance(source_document_id, str):
+                        source_document_id = ""
+
                     # Instantiate a `Violation` containing information about this referential integrity violation.
                     violation = Violation(
                         source_collection_name=source_collection_name,
@@ -212,7 +238,6 @@ def scan(
     db: Database,
     schema_view: SchemaView,
     references: ReferenceList,
-    collection_names: List[str],
     names_of_source_collections_to_skip: List[str],
     user_wants_to_locate_misplaced_documents: bool = False,
     console: Console = default_console,
@@ -226,7 +251,6 @@ def scan(
     :param schema_view: A SchemaView bound to the schema with which that database complies
                         (except that it may not be compliant in terms of referential integrity)
     :param references: A `ReferenceList` derived from the schema
-    :param collection_names: List of collection names that are described by the schema
     :param names_of_source_collections_to_skip: List of source collections to skip
     :param user_wants_to_locate_misplaced_documents: Whether the user wants the function to proceed to search illegal
                                                      collections after failing to find the referenced document among
@@ -235,9 +259,6 @@ def scan(
     :param verbose: Whether you want the function to print a higher-than-normal amount of information to the console
     """
 
-    # Get a dictionary that maps source class names to the names of their fields that can contain references.
-    reference_field_names_by_source_class_name = references.get_reference_field_names_by_source_class_name()
-
     # Initialize a progress bar.
     custom_progress = init_progress_bar()
 
@@ -317,10 +338,8 @@ def scan(
                 violations = scan_outgoing_references(
                     document=document,
                     schema_view=schema_view,
-                    reference_field_names_by_source_class_name=reference_field_names_by_source_class_name,
                     references=references,
                     finder=finder,
-                    collection_names=collection_names,
                     source_collection_name=source_collection_name,
                     user_wants_to_locate_misplaced_documents=user_wants_to_locate_misplaced_documents,
                 )