refscan 0.2.2__tar.gz → 0.2.4__tar.gz

{refscan-0.2.2 → refscan-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: refscan
-Version: 0.2.2
+Version: 0.2.4
 Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
 Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
 Requires-Python: >=3.9,<4.0

{refscan-0.2.2 → refscan-0.2.4}/pyproject.toml

@@ -2,7 +2,7 @@
 
 [tool.poetry]
 name = "refscan"
-version = "v0.2.2"
+version = "v0.2.4"
 description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations"
 authors = []
 readme = "README.md"

{refscan-0.2.2 → refscan-0.2.4}/refscan/grapher.py

@@ -48,6 +48,18 @@ def encode_json_value_as_base64_str(json_value: Union[dict, list]) -> str:
     return encoded_bytes_as_string
 
 
+def is_class_abstract(class_name: str, schema_view: linkml_runtime.SchemaView) -> bool:
+    r"""
+    Returns whether the specified schema class is abstract.
+
+    Note: The `abstract` property can contain `True`, `False`, or `None`.
+
+    Reference: https://linkml.io/linkml/schemas/inheritance.html#abstract-classes-and-slots
+    """
+
+    return schema_view.get_class(class_name).abstract is True
+
+
 def graph(
     schema_view: linkml_runtime.SchemaView,
     subject: Subject,
@@ -106,12 +118,17 @@ def graph(
         source_name = r.source_collection_name if subject == Subject.collection else r.source_class_name
         target_name = r.target_collection_name if subject == Subject.collection else r.target_class_name
 
+        # If the subject of the graph is "class", determine whether the source class is abstract. Same for the target class.
+        # Note: This will allow us to display abstract classes differently from concrete classes in the diagram.
+        is_source_abstract = is_class_abstract(source_name, schema_view) if subject == Subject.class_ else None
+        is_target_abstract = is_class_abstract(target_name, schema_view) if subject == Subject.class_ else None
+
         # Append a node for the source subject if we haven't already done so. Same for the target subject.
         node_ids = [n["data"]["id"] for n in nodes]
         if source_name not in node_ids:
-            nodes.append(dict(data=dict(id=source_name)))
+            nodes.append(dict(data=dict(id=source_name, is_abstract=is_source_abstract)))
         if target_name not in node_ids:
-            nodes.append(dict(data=dict(id=target_name)))
+            nodes.append(dict(data=dict(id=target_name, is_abstract=is_target_abstract)))
 
         # Append an edge for this source subject-to-target subject relationship if we haven't already done so.
         edge_id = f"{source_name}__to__{target_name}"
@@ -147,6 +164,8 @@ def graph(
         dict(
             app_version=get_package_metadata("Version"),
             schema_version=schema_view.schema.version,
+            subject_singular="class" if subject == Subject.class_ else "collection",
+            subject_plural="classes" if subject == Subject.class_ else "collections",
         )
     )
     html_result = html_template.replace("{{ graph_data_json_base64 }}", graph_data_json_base64_str)

refscan-0.2.4/refscan/lib/Finder.py

@@ -0,0 +1,184 @@
+from typing import List, Optional, Dict
+
+from pymongo.database import Database
+from pymongo.client_session import ClientSession
+
+
+class Finder:
+    r"""
+    A class that can be used to find a document in a MongoDB database, in a way that uses past searches to try to
+    speed up future searches.
+    """
+
+    def __init__(self, database: Database):
+        r"""
+        Initializes an instance of the `Finder` class. The instance is bound to the specified MongoDB database.
+        """
+        self.db = database
+
+        # Initialize our cache of names of collections we most recently found referenced document `id`s in.
+        #
+        # Note: This is a queue we will use to keep track of which collections we most recently found referenced
+        #       document `id`s in. This will enable us to _begin_ searching in those same collections for subsequent
+        #       searches. The idea that this will speed things up is based on the assumption that references emanating
+        #       from source documents that we process _temporally close_ to one another, likely point to target
+        #       documents residing _in the same collections_ as one another.
+        #
+        self.cached_collection_names_where_recently_found = []
+        self.cache_size = 2  # we'll cache up to 2 collection names
+
+        # Initialize our cache of document `id` presences/absences by collection.
+        #
+        # Note: This is a dictionary we will use to keep track of the referenced `id`s we find in each collection,
+        #       and the referenced `id`s we fail to find (i.e. confirm do _not_ exist) in each collection. This will
+        #       enable us to avoid querying the database for the same information that we have previously obtained
+        #       by querying the database.
+        #
+        # Note: The top-level items in the dictionary have keys that are collection names and values that are
+        #       dictionaries. Each of those dictionaries has keys that are document `id`s and values that are boolean
+        #       flags, which indicate whether we know that a document having that `id` exists in that collection
+        #       (`True`) or we know that a document having that `id` does _not_ exist in that collection (`False`).
+        #       Here's an example value of this attribute:
+        #       ```
+        #       {
+        #           "biosample_set": {"nmdc:bsm-00-000001": True, "nmdc:bsm-00-000002": False},
+        #           "study_set": {"nmdc:sty-00-000001": True},
+        #       }
+        #       ```
+        #
+        # Note: If this program ever leads to memory consumption (RAM usage) issues in practice, consider
+        #       redesigning this cache and/or allowing the user to disable its use.
+        #
+        self.cached_id_presence_by_collection: Dict[str, Dict[str, bool]] = {}
+
+    def _set_name_of_collection_most_recently_found_in(self, collection_name: str):
+        r"""
+        Helper function that updates our cache of collection names in which target documents were most recently found,
+        so that the specified collection is at the front/start of the list. That will make it so it gets searched first
+        next time.
+        """
+        # Make a concise alias for the instance attribute (both will refer to the same data structure).
+        queue = self.cached_collection_names_where_recently_found
+
+        # Check whether either (a) the queue is empty or (b) this collection name is not at the front of the queue.
+        # Example: [] is empty, or "b_set" is not the first item in ["a_set", "b_set", "c_set"]
+        if len(queue) == 0 or queue[0] != collection_name:
+
+            # If this collection name is already (elsewhere) in the queue, remove it from that position, automatically
+            # updating the list indexes of subsequent list items (so there is no "gap" in the list).
+            # Example: Removing "b_set" from ["a_set", "b_set", "c_set"] → ["a_set", "c_set"]
+            if collection_name in queue:
+                queue.remove(collection_name)
+
+            # Prepend this collection name to the beginning of the queue, dropping any excess collection names.
+            # Example: Prepending ["b_set"] to ["a_set", "c_set"] → ["b_set", "a_set", "c_set"]
+            queue = [collection_name] + queue[0 : self.cache_size - 1]
+
+            # Note: This unnecessary assignment is here only to tell my IDE that `queue` isn't an unused variable.
+            self.cached_collection_names_where_recently_found = queue
+
+    def _optimize_collection_search_order(self, names_of_eligible_collections: List[str]) -> List[str]:
+        r"""
+        Helper function that returns an optimized order in which the specified collections could be searched, in order
+        to take advantage of things the instance "remembers" from recent searches.
+        """
+        ordered_collection_names = names_of_eligible_collections.copy()
+
+        # Iterate over the collection names in the queue, from back (oldest) to front (newest).
+        for cached_collection_name in reversed(self.cached_collection_names_where_recently_found):
+
+            # If this cached collection name is among the eligible ones, move it to the front of the list.
+            # Example: Since "b_set" is in ["a_set", "b_set", "c_set"] → ["b_set, "a_set", "c_set"]
+            if cached_collection_name in names_of_eligible_collections:
+                ordered_collection_names.remove(cached_collection_name)
+                ordered_collection_names.insert(0, cached_collection_name)
+
+        return ordered_collection_names
+
+    def _set_cached_id_presence_in_collection(self, collection_name: str, document_id: str, is_present: bool) -> None:
+        r"""
+        Helper function that updates our cache of document `id` presences/absences, setting the presence flag for the
+        specified document `id` in the specified collection.
+        """
+        if collection_name not in self.cached_id_presence_by_collection:
+            self.cached_id_presence_by_collection[collection_name] = {}  # creates key if it does not exist
+        self.cached_id_presence_by_collection[collection_name][document_id] = is_present
+
+    def _get_cached_id_presence_in_collection(self, collection_name: str, document_id: str) -> Optional[bool]:
+        r"""
+        Helper function that checks our cache of document `id` presences/absences, returning the presence/absence flag,
+        if any, for the specified document `id` in the specified collection.
+        """
+        if (
+            collection_name not in self.cached_id_presence_by_collection
+            or document_id not in self.cached_id_presence_by_collection[collection_name]
+        ):
+            return None
+        else:
+            return self.cached_id_presence_by_collection[collection_name][document_id]
+
+    def check_whether_document_having_id_exists_among_collections(
+        self, document_id: str, collection_names: List[str], client_session: ClientSession = None
+    ) -> Optional[str]:
+        r"""
+        Checks whether any document in any of the specified collections has the specified value in its `id` field.
+        Returns the name of the first collection, if any, containing such a document. If none of the collections
+        contain such a document, the function returns `None`.
+
+        If a `pymongo.client_session.ClientSession` is specified, this function will access the database within
+        the context of that session. If a transaction is pending on that session, this feature can be used
+        to check whether the specified document _would_ exist if that transaction were to be committed.
+
+        References:
+        - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.find_one
+        - https://pymongo.readthedocs.io/en/stable/api/pymongo/client_session.html#transactions
+        """
+
+        # Initialize the name of the containing collection to `None`.
+        name_of_collection_containing_target_document = None
+
+        # Optimize the search order of the collections, based upon the outcomes of recent searches.
+        ordered_collection_names = self._optimize_collection_search_order(collection_names)
+
+        # Search the collections.
+        query_filter = dict(id=document_id)
+        for collection_name in ordered_collection_names:
+
+            # Before querying the database, check our cache of document `id` presences/absences per collection.
+            is_id_present: Optional[bool] = self._get_cached_id_presence_in_collection(
+                collection_name=collection_name, document_id=document_id
+            )
+
+            # If we already know it's _not_ in this collection, stop evaluating this collection and, instead,
+            # proceed to evaluate the next collection.
+            if is_id_present is False:
+                continue  # stop evaluating this collection
+
+            # If we already know it _is_ in this collection, record the collection name, update the cache of names
+            # of collections where a referenced document was recently found, and skip searching additional collections.
+            if is_id_present is True:
+                name_of_collection_containing_target_document = collection_name
+                self._set_name_of_collection_most_recently_found_in(collection_name=collection_name)
+                break
+
+            # If we find it in the collection in the database, record the collection name, update both (a) the cache
+            # of which documents we have found in which collections, and (b) the cache of names of collections where
+            # a referenced document was recently found, and skip searching additional collections.
+            collection = self.db.get_collection(collection_name)
+            if collection.find_one(query_filter, projection=["_id"], session=client_session) is not None:
+                name_of_collection_containing_target_document = collection_name
+
+                # Record that a document having this `id` is _present_ in this collection.
+                self._set_cached_id_presence_in_collection(
+                    collection_name=collection_name, document_id=document_id, is_present=True
+                )
+
+                self._set_name_of_collection_most_recently_found_in(collection_name=collection_name)
+                break
+
+            # If we made it this far, record that a document having this `id` does _not_ exist in this collection.
+            self._set_cached_id_presence_in_collection(
+                collection_name=collection_name, document_id=document_id, is_present=False
+            )
+
+        return name_of_collection_containing_target_document

{refscan-0.2.2 → refscan-0.2.4}/refscan/scanner.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Dict
 
 from pymongo.database import Database
+from pymongo.client_session import ClientSession
 from rich.console import Console
 from linkml_runtime import SchemaView
 
@@ -24,6 +25,7 @@ def scan_outgoing_references(
     finder: Finder,
     collection_names: List[str],
     source_collection_name: str,
+    client_session: ClientSession = None,
     user_wants_to_locate_misplaced_documents: bool = False,
 ) -> ViolationList:
     r"""
@@ -40,6 +42,10 @@ def scan_outgoing_references(
                                                      all the legal collections
     :param source_collection_name: Name of collection in which source document resides (this is included in the
                                    violation report in an attempt to facilitate investigation of violations)
+    :param client_session: A `pymongo.client_session.ClientSession` instance that, if specified, will be used when
+                           searching for referenced documents. If a transaction happens to be pending on that session,
+                           the scan will effectively happen on the database as it _would_ exist if that transaction
+                           were to be committed.
     :param collection_names: List of collection names that are described by the schema
     """
 
@@ -77,7 +83,7 @@ def scan_outgoing_references(
             for target_id in target_ids:
                 name_of_collection_containing_target_document = (
                     finder.check_whether_document_having_id_exists_among_collections(
-                        collection_names=target_collection_names, document_id=target_id
+                        collection_names=target_collection_names, document_id=target_id, client_session=client_session
                     )
                 )
                 if name_of_collection_containing_target_document is None:
@@ -88,7 +94,9 @@ def scan_outgoing_references(
                         names_of_ineligible_collections = list(set(collection_names) - set(target_collection_names))
                         name_of_collection_containing_target_document = (
                             finder.check_whether_document_having_id_exists_among_collections(
-                                collection_names=names_of_ineligible_collections, document_id=target_id
+                                collection_names=names_of_ineligible_collections,
+                                document_id=target_id,
+                                client_session=client_session,
                             )
                         )
 

{refscan-0.2.2 → refscan-0.2.4}/refscan/templates/graph.template.html

@@ -19,6 +19,15 @@
             list-style-type: none;
             margin: 0;
         }
+        .text-button {
+            color: inherit;
+            text-decoration: underline;
+            background: none;
+            border: none;
+            margin: 0;
+            padding: 0;
+            cursor: pointer;
+        }
     </style>
 </head>
 <body>
@@ -82,6 +91,8 @@
             li.textContent = `refscan version ${graphMetadata["app_version"]}`;
             metadataListEl.appendChild(li);
         }
+        let subjectSingular = graphMetadata["subject_singular"];
+        let subjectPlural = graphMetadata["subject_plural"];
         metadataContainerEl.appendChild(metadataListEl);
 
         // Base64-decode the graph data and parse it as JSON.
@@ -174,7 +185,19 @@
                         color: "#0000FF", // text color
                         backgroundColor: "#0000FF"
                     }
-                }
+                },
+                {
+                    selector: "node.isAbstract",
+                    style: {
+                        opacity: 0.5,
+                    }
+                },
+                {
+                    selector: "edge.isAbstract",
+                    style: {
+                        opacity: 0.5,
+                    }
+                },
             ],
         });
 
@@ -214,6 +237,60 @@
                 .removeClass("incomer")
                 .removeClass("inFocus");
         });
+
+        /**
+         * Sets up a text button that can be used to toggle the visibility of
+         * nodes that represent abstract things.
+         */
+        function setupAbstractNodeVisibilityToggler() {
+            // Get a collection of all nodes that represent abstract things.
+            // Note: For conciseness, we refer to them as "abstract nodes" here.
+            // Reference: https://js.cytoscape.org/#eles.filter
+            const abstractNodes = cy.filter(function(el, idx, allEls) {
+                return el.isNode() && el.data("is_abstract");
+            });
+
+            // If there are any abstract nodes, give them (and the edges connected to them)
+            // a specific "stylesheet" class and set up their visibility toggler.
+            if (abstractNodes.length > 0) {
+                abstractNodes.addClass("isAbstract");
+                abstractNodes.connectedEdges().addClass("isAbstract");
+                
+                // Set up a clickable element that can be used to show/hide the abstract nodes.
+                // Note: When a node gets hidden, its connected edges will be automatically hidden also.
+                const strings = {
+                    prefix: `Abstract ${subjectPlural} are `,
+                    visible: "visible",
+                    hidden: "hidden",
+                };
+                let removedNodes = cy.collection(); // initially empty
+                const prefixEl = document.createElement("span");
+                prefixEl.textContent =strings.prefix;
+                const togglerEl = document.createElement("button");
+                togglerEl.classList.add("text-button");
+                togglerEl.title = "Toggle visibility";
+                const togglerTextEl = document.createElement("span");
+                togglerTextEl.textContent = strings.visible;
+                togglerEl.appendChild(togglerTextEl);
+                togglerEl.addEventListener("click", () => {
+                    if (removedNodes.length > 0) {
+                        // Add them back into the graph.
+                        removedNodes.restore();
+                        removedNodes = cy.collection();
+                        togglerTextEl.textContent = strings.visible;
+                    } else {
+                        // Remove them from the graph, but save them in memory.
+                        removedNodes = cy.remove(abstractNodes);
+                        togglerTextEl.textContent = strings.hidden;
+                    }
+                });
+                metadataContainerEl.appendChild(prefixEl);
+                metadataContainerEl.appendChild(togglerEl);
+            }
+        }
+
+        // Set up the abstract node visibility toggler.
+        setupAbstractNodeVisibilityToggler();
     })();
 </script>
 </body>

refscan-0.2.2/refscan/lib/Finder.py

@@ -1,139 +0,0 @@
-from typing import List, Optional, Dict
-
-from pymongo.database import Database
-
-
-class Finder:
-    r"""
-    A class that can be used to find a document in a MongoDB database,
-    in a way that uses past searches to try to speed up future searches.
-    """
-
-    def __init__(self, database: Database):
-        self.db = database
-
-        # Initialize a variable we can use to keep track of the collections in which we most recently found targeted
-        # documents.
-        #
-        # Note: This will enable us to _start_ searching in these same collections next time.
-        #
-        self.names_of_collections_most_recently_found_in = []
-        self.cache_size = 2  # we'll cache up to 2 collection names
-
-        # Initialize a dictionary we can use to keep track of the referenced `id`s we have found in a given collection,
-        # and the referenced `id`s we have confirmed do _not_ exist in a given collection.
-        #
-        # The top-level items have keys that are collection names and values that are dictionaries. The latter
-        # dictionaries have keys that are `id` values and values that are boolean flags. The boolean flags indicate
-        # whether we have found a document having that `id` in that collection; where `True` means we have,
-        # and `False` means we search and _did not_ find such a document in that collection.
-        #
-        # Example:
-        # ```
-        # {
-        #     "biosample_set": {"bsm-00-000001": True, "bsm-00-000002": False},
-        #     "study_set": {"sty-00-000001": True},
-        # }
-        # ```
-        #
-        # Note: This will enable us to _avoid_ searching for the same `id` values in the same collections in the future,
-        #       decreasing execution time (in exchange for memory consumption).
-        #
-        # TODO: Measure memory usage when scanning the NMDC database, reverting this perf. optimization if necessary.
-        #
-        self.cached_id_presence_by_collection: Dict[str, Dict[str, bool]] = {}
-
-    def _set_cached_id_presence_in_collection(self, collection_name: str, document_id: str, is_present: bool) -> None:
-        r"""
-        Helper function that updates our cache of `id` presences/absences, setting the presence flag for the specified
-        document `id` in the specified collection.
-        """
-        if collection_name not in self.cached_id_presence_by_collection:
-            self.cached_id_presence_by_collection[collection_name] = {}
-        self.cached_id_presence_by_collection[collection_name][document_id] = is_present
-
-    def _get_cached_id_presence_in_collection(self, collection_name: str, document_id: str) -> Optional[bool]:
-        r"""
-        Helper function that checks our cache of `id` presences/absences, getting the presence/absence flag, if any,
-        for the specified document `id` in the specified collection.
-        """
-        if (
-            collection_name not in self.cached_id_presence_by_collection
-            or document_id not in self.cached_id_presence_by_collection[collection_name]
-        ):
-            return None
-        else:
-            return self.cached_id_presence_by_collection[collection_name][document_id]
-
-    def check_whether_document_having_id_exists_among_collections(
-        self, document_id: str, collection_names: List[str]
-    ) -> Optional[str]:
-        r"""
-        Checks whether any document in any of the specified collections has the specified value in its `id` field.
-        Returns the name of the first collection, if any, containing such a document. If none of the collections
-        contain such a document, the function returns `None`.
-
-        TODO: This function refers to "cache" as though we are only dealing with one—but we are now dealing with two
-              (see the two instance attributes of this class). The outstanding task here is to update the comments
-              in this function to be more specific about which cache is being referenced in a given statement.
-
-        References:
-        - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.find_one
-        """
-        names_of_collections_to_search = collection_names.copy()
-
-        # Make a concise alias.
-        cache = self.names_of_collections_most_recently_found_in
-
-        # If any cached collection name is among the ones eligible to search, move it to the front of list.
-        for cached_collection_name in reversed(cache):  # goes in reverse, so latest addition ends up in front
-            if cached_collection_name in collection_names:
-                names_of_collections_to_search.remove(cached_collection_name)
-                names_of_collections_to_search.insert(0, cached_collection_name)  # makes it the first item
-
-        # Search the collections in their current order.
-        name_of_collection_containing_target_document = None
-        query_filter = dict(id=document_id)
-        for collection_name in names_of_collections_to_search:
-
-            # Before querying the database, check our in-memory record of `id` presences in this collection.
-            is_id_present: Optional[bool] = self._get_cached_id_presence_in_collection(
-                collection_name=collection_name, document_id=document_id
-            )
-
-            # If we already know it's _not_ in this collection, stop evaluating this collection and, instead,
-            # proceed to evaluate the next collection.
-            if is_id_present is False:
-                continue  # stop evaluating this collection
-
-            # If we _don't know_ whether it is in the collection or not, we can't take any shortcuts. Aw, shucks.
-            if is_id_present is None:
-                pass
-
-            # If it _is_ in this collection—based on either what we already know, or what we find now—cache
-            # the collection name and skip searching additional collections for documents having this `id`.
-            collection = self.db.get_collection(collection_name)
-            if is_id_present or collection.find_one(query_filter, projection=["_id"]) is not None:
-                name_of_collection_containing_target_document = collection_name
-
-                # Put this collection name at the front/start of the cache.
-                if len(cache) == 0 or cache[0] != collection_name:
-                    if collection_name in cache:  # if it's elsewhere in the cache, remove it from the cache
-                        cache.remove(collection_name)
-                    cache = [collection_name] + cache[0 : self.cache_size - 1]  # prepends item, dropping any excess
-                    self.names_of_collections_most_recently_found_in = cache  # persists it to the instance attribute
-
-                # Record that a document having this `id` _exists_ in this collection.
-                # Note: This assignment will be redundant if we got here due to `is_id_present` being `True`.
-                self._set_cached_id_presence_in_collection(
-                    collection_name=collection_name, document_id=document_id, is_present=True
-                )
-
-                break
-
-            # Record that a document having this `id` does _not_ exist in this collection.
-            self._set_cached_id_presence_in_collection(
-                collection_name=collection_name, document_id=document_id, is_present=False
-            )
-
-        return name_of_collection_containing_target_document