refscan 0.1.0__py3-none-any.whl

refscan/lib/Finder.py

@@ -0,0 +1,54 @@
+from typing import List
+
+from pymongo.database import Database
+
+
+class Finder:
+    r"""
+    A class that can be used to find a document in a MongoDB database,
+    in a way that uses past searches to try to speed up future searches.
+    """
+
+    def __init__(self, database: Database):
+        self.db = database
+
+        # Initialize a variable we can use to keep track of the collection in which we most recently found a document.
+        #
+        # Note: This will enable us to _start_ searching in that same collection (if it's among the eligible ones)
+        #       the next time we are trying to find any document.
+        #
+        self.name_of_most_recently_found_in_collection = None
+
+    def check_whether_document_having_id_exists_among_collections(
+            self,
+            document_id: str,
+            collection_names: List[str]
+    ) -> bool:
+        r"""
+        Checks whether any documents in any of the specified collections
+        have the specified value in their `id` field. Returns `True` if
+        any does; else returns `False`.
+
+        References:
+        - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.find_one
+        """
+        names_of_collections_to_search = collection_names.copy()
+
+        # If our cached collection name is among the ones eligible to search, move it to the front of the list.
+        cached_collection_name = self.name_of_most_recently_found_in_collection  # makes a more concise alias
+        if cached_collection_name in collection_names:
+            names_of_collections_to_search.remove(cached_collection_name)
+            names_of_collections_to_search.insert(0, cached_collection_name)  # makes it the first item
+
+        # Search the collections in their current order.
+        is_found = False
+        query_filter = dict(id=document_id)
+        for collection_name in names_of_collections_to_search:
+            is_found = self.db.get_collection(collection_name).find_one(query_filter, projection=["_id"]) is not None
+
+            # If we found the document, cache the collection name and stop searching.
+            if is_found:
+                self.name_of_most_recently_found_in_collection = collection_name
+                break
+
+        return is_found

refscan/lib/Reference.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, order=True)
+class Reference:
+    """
+    A generic reference to a document in a collection.
+
+    Note: `frozen` means the instances are immutable.
+    Note: `order` means the instances have methods that help with sorting. For example, an `__eq__` method that
+          can be used to compare instances of the class as thought they were tuples of those instances' fields.
+    """
+    source_collection_name: str = field()  # e.g. "study_set"
+    source_class_name: str = field()  # e.g. "Study"
+    source_field_name: str = field()  # e.g. "part_of"
+    target_collection_name: str = field()  # e.g. "study_set" (reminder: a study can be part of another study)
+    target_class_name: str = field()  # e.g. "Study"

refscan/lib/ReferenceList.py

@@ -0,0 +1,146 @@
+from typing import List, Dict
+from pathlib import Path
+from dataclasses import fields, astuple
+from collections import UserList
+from itertools import groupby
+import csv
+
+from rich.table import Table, Column
+
+from refscan.lib.Reference import Reference
+
+
+class ReferenceList(UserList):
+    """
+    A list of references.
+
+    Note: `UserList` is a base class that facilitates the implementation of custom list classes.
+          One thing it does is enable sorting via `sorted(the_list)`.
+    """
+
+    def get_source_collection_names(self) -> list[str]:
+        """
+        Returns the distinct `source_collection_names` values among all references in the list.
+        """
+        distinct_source_collection_names = []
+        for reference in self.data:
+            if reference.source_collection_name not in distinct_source_collection_names:
+                distinct_source_collection_names.append(reference.source_collection_name)
+        return distinct_source_collection_names
+
+    def get_source_field_names_of_source_collection(self, collection_name: str) -> list[str]:
+        """
+        Returns the distinct source field names of the specified source collection.
+        """
+        distinct_source_field_names = []
+        for reference in self.data:
+            if reference.source_collection_name == collection_name:
+                if reference.source_field_name not in distinct_source_field_names:
+                    distinct_source_field_names.append(reference.source_field_name)
+        return distinct_source_field_names
+
+    def get_target_collection_names(
+            self,
+            source_class_name: str,
+            source_field_name: str,
+    ) -> list[str]:
+        """
+        Returns a list of the names of the collections in which a [target] document referenced by the specified field
+        of a [source] document representing an instance of the specified schema class, might exist.
+
+        TODO: Execution spends a lot of time in this function. Consider replacing it with something faster (e.g. a LUT).
+        """
+        target_collection_names = []
+        for reference in self.data:  # note: in a `UserList`, `self.data` refers to the underlying list data structure
+
+            # If this reference's source describes the specified source, record the reference's target collection name.
+            if reference.source_class_name == source_class_name and reference.source_field_name == source_field_name:
+                target_collection_names.append(reference.target_collection_name)
+
+        distinct_target_collection_names = list(set(target_collection_names))
+        return distinct_target_collection_names
+
+    def get_groups(self, field_names: list[str]) -> list[tuple[str, str, str, str, list[str]]]:
+        r"""
+        Returns an iterable of groups, where each group has a distinct combination of values in the specified fields.
+
+        Note: This method can be used to "consolidate" references that have the same source collection name,
+              source field name, and target collection name (i.e. ones that only differ by target class name).
+        """
+
+        def make_group_key(reference: Reference) -> tuple:
+            """Helper function that returns a key that can be used to group references."""
+            values = []
+            for field_name in field_names:
+                if not hasattr(reference, field_name):
+                    raise ValueError(f"No such field: {field_name}")
+                values.append(getattr(reference, field_name))
+            return tuple(values)
+
+        groups = groupby(sorted(self.data), key=make_group_key)
+        return groups
+
+    def dump_to_tsv_file(self, file_path: str | Path) -> None:
+        r"""
+        Helper function that dumps the references to a TSV file at the specified path.
+        """
+        column_names = [field_.name for field_ in fields(Reference)]
+        with open(file_path, "w", newline="") as tsv_file:
+            writer = csv.writer(tsv_file, delimiter="\t")
+            writer.writerow(column_names)  # header row
+            for reference in self.data:
+                writer.writerow(astuple(reference))  # data row
+
+    def get_reference_field_names_by_source_class_name(self) -> Dict[str, List[str]]:
+        r"""
+        Returns a dictionary that maps source class names to a list of the
+        names of the fields of that class that contain references.
+
+        Example: {"Study": ["part_of"]}
+        """
+        reference_field_names_by_source_class_name: Dict[str, List[str]] = {}
+        for reference in self.data:
+            source_class_name = reference.source_class_name
+            source_field_name = reference.source_field_name
+
+            # Initialize a dictionary item for this source class, if one doesn't already exist.
+            if source_class_name not in reference_field_names_by_source_class_name.keys():
+                reference_field_names_by_source_class_name[source_class_name] = []
+
+            # If the source field name isn't already in this class's list, append it.
+            if source_field_name not in reference_field_names_by_source_class_name[source_class_name]:
+                reference_field_names_by_source_class_name[source_class_name].append(source_field_name)
+
+        return reference_field_names_by_source_class_name
+
+    def as_table(self) -> Table:
+        r"""
+        Returns the references as a `rich.Table` instance.
+        """
+
+        # Make the data rows for the table. Data rows that would have had the same
+        # (combination of) the following fields, get consolidated into a single row;
+        # and all the target class names are displayed as a list on that row.
+        fields_to_group_rows_by = ["source_collection_name",
+                                   "source_class_name",
+                                   "source_field_name",
+                                   "target_collection_name"]
+        groups = self.get_groups(fields_to_group_rows_by)
+        data_rows: list[tuple[str, str, str, str, str]] = []
+        for key, group in groups:
+            target_class_names = list(set([ref.target_class_name for ref in group]))  # omit duplicate class names
+            row = (key[0], key[1], key[2], key[3], ", ".join(target_class_names))
+            data_rows.append(row)
+
+        # Initialize the table, then add the data rows to it.
+        table = Table(Column(header="Source collection", footer=f"{len(data_rows)} rows"),
+                      Column(header="Source class"),
+                      Column(header="Source field"),
+                      Column(header="Target collection"),
+                      Column(header="Target class(es)"),
+                      title="References",
+                      show_footer=True)
+        for row in data_rows:
+            table.add_row(*row)
+
+        return table

refscan/lib/Violation.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, order=True)
+class Violation:
+    """
+    A specific reference that lacks integrity.
+    """
+    source_collection_name: str = field()
+    source_field_name: str = field()
+    source_document_object_id: str = field()
+    source_document_id: str = field()
+    target_id: str = field()

refscan/lib/ViolationList.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+from dataclasses import fields, astuple
+from collections import UserList
+import csv
+
+from refscan.lib.Violation import Violation
+
+
+class ViolationList(UserList):
+    """
+    A list of violations.
+    """
+
+    def dump_to_tsv_file(self, file_path: str | Path) -> None:
+        """
+        Helper function that dumps the violations to a TSV file at the specified path.
+        """
+        column_names = [field_.name for field_ in fields(Violation)]
+        with open(file_path, "w", newline="") as tsv_file:
+            writer = csv.writer(tsv_file, delimiter="\t")
+            writer.writerow(column_names)  # header row
+            for violation in self.data:
+                writer.writerow(astuple(violation))  # data row

refscan/lib/constants.py

@@ -0,0 +1,8 @@
+from rich.console import Console
+
+# Instantiate a Rich console for fancy console output.
+# Reference: https://rich.readthedocs.io/en/stable/console.html
+console = Console()
+
+# Note: This is the only schema class name hard-coded into this script.
+DATABASE_CLASS_NAME = "Database"

refscan/lib/helpers.py

@@ -0,0 +1,123 @@
+from typing import Optional
+from functools import cache
+
+from pymongo import MongoClient, timeout
+from linkml_runtime import SchemaView
+from rich.progress import (
+    Progress,
+    TextColumn,
+    MofNCompleteColumn,
+    BarColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn
+)
+
+from refscan.lib.constants import DATABASE_CLASS_NAME, console
+
+
+def connect_to_database(mongo_uri: str, database_name: str, verbose: bool = True) -> MongoClient:
+    """
+    Returns a Mongo client. Raises an exception if the database is not accessible.
+    """
+    mongo_client: MongoClient = MongoClient(host=mongo_uri, directConnection=True)
+
+    with (timeout(5)):  # if any message exchange takes > 5 seconds, this will raise an exception
+        (host, port_number) = mongo_client.address
+
+        if verbose:
+            console.print(f'Connected to MongoDB server: "{host}:{port_number}"')
+
+        # Check whether the database exists on the MongoDB server.
+        if database_name not in mongo_client.list_database_names():
+            raise ValueError(f'Database "{database_name}" not found on the MongoDB server.')
+
+    return mongo_client
+
+
+def get_collection_names_from_schema(
+        schema_view: SchemaView
+) -> list[str]:
+    """
+    Returns the names of the slots of the `Database` class that describe database collections.
+
+    :param schema_view: A `SchemaView` instance
+    """
+    collection_names = []
+
+    for slot_name in schema_view.class_slots(DATABASE_CLASS_NAME):
+        slot_definition = schema_view.induced_slot(slot_name, DATABASE_CLASS_NAME)
+
+        # Filter out any hypothetical (future) slots that don't correspond to a collection (e.g. `db_version`).
+        if slot_definition.multivalued and slot_definition.inlined_as_list:
+            collection_names.append(slot_name)
+
+        # Filter out duplicate names. This is to work around the following issues in the schema:
+        # - https://github.com/microbiomedata/nmdc-schema/issues/1954
+        # - https://github.com/microbiomedata/nmdc-schema/issues/1955
+        collection_names = list(set(collection_names))
+
+    return collection_names
+
+
+@cache  # memoizes the decorated function
+def translate_class_uri_into_schema_class_name(schema_view: SchemaView, class_uri: str) -> Optional[str]:
+    r"""
+    Returns the name of the schema class that has the specified value as its `class_uri`.
+
+    Example `"nmdc:Biosample" (a `class_uri` value) -> "Biosample" (a class name)
+
+    References:
+    - https://linkml.io/linkml/developers/schemaview.html#linkml_runtime.utils.schemaview.SchemaView.all_classes
+    - https://linkml.io/linkml/code/metamodel.html#linkml_runtime.linkml_model.meta.ClassDefinition.class_uri
+    """
+    schema_class_name = None
+    all_class_definitions_in_schema = schema_view.all_classes()
+    for class_name, class_definition in all_class_definitions_in_schema.items():
+        if class_definition.class_uri == class_uri:
+            schema_class_name = class_definition.name
+            break
+    return schema_class_name
+
+
+def derive_schema_class_name_from_document(schema_view: SchemaView, document: dict) -> Optional[str]:
+    r"""
+    Returns the name of the schema class, if any, of which the specified document claims to represent an instance.
+
+    This function is written under the assumption that the document has a `type` field whose value is the `class_uri`
+    belonging to the schema class of which the document represents an instance. Slot definition for such a field:
+    https://github.com/microbiomedata/berkeley-schema-fy24/blob/fc2d9600/src/schema/basic_slots.yaml#L420-L436
+    """
+    schema_class_name = None
+    if "type" in document and isinstance(document["type"], str):
+        class_uri = document["type"]
+        schema_class_name = translate_class_uri_into_schema_class_name(schema_view, class_uri)
+    return schema_class_name
+
+
+def init_progress_bar() -> Progress:
+    r"""
+    Initialize a progress bar that shows the elapsed time, M-of-N completed count, and more.
+
+    Reference: https://rich.readthedocs.io/en/stable/progress.html?highlight=progress#columns
+    """
+    custom_progress = Progress(
+        TextColumn("[progress.description]{task.description}"),
+        TextColumn("[red]{task.fields[num_violations]}[/red] violations in"),
+        MofNCompleteColumn(),
+        TextColumn("source documents"),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        BarColumn(),
+        TimeElapsedColumn(),
+        TextColumn("elapsed"),
+        TimeRemainingColumn(elapsed_when_finished=True),
+        TextColumn("{task.fields[remaining_time_label]}"),
+        console=console,
+        refresh_per_second=1,
+    )
+
+    return custom_progress
+
+
+def get_lowercase_key(key_value_pair: tuple) -> str:
+    r"""Returns the key from a `(key, value)` tuple, in lowercase."""
+    return key_value_pair[0].lower()

refscan/refscan.py

@@ -0,0 +1,308 @@
+from pathlib import Path
+from typing import List, Optional
+from typing_extensions import Annotated
+
+import typer
+import linkml_runtime
+
+from refscan.lib.Finder import Finder
+from refscan.lib.constants import DATABASE_CLASS_NAME, console
+from refscan.lib.helpers import (
+    connect_to_database,
+    get_collection_names_from_schema,
+    derive_schema_class_name_from_document,
+    init_progress_bar,
+    get_lowercase_key,
+)
+from refscan.lib.Reference import Reference
+from refscan.lib.ReferenceList import ReferenceList
+from refscan.lib.Violation import Violation
+from refscan.lib.ViolationList import ViolationList
+
+app = typer.Typer(
+    help="Scans the NMDC MongoDB database for referential integrity violations.",
+    add_completion=False,  # hides the shell completion options from `--help` output
+    rich_markup_mode="markdown",  # enables use of Markdown in docstrings and CLI help
+)
+
+
+@app.command("scan")
+def scan(
+        # Reference: https://typer.tiangolo.com/tutorial/parameter-types/path/
+        schema_file_path: Annotated[Path, typer.Option(
+            "--schema",
+            dir_okay=False,
+            writable=False,
+            readable=True,
+            resolve_path=True,
+            help="Filesystem path at which the YAML file representing the schema is located.",
+        )],
+        database_name: Annotated[str, typer.Option(
+            help="Name of the database.",
+        )] = "nmdc",
+        mongo_uri: Annotated[str, typer.Option(
+            envvar="MONGO_URI",
+            help="Connection string for accessing the MongoDB server. If you have Docker installed, "
+                 "you can spin up a temporary MongoDB server at the default URI by running: "
+                 "`$ docker run --rm --detach -p 27017:27017 mongo`",
+        )] = "mongodb://localhost:27017",
+        verbose: Annotated[bool, typer.Option(
+            help="Show verbose output.",
+        )] = False,
+        # Reference: https://typer.tiangolo.com/tutorial/multiple-values/multiple-options/
+        skip_source_collection: Annotated[Optional[List[str]], typer.Option(
+            "--skip-source-collection", "--skip",
+            help="Name of collection you do not want to search for referring documents. "
+                 "Option can be used multiple times.",
+        )] = None,
+        reference_report_file_path: Annotated[Optional[Path], typer.Option(
+            "--reference-report",
+            dir_okay=False,
+            writable=True,
+            readable=False,
+            resolve_path=True,
+            help="Filesystem path at which you want the program to generate its reference report.",
+        )] = "references.tsv",
+        violation_report_file_path: Annotated[Optional[Path], typer.Option(
+            "--violation-report",
+            dir_okay=False,
+            writable=True,
+            readable=False,
+            resolve_path=True,
+            help="Filesystem path at which you want the program to generate its violation report.",
+        )] = "violations.tsv",
+):
+    """
+    Scans the NMDC MongoDB database for referential integrity violations.
+    """
+    # Instantiate a `linkml_runtime.SchemaView` bound to the specified schema.
+    if verbose:
+        console.print(f"Schema YAML file: {schema_file_path}")
+    schema_view = linkml_runtime.SchemaView(schema_file_path)
+
+    # Show high-level information about the schema and LinkML runtime.
+    console.print(f"Schema version:         {schema_view.schema.version}")
+    console.print(f"LinkML runtime version: {linkml_runtime.__version__}")
+
+    # Make a more self-documenting alias for the CLI option that can be specified multiple times.
+    names_of_source_collections_to_skip: list[str] = [] if skip_source_collection is None else skip_source_collection
+
+    # Get a list of collection names (technically, `Database` slot names) from the schema.
+    # e.g. ["study_set", ...]
+    collection_names = get_collection_names_from_schema(schema_view)
+    console.print(f"Collections described by schema: {len(collection_names)}")
+
+    # For each collection, determine the names of the classes whose instances can be stored in that collection.
+    collection_name_to_class_names = {}  # example: { "study_set": ["Study"] }
+    for collection_name in sorted(collection_names):
+        slot_definition = schema_view.induced_slot(collection_name, DATABASE_CLASS_NAME)
+        name_of_eligible_class = slot_definition.range
+        names_of_eligible_classes = schema_view.class_descendants(name_of_eligible_class)  # includes own class name
+        collection_name_to_class_names[collection_name] = names_of_eligible_classes
+
+    # Initialize the list of references. A reference is effectively a "foreign key" (i.e. a pointer).
+    references = ReferenceList()
+
+    # For each class whose instances can be stored in each collection, determine which of its slots can be a reference.
+    sorted_collection_names_to_class_names = sorted(collection_name_to_class_names.items(), key=get_lowercase_key)
+    for collection_name, class_names in sorted_collection_names_to_class_names:
+        for class_name in class_names:
+            for slot_name in schema_view.class_slots(class_name):
+
+                # Get the slot definition in the context of its use on this particular class.
+                slot_definition = schema_view.induced_slot(slot_name=slot_name, class_name=class_name)
+
+                # Determine the slot's "effective" range, by taking into account its `any_of` constraints (if defined).
+                #
+                # Note: The `any_of` constraints constrain the slot's "effective" range beyond that described by the
+                #       induced slot definition's `range` attribute. `SchemaView` does not seem to provide the result
+                #       of applying those additional constraints, so we do it manually here (if any are defined).
+                #
+                # Reference: https://github.com/orgs/linkml/discussions/2101#discussion-6625646
+                #
+                names_of_eligible_target_classes: list[str] = []
+                if "any_of" in slot_definition and len(slot_definition.any_of) > 0:  # use the `any_of` attribute
+                    for slot_expression in slot_definition.any_of:
+                        if slot_expression.range in schema_view.all_classes():
+                            own_and_descendant_class_names = schema_view.class_descendants(slot_expression.range)
+                            names_of_eligible_target_classes.extend(own_and_descendant_class_names)
+                else:  # use the `range` attribute
+                    if slot_definition.range not in schema_view.all_classes():  # if it's not a class name, abort
+                        continue
+                    else:
+                        # Get the specified class name and the names of all classes that inherit from it.
+                        own_and_descendant_class_names = schema_view.class_descendants(slot_definition.range)
+                        names_of_eligible_target_classes.extend(own_and_descendant_class_names)
+
+                # Remove duplicate class names.
+                names_of_eligible_target_classes = list(set(names_of_eligible_target_classes))
+
+                # For each of those classes whose instances can be stored in any collection, catalog a reference.
+                for name_of_eligible_target_class in names_of_eligible_target_classes:
+                    for target_collection_name, class_names_in_collection in collection_name_to_class_names.items():
+                        if name_of_eligible_target_class in class_names_in_collection:
+                            reference = Reference(source_collection_name=collection_name,
+                                                  source_class_name=class_name,
+                                                  source_field_name=slot_name,
+                                                  target_collection_name=target_collection_name,
+                                                  target_class_name=name_of_eligible_target_class)
+                            references.append(reference)
+
+    console.print(f"Slot-to-ID references described by schema: {len(references)}")
+
+    # Create a reference report in TSV format.
+    console.print(f"Writing reference report: {reference_report_file_path}")
+    references.dump_to_tsv_file(file_path=reference_report_file_path)
+
+    # Display a table of references.
+    if verbose:
+        console.print(references.as_table())
+
+    # Get a dictionary that maps source class names to the names of their fields that can contain references.
+    reference_field_names_by_source_class_name = references.get_reference_field_names_by_source_class_name()
+
+    # Initialize a progress bar.
+    custom_progress = init_progress_bar()
+
+    # Connect to the MongoDB server and verify the database is accessible.
+    mongo_client = connect_to_database(mongo_uri, database_name)
+
+    db = mongo_client.get_database(database_name)
+
+    # Make a finder bound to this database.
+    # Note: A finder is a wrapper around a database that adds some caching that speeds up searches in some situations.
+    finder = Finder(database=db)
+
+    source_collections_and_their_violations: dict[str, ViolationList] = {}
+    with custom_progress as progress:
+
+        # Filter out the collections that the schema says can contain references, but that don't exist in the database.
+        source_collection_names_in_db = []
+        for collection_name in references.get_source_collection_names():
+            if db.get_collection(collection_name) is None:
+                console.print(f"🤷  [orange]Database lacks collection:[/orange] {collection_name}")
+            else:
+                source_collection_names_in_db.append(collection_name)
+
+        # Process each collection, checking for referential integrity violations;
+        # using the reference catalog created earlier to know which collections can
+        # contain "referrers" (documents), which of their slots can contain references (fields),
+        # and which collections can contain the referred-to "referees" (documents).
+        for source_collection_name in sorted(source_collection_names_in_db):
+
+            # If this source collection is one of the ones the user wanted to skip, skip it now.
+            if source_collection_name in names_of_source_collections_to_skip:
+                console.print(f"⚠️  [orange][bold]Skipping source collection:[/bold][/orange] {source_collection_name}")
+                continue
+
+            collection = db.get_collection(source_collection_name)
+
+            # Prepare the query we will use to fetch documents from this collection. The documents we will fetch are
+            # those that have _any_ of the fields (of classes whose instances are allowed to reside in this collection)
+            # that the schema allows to contain a reference to an instance.
+            source_field_names = references.get_source_field_names_of_source_collection(source_collection_name)
+            or_terms = [{field_name: {'$exists': True}} for field_name in source_field_names]
+            query_filter = {'$or': or_terms}
+            if verbose:
+                console.print(f"{query_filter=}")
+
+            # Ensure the fields we fetch include:
+            # - "id" (so we can produce a more user-friendly report later)
+            # - "type" (so we can map the document to a schema class)
+            additional_field_names_for_projection = []
+            if "id" not in source_field_names:
+                additional_field_names_for_projection.append("id")
+            if "type" not in source_field_names:
+                additional_field_names_for_projection.append("type")
+            query_projection = source_field_names + additional_field_names_for_projection
+            if verbose:
+                console.print(f"{query_projection=}")
+
+            # Set up the progress bar for the task of scanning those documents.
+            num_relevant_documents = collection.count_documents(query_filter)
+            task_id = progress.add_task(f"{source_collection_name}",
+                                        total=num_relevant_documents,
+                                        num_violations=0,
+                                        remaining_time_label="remaining")
+
+            # Advance the progress bar by 0 (this makes it so that, even if there are 0 relevant documents, the progress
+            # bar does not continue incrementing its "elapsed time" even after a subsequent task has begun).
+            progress.update(task_id, advance=0)
+
+            # Initialize the violation list for this collection.
+            source_collections_and_their_violations[source_collection_name] = ViolationList()
+
+            # Process each relevant document.
+            for document in collection.find(query_filter, projection=query_projection):
+
+                # Get the document's `id` so that we can include it in this script's output.
+                source_document_object_id = document["_id"]
+                source_document_id = document["id"] if "id" in document else None
+
+                # Get the document's schema class name so that we can interpret its fields accordingly.
+                source_class_name = derive_schema_class_name_from_document(schema_view, document)
+
+                # Get the names of that class's fields that can contain references.
+                names_of_reference_fields = reference_field_names_by_source_class_name.get(source_class_name, [])
+
+                # Check each field that both (a) exists in the document and (b) can contain a reference.
+                for field_name in names_of_reference_fields:
+                    if field_name in document:
+                        # Determine which collections can contain the referenced document, based upon
+                        # the schema class of which this source document is an instance.
+                        target_collection_names = references.get_target_collection_names(
+                            source_class_name=source_class_name,
+                            source_field_name=field_name,
+                        )
+
+                        # Handle both the multi-value (array) and the single-value (scalar) case,
+                        # normalizing the value or values into a list of values in either case.
+                        if type(document[field_name]) is list:
+                            target_ids = document[field_name]
+                        else:
+                            target_id = document[field_name]
+                            target_ids = [target_id]  # makes a one-item list
+
+                        for target_id in target_ids:
+                            target_exists = finder.check_whether_document_having_id_exists_among_collections(
+                                collection_names=target_collection_names,
+                                document_id=target_id
+                            )
+                            if not target_exists:
+                                violation = Violation(source_collection_name=source_collection_name,
+                                                      source_field_name=field_name,
+                                                      source_document_object_id=source_document_object_id,
+                                                      source_document_id=source_document_id,
+                                                      target_id=target_id)
+                                source_collections_and_their_violations[source_collection_name].append(violation)
+                                if verbose:
+                                    console.print(f"Failed to find document having `id` '{target_id}' "
+                                                  f"among collections: {target_collection_names}. "
+                                                  f"{violation=}")
+
+                # Advance the progress bar to account for the current document's contribution to the violations count.
+                progress.update(task_id,
+                                advance=1,
+                                num_violations=len(source_collections_and_their_violations[source_collection_name]))
+
+            # Update the progress bar to indicate the current task is complete.
+            progress.update(task_id, remaining_time_label="done")
+
+    # Close the connection to the MongoDB server.
+    mongo_client.close()
+
+    # Create a violation report in TSV format — for all collections combined.
+    all_violations = ViolationList()
+    for collection_name, violations in sorted(source_collections_and_their_violations.items(), key=get_lowercase_key):
+        console.print(f"Number of violations in {collection_name}: {len(violations)}")
+        all_violations.extend(violations)
+        if verbose:
+            console.print(violations)
+
+    console.print(f"Total violations: {len(all_violations)}")
+    console.print(f"Writing violation report: {violation_report_file_path}")
+    all_violations.dump_to_tsv_file(file_path=violation_report_file_path)
+
+
+if __name__ == "__main__":
+    app()

refscan-0.1.0.dist-info/METADATA

@@ -0,0 +1,68 @@
+Metadata-Version: 2.1
+Name: refscan
+Version: 0.1.0
+Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
+Home-page: https://github.com/microbiomedata/refscan
+Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: linkml-runtime (>=1.7.5,<2.0.0)
+Requires-Dist: pymongo (>=4.7.1,<5.0.0)
+Requires-Dist: rich (>=13.7.1,<14.0.0)
+Requires-Dist: typer (>=0.12.3,<0.13.0)
+Project-URL: Documentation, https://github.com/microbiomedata/refscan
+Project-URL: Repository, https://github.com/microbiomedata/refscan
+Description-Content-Type: text/markdown
+
+# refscan
+
+`refscan` is a command-line tool people can use to scan the NMDC MongoDB database
+for referential integrity violations.
+
+```mermaid
+%% This is the source code of a Mermaid diagram, which GitHub will render as a diagram.
+%% Note: PyPI does not render Mermaid diagrams, and instead displays their source code.
+%%       Reference: https://github.com/pypi/warehouse/issues/13083
+graph LR
+    schema[LinkML<br>schema]
+    database[(MongoDB<br>database)]
+    script[["refscan.py"]]
+    violations["List of<br>violations"]
+    references["List of<br>references"]:::dashed_border
+    schema --> script
+    database --> script
+    script -.-> references
+    script --> violations
+    
+    classDef dashed_border stroke-dasharray: 5 5
+```
+
+## Assumptions
+
+`refscan` was designed under some assumptions about the schema and database, including:
+
+1. Each source document (i.e. document containing references) has a field named `type`, whose value (a string) is the [class_uri](https://linkml.io/linkml/code/metamodel.html#linkml_runtime.linkml_model.meta.ClassDefinition.class_uri) of the schema class of which the document represents an instance. For example, the `type` field of each document in the `study_set` collection has the value `"nmdc:Study"`. 
+
+## Development status
+
+`refscan` is in early development and its author does not recommend anyone use it for anything without reviewing its code first.
+
+## Tips
+
+`refscan` requires the user to specify the path to a schema in YAML format. If you have `curl` installed, you can download a YAML file from GitHub by running the following command (after replacing the `{...}` placeholders and customizing the path):
+
+```shell
+# Download the raw content of https://github.com/{user_or_org}/{repo}/blob/{branch}/path/to/schema.yaml
+curl -o schema.yaml https://raw.githubusercontent.com/{user_or_org}/{repo}/{branch}/path/to/schema.yaml
+```
+
+For example:
+
+```shell
+# Download the raw content of https://github.com/microbiomedata/berkeley-schema-fy24/blob/main/nmdc_schema/nmdc_materialized_patterns.yaml
+curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/berkeley-schema-fy24/main/nmdc_schema/nmdc_materialized_patterns.yaml
+```
+

refscan-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+refscan/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+refscan/lib/Finder.py,sha256=La-fcXB0STwMQkO2x6Ni5Ip2dXpy0GFk8Nhhal8ygGA,2284
+refscan/lib/Reference.py,sha256=KvG7VnhhurT77Q__KvSbsgUhW0ADJsFtCOTdFK6P5GQ,780
+refscan/lib/ReferenceList.py,sha256=dEZC_aoElQyCX7mA5_S2FREzAkfRkMwaUlr0cVniiew,6838
+refscan/lib/Violation.py,sha256=Mruh9Zu6xWwu__AOWPM1B8Gl5qUreT4KRvlCX4mTsoQ,350
+refscan/lib/ViolationList.py,sha256=9_1dQyKR2Ubt0Cob-47UTcYRKnsdHAs9GYCIsDsbeBk,750
+refscan/lib/constants.py,sha256=btLeV-j7370aM8kPRonxQlDz0bDk6eAFar63wCz7bsk,279
+refscan/lib/helpers.py,sha256=_yaWVKjZKVrgvzkza7lrEQV1iM-jEeu5P9gtnYp3y2Y,4865
+refscan/refscan.py,sha256=Q7mUGKXnsLuUtURpCVe4Md266QKf0x3aCHvzXFQF7-g,17153
+refscan-0.1.0.dist-info/METADATA,sha256=AGnM9Tb8XPxNOhhMm5LhPdrg-2uSGG1Yg0NxxVlkASw,3067
+refscan-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+refscan-0.1.0.dist-info/entry_points.txt,sha256=BT0Xy99JIfJVheWTfd57mzeuFHXUikRDvf9ArOmEb7k,47
+refscan-0.1.0.dist-info/RECORD,,

refscan-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any

refscan-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+refscan=refscan.refscan:app
+