refscan 0.0.0__py3-none-any.whl

refscan/__init__.py

@@ -0,0 +1,18 @@
+from importlib.metadata import metadata, PackageNotFoundError
+
+
+def get_package_metadata(key: str) -> str:
+    r"""
+    Returns metadata about the installed package.
+
+    References:
+    - https://docs.python.org/3/library/importlib.metadata.html#distribution-metadata
+    - https://github.com/python-poetry/poetry/issues/273#issuecomment-570999678
+    """
+    metadata_value = ""
+    try:
+        package_metadata = metadata("refscan")
+        metadata_value = package_metadata.get(key, "")
+    except PackageNotFoundError:
+        pass
+    return metadata_value

refscan/cli/cli.py

@@ -0,0 +1,27 @@
+import typer
+
+from refscan.cli.version import version
+from refscan.cli.scan import scan
+from refscan.cli.graph import graph
+
+app = typer.Typer(
+    no_args_is_help=True,  # treats the absence of args like the `--help` arg
+    add_completion=False,  # hides the shell completion options from `--help` output
+    rich_markup_mode="markdown",  # enables use of Markdown in docstrings and CLI help
+)
+
+# Add commands—implemented in other modules—to our Typer app.
+# Reference: https://typer.tiangolo.com/tutorial/one-file-per-command/
+#
+# Note: Instead of annotating (with a decorator) a function defined in this module, we decorate a function
+#       that we import from a different module. While both approaches result in the "no-longer-bare" function
+#       being defined in this module, only the latter approach allows for the "bare" function to be defined
+#       in a different module (which I think will facilitate code maintenance).
+#       Reference: https://github.com/fastapi/typer/issues/178#issuecomment-738853400
+#
+app.command()(version)
+app.command()(scan)
+app.command()(graph)
+
+if __name__ == "__main__":
+    app()

refscan/cli/graph.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+from typing import Optional
+from typing_extensions import Annotated
+
+import typer
+import linkml_runtime
+
+from refscan import grapher
+from refscan.lib.constants import console
+from refscan.lib.helpers import print_section_header
+
+
+def graph(
+    # Reference: https://typer.tiangolo.com/tutorial/parameter-types/path/
+    schema_file_path: Annotated[
+        Path,
+        typer.Option(
+            "--schema",
+            dir_okay=False,
+            writable=False,
+            readable=True,
+            resolve_path=True,
+            help="Filesystem path at which the YAML file representing the schema is located.",
+        ),
+    ],
+    graph_file_path: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--graph",
+            dir_okay=False,
+            writable=True,
+            readable=False,
+            resolve_path=True,
+            help="Filesystem path at which you want **refscan** to generate the graph.",
+        ),
+    ] = "graph.html",
+    subject: Annotated[
+        grapher.Subject,
+        typer.Option(
+            "--subject",
+            case_sensitive=False,
+            help="Whether you want each node of the graph to represent a collection or a class.",
+        ),
+    ] = grapher.Subject.collection,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            help="Show verbose output.",
+        ),
+    ] = False,
+):
+    r"""
+    Generate an interactive graph of the references described by a schema.
+    """
+
+    print_section_header(console, text="Reading schema")
+
+    # Instantiate a `linkml_runtime.SchemaView` bound to the specified schema.
+    if verbose:
+        console.print(f"Schema YAML file: {schema_file_path}")
+    schema_view = linkml_runtime.SchemaView(schema_file_path)
+
+    # Show high-level information about the schema.
+    console.print(f"Schema version: {schema_view.schema.version}")
+
+    html_result = grapher.graph(
+        schema_view=schema_view,
+        subject=subject,
+        verbose=verbose,
+    )
+
+    if verbose:
+        console.print(html_result)
+
+    with open(graph_file_path, "w") as f:
+        f.write(html_result)
+
+    console.print(f"Graph generated at: {graph_file_path}")
+    console.print()

refscan/cli/scan.py

@@ -0,0 +1,211 @@
+from pathlib import Path
+from typing import List, Optional, Dict
+from typing_extensions import Annotated
+
+import typer
+import linkml_runtime
+
+from refscan.lib.constants import console
+from refscan.lib.helpers import (
+    connect_to_database,
+    get_collection_name_to_class_names_map,
+    get_collection_names_from_schema,
+    get_lowercase_key,
+    print_section_header,
+    identify_references,
+)
+from refscan.lib.ViolationList import ViolationList
+from refscan.scanner import scan as refscan_scan
+from refscan import get_package_metadata
+
+app_version = get_package_metadata("Version")
+
+
+def scan(
+    # Reference: https://typer.tiangolo.com/tutorial/parameter-types/path/
+    schema_file_path: Annotated[
+        Path,
+        typer.Option(
+            "--schema",
+            dir_okay=False,
+            writable=False,
+            readable=True,
+            resolve_path=True,
+            help="Filesystem path at which the YAML file representing the schema is located.",
+        ),
+    ],
+    database_name: Annotated[
+        str,
+        typer.Option(
+            help="Name of the database.",
+        ),
+    ] = "nmdc",
+    mongo_uri: Annotated[
+        str,
+        typer.Option(
+            envvar="MONGO_URI",
+            help=(
+                "Connection string for accessing the MongoDB server. If you have Docker installed, "
+                "you can spin up a temporary MongoDB server at the default URI by running: "
+                "`$ docker run --rm --detach -p 27017:27017 mongo`"
+            ),
+        ),
+    ] = "mongodb://localhost:27017",
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            help="Show verbose output.",
+        ),
+    ] = False,
+    # Reference: https://typer.tiangolo.com/tutorial/multiple-values/multiple-options/
+    skip_source_collection: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            "--skip-source-collection",
+            "--skip",
+            help=(
+                "Name of collection you do not want to search for referring documents. "
+                "Option can be used multiple times."
+            ),
+        ),
+    ] = None,
+    reference_report_file_path: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--reference-report",
+            dir_okay=False,
+            writable=True,
+            readable=False,
+            resolve_path=True,
+            help="Filesystem path at which you want the program to generate its reference report.",
+        ),
+    ] = "references.tsv",
+    violation_report_file_path: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--violation-report",
+            dir_okay=False,
+            writable=True,
+            readable=False,
+            resolve_path=True,
+            help="Filesystem path at which you want the program to generate its violation report.",
+        ),
+    ] = "violations.tsv",
+    user_wants_to_skip_scan: Annotated[
+        bool,
+        typer.Option(
+            "--no-scan",
+            help="Generate a reference report, but do not scan the database for violations.",
+        ),
+    ] = False,
+    user_wants_to_locate_misplaced_documents: Annotated[
+        bool,
+        typer.Option(
+            "--locate-misplaced-documents",
+            help=(
+                "For each referenced document not found in any of the collections the schema _allows_, "
+                "also search for it in all _other_ collections."
+            ),
+        ),
+    ] = False,
+):
+    r"""
+    Scan the NMDC MongoDB database for referential integrity violations.
+    """
+    # Instantiate a `linkml_runtime.SchemaView` bound to the specified schema.
+    if verbose:
+        console.print(f"Schema YAML file: {schema_file_path}")
+    schema_view = linkml_runtime.SchemaView(schema_file_path)
+
+    # Show high-level information about the application and schema.
+    console.print(f"refscan version: {app_version}")
+    console.print(f"Schema version: {schema_view.schema.version}")
+
+    # Show a header on the console, to tell the user which stage of execution we're entering.
+    print_section_header(console, text="Identifying references")
+
+    # Make a more self-documenting alias for the CLI option that can be specified multiple times.
+    names_of_source_collections_to_skip: list[str] = [] if skip_source_collection is None else skip_source_collection
+
+    # Get a list of collection names (technically, names of `Database` slots that satisfy some criteria)
+    # from the schema; e.g. ["study_set", "biosample_set", ...]
+    collection_names = get_collection_names_from_schema(schema_view)
+    console.print(f"Collections described by schema: {len(collection_names)}")
+
+    # For each collection, determine the names of the classes whose instances can be stored in that collection.
+    collection_name_to_class_names = get_collection_name_to_class_names_map(schema_view=schema_view)
+
+    # Identify the inter-document references that the schema allows to exist.
+    references = identify_references(
+        schema_view=schema_view, collection_name_to_class_names=collection_name_to_class_names
+    )
+    console.print(f"References described by schema: {len(references)}")
+
+    num_collections_having_references = references.count_source_collections()
+    console.print(f"Collections containing references: {num_collections_having_references}")
+    console.print()  # newline
+
+    # Create a reference report in TSV format.
+    console.print(f"Writing reference report: {reference_report_file_path}")
+    references.dump_to_tsv_file(file_path=reference_report_file_path)
+
+    # Display a table of references.
+    if verbose:
+        console.print(references.as_table())
+
+    # If the user opted to skip the scanning step, exit the script.
+    if user_wants_to_skip_scan:
+        console.print()
+        console.print("Skipping scan and exiting.")
+        console.print()
+        raise typer.Exit(code=0)
+
+    print_section_header(console, text="Scanning for violations")
+
+    # Connect to the MongoDB server and verify the database is accessible.
+    mongo_client = connect_to_database(mongo_uri, database_name)
+    db = mongo_client.get_database(database_name)
+
+    # Perform the scan.
+    source_collections_and_their_violations: Dict[str, ViolationList] = refscan_scan(
+        db=db,
+        schema_view=schema_view,
+        references=references,
+        names_of_source_collections_to_skip=names_of_source_collections_to_skip,
+        user_wants_to_locate_misplaced_documents=user_wants_to_locate_misplaced_documents,
+        verbose=verbose,
+        console=console,
+    )
+
+    # Close the connection to the MongoDB server.
+    mongo_client.close()
+
+    print_section_header(console, text="Summarizing results")
+
+    # Make a list of all violations among all collections.
+    all_violations = ViolationList()
+    for collection_name, violations in sorted(source_collections_and_their_violations.items(), key=get_lowercase_key):
+        all_violations.extend(violations)
+
+        # Print a message indicating the number of violations in this collection.
+        num_violations = len(violations)
+        color_name = "white" if num_violations == 0 else "red"
+        console.print(
+            f"Number of violations in {collection_name}: "
+            f"[{color_name} not bold]{num_violations}[/{color_name} not bold]"
+        )
+
+        if verbose:
+            console.print(violations)
+
+    num_all_violations = len(all_violations)
+    color_name = "white" if num_all_violations == 0 else "red"
+    console.print()  # newline
+    console.print(f"Total violations: " f"[{color_name}]{num_all_violations}[/{color_name}]")
+    console.print()  # newline
+
+    # Create a TSV-formatted violation report that lists all violations among all collections.
+    console.print(f"Writing violation report: {violation_report_file_path}")
+    all_violations.dump_to_tsv_file(file_path=violation_report_file_path)
+    console.print()  # newline

refscan/cli/version.py

@@ -0,0 +1,14 @@
+import typer
+
+from refscan.lib.constants import console
+from refscan import get_package_metadata
+
+app_version = get_package_metadata("Version")
+
+
+def version() -> None:
+    r"""
+    Show version number and exit.
+    """
+    console.print(f"[white]{app_version}[/white]")
+    raise typer.Exit()

refscan/grapher.py

@@ -0,0 +1,173 @@
+from enum import Enum
+from typing import Union, List, Dict, Any
+import json
+import base64
+from importlib import resources
+
+import linkml_runtime
+
+from refscan import get_package_metadata
+from refscan.lib.constants import console
+from refscan.lib.helpers import (
+    get_collection_name_to_class_names_map,
+    print_section_header,
+    get_collection_names_from_schema,
+    identify_references,
+)
+
+# Define some type aliases that we can reference later, to simplify the latter type hints.
+# TODO: Update these type hints to not use `Any`.
+NodeType = Dict[str, Dict[str, Any]]
+EdgeType = Dict[str, Dict[str, Any]]
+
+
+class Subject(str, Enum):
+    r"""The subject (i.e. focal point) of the graph."""
+
+    collection = "collection"
+    class_ = "class"
+
+
+def load_template(resource_path: str) -> str:
+    r"""
+    Returns the contents of a template file as a string.
+
+    Note: We do this via `importlib.resources` instead of a regular `open()` so
+          that the path is accurate both when this script is run in a development
+          environment and when this script is run when installed from PyPI,
+          instead of it only being accurate in the former case.
+          Reference: https://docs.python.org/3.9/library/importlib.html#importlib.resources.files
+    """
+    package_name = "refscan"
+    return resources.files(package_name).joinpath(resource_path).read_text(encoding="utf-8")
+
+
+def encode_json_value_as_base64_str(json_value: Union[dict, list]) -> str:
+    r"""Helper function that encodes the specified JSON value as a base64 string."""
+
+    value_as_string = json.dumps(json_value)
+    string_as_bytes = value_as_string.encode("utf-8")
+    encoded_bytes = base64.b64encode(string_as_bytes)
+    encoded_bytes_as_string = encoded_bytes.decode("utf-8")
+    return encoded_bytes_as_string
+
+
+def is_class_abstract(class_name: str, schema_view: linkml_runtime.SchemaView) -> bool:
+    r"""
+    Returns whether the specified schema class is abstract.
+
+    Note: The `abstract` property can contain `True`, `False`, or `None`.
+
+    Reference: https://linkml.io/linkml/schemas/inheritance.html#abstract-classes-and-slots
+    """
+
+    return schema_view.get_class(class_name).abstract is True
+
+
+def graph(
+    schema_view: linkml_runtime.SchemaView,
+    subject: Subject,
+    verbose: bool = False,
+) -> str:
+    r"""
+    Generates an interactive graph (network diagram) of the references described by a schema.
+    """
+
+    # Show a header on the console, to tell the user which stage of execution we're entering.
+    print_section_header(console, text="Identifying references")
+
+    # Get a list of collection names (technically, `Database` slot names) from the schema.
+    # e.g. ["study_set", ...]
+    collection_names = get_collection_names_from_schema(schema_view)
+    console.print(f"Collections described by schema: {len(collection_names)}")
+
+    # For each collection, determine the names of the classes whose instances can be stored in that collection.
+    collection_name_to_class_names = get_collection_name_to_class_names_map(schema_view=schema_view)
+
+    # Identify the inter-document references that the schema allows to exist.
+    references = identify_references(
+        schema_view=schema_view, collection_name_to_class_names=collection_name_to_class_names
+    )
+    console.print(f"References described by schema: {len(references)}")
+
+    console.print(f"References: {len(references)}")
+    if verbose:
+        console.print(references)
+
+    print_section_header(console, text="Generating graph")
+
+    # Generate a list of elements (i.e. nodes and edges) for use by the `cytoscape` JavaScript library.
+    #
+    # Note: Nodes are represented like this (here are two examples):
+    #       ```
+    #       { data: { id: "data_generation_set" } }
+    #       { data: { id: "study_set" } }
+    #       ```
+    #       Edges are represented like this (here is one example):
+    #       ```
+    #       { data: { id: "data_generation_set__to__study_set", source: "data_generation_set", target: "study_set" } }
+    #       ```
+    #
+    # Reference: https://js.cytoscape.org/#notation/elements-json
+    #
+    nodes: List[NodeType] = []
+    edges: List[EdgeType] = []
+    for r in references:
+        # Get the source and target names depending upon the subject of the graph.
+        source_name = r.source_collection_name if subject == Subject.collection else r.source_class_name
+        target_name = r.target_collection_name if subject == Subject.collection else r.target_class_name
+
+        # If the subject of the graph is "class", determine whether the source class is abstract. Same for the target class.
+        # Note: This will allow us to display abstract classes differently from concrete classes in the diagram.
+        is_source_abstract = is_class_abstract(source_name, schema_view) if subject == Subject.class_ else None
+        is_target_abstract = is_class_abstract(target_name, schema_view) if subject == Subject.class_ else None
+
+        # Append a node for the source subject if we haven't already done so. Same for the target subject.
+        node_ids = [n["data"]["id"] for n in nodes]
+        if source_name not in node_ids:
+            nodes.append(dict(data=dict(id=source_name, is_abstract=is_source_abstract)))
+        if target_name not in node_ids:
+            nodes.append(dict(data=dict(id=target_name, is_abstract=is_target_abstract)))
+
+        # Append an edge for this source subject-to-target subject relationship if we haven't already done so.
+        edge_id = f"{source_name}__to__{target_name}"
+        edge_ids = [e["data"]["id"] for e in edges]
+        if edge_id not in edge_ids:
+            edge = dict(data=dict(id=edge_id, source=source_name, target=target_name, source_fields=[]))
+            edges.append(edge)
+
+        # Ensure this edge's `data.source_fields` list accounts for this reference's source field name.
+        edge = next(e for e in edges if e["data"]["id"] == edge_id)  # gets the matching edge
+        if r.source_field_name not in edge["data"]["source_fields"]:  # avoids repeating the field name
+            edge["data"]["source_fields"].append(r.source_field_name)
+
+    # Replace each edge's `source_fields` property with a `label` property containing a comma-delimited string.
+    # Example: if `source_fields` contains ["a", "b", "c"] -> `label` will contain "a, b, c"
+    # Reference: https://docs.python.org/3/library/stdtypes.html#dict.pop
+    for edge in edges:
+        source_fields: list = edge["data"].pop("source_fields", [])  # removes the `source_fields` property
+        edge["data"]["label"] = ", ".join(source_fields)
+
+    console.print(f"Nodes: {len(nodes)}")
+    console.print(f"Edges: {len(edges)}")
+    console.print()
+
+    elements = nodes + edges  # join the lists
+
+    # Load the HTML template file.
+    html_template = load_template(r"templates/graph.template.html")
+
+    # Generate an HTML file (based upon the template) that contains those elements.
+    graph_data_json_base64_str = encode_json_value_as_base64_str(elements)
+    graph_metadata_json_base64_str = encode_json_value_as_base64_str(
+        dict(
+            app_version=get_package_metadata("Version"),
+            schema_version=schema_view.schema.version,
+            subject_singular="class" if subject == Subject.class_ else "collection",
+            subject_plural="classes" if subject == Subject.class_ else "collections",
+        )
+    )
+    html_result = html_template.replace("{{ graph_data_json_base64 }}", graph_data_json_base64_str)
+    html_result = html_result.replace("{{ graph_metadata_json_base64 }}", graph_metadata_json_base64_str)
+
+    return html_result