nmdc-runtime 2.7.0__py3-none-any.whl → 2.9.0__py3-none-any.whl

nmdc_runtime/site/ops.py

@@ -1,5 +1,6 @@
 import csv
 import json
+import logging
 import mimetypes
 import os
 import subprocess
@@ -9,10 +10,10 @@ from datetime import datetime, timezone
 from io import BytesIO, StringIO
 from pprint import pformat
 from toolz.dicttoolz import keyfilter
-from typing import Tuple
+from typing import Tuple, Set, Union
 from zipfile import ZipFile
 from itertools import chain
-
+from ontology_loader.ontology_load_controller import OntologyLoaderController
 import pandas as pd
 import requests
 
@@ -26,6 +27,7 @@ from dagster import (
     Failure,
     List,
     MetadataValue,
+    Noneable,
     OpExecutionContext,
     Out,
     Output,
@@ -36,12 +38,13 @@ from dagster import (
     Optional,
     Field,
     Permissive,
-    Bool,
+    In,
+    Nothing,
 )
 from gridfs import GridFS
 from linkml_runtime.utils.dictutils import as_simple_dict
 from linkml_runtime.utils.yamlutils import YAMLRoot
-from nmdc_runtime.api.db.mongo import get_mongo_db
+from nmdc_runtime.api.db.mongo import get_mongo_db, validate_json
 from nmdc_runtime.api.core.idgen import generate_one_id
 from nmdc_runtime.api.core.metadata import (
     _validate_changesheet,
@@ -103,7 +106,6 @@ from nmdc_runtime.util import (
     get_names_of_classes_in_effective_range_of_slot,
     pluralize,
     put_object,
-    validate_json,
     specialize_activity_set_docs,
     collection_name_to_class_names,
     class_hierarchy_as_list,
@@ -113,11 +115,14 @@ from nmdc_runtime.util import (
 from nmdc_schema import nmdc
 from nmdc_schema.nmdc import Database as NMDCDatabase
 from pydantic import BaseModel
-from pymongo import InsertOne
+from pymongo import InsertOne, UpdateOne
 from pymongo.database import Database as MongoDatabase
 from starlette import status
 from toolz import assoc, dissoc, get_in, valfilter, identity
 
+# batch size for writing documents to alldocs
+BULK_WRITE_BATCH_SIZE = 2000
+
 
 @op
 def hello(context):
@@ -475,53 +480,6 @@ def get_json_in(context):
     return rv.json()
 
 
-def ensure_data_object_type(docs: Dict[str, list], mdb: MongoDatabase):
-    """Does not ensure ordering of `docs`."""
-
-    if ("data_object_set" not in docs) or len(docs["data_object_set"]) == 0:
-        return docs, 0
-
-    do_docs = docs["data_object_set"]
-
-    class FileTypeEnumBase(BaseModel):
-        name: str
-        description: str
-        filter: str  # JSON-encoded data_object_set mongo collection filter document
-
-    class FileTypeEnum(FileTypeEnumBase):
-        id: str
-
-    temp_collection_name = f"tmp.data_object_set.{ObjectId()}"
-    temp_collection = mdb[temp_collection_name]
-    temp_collection.insert_many(do_docs)
-    temp_collection.create_index("id")
-
-    def fte_matches(fte_filter: str):
-        return [
-            dissoc(d, "_id") for d in mdb.temp_collection.find(json.loads(fte_filter))
-        ]
-
-    do_docs_map = {d["id"]: d for d in do_docs}
-
-    n_docs_with_types_added = 0
-
-    for fte_doc in mdb.file_type_enum.find():
-        fte = FileTypeEnum(**fte_doc)
-        docs_matching = fte_matches(fte.filter)
-        for doc in docs_matching:
-            if "data_object_type" not in doc:
-                do_docs_map[doc["id"]] = assoc(doc, "data_object_type", fte.id)
-                n_docs_with_types_added += 1
-
-    mdb.drop_collection(temp_collection_name)
-    return (
-        assoc(
-            docs, "data_object_set", [dissoc(v, "_id") for v in do_docs_map.values()]
-        ),
-        n_docs_with_types_added,
-    )
-
-
 @op(required_resource_keys={"runtime_api_site_client", "mongo"})
 def perform_mongo_updates(context, json_in):
     mongo = context.resources.mongo
@@ -530,8 +488,6 @@ def perform_mongo_updates(context, json_in):
 
     docs = json_in
     docs, _ = specialize_activity_set_docs(docs)
-    docs, n_docs_with_types_added = ensure_data_object_type(docs, mongo.db)
-    context.log.info(f"added `data_object_type` to {n_docs_with_types_added} docs")
     context.log.debug(f"{docs}")
 
     rv = validate_json(
@@ -600,22 +556,25 @@ def add_output_run_event(context: OpExecutionContext, outputs: List[str]):
         "study_type": str,
         "gold_nmdc_instrument_mapping_file_url": str,
         "include_field_site_info": bool,
+        "enable_biosample_filtering": bool,
     },
     out={
         "study_id": Out(str),
         "study_type": Out(str),
         "gold_nmdc_instrument_mapping_file_url": Out(str),
         "include_field_site_info": Out(bool),
+        "enable_biosample_filtering": Out(bool),
     },
 )
 def get_gold_study_pipeline_inputs(
     context: OpExecutionContext,
-) -> Tuple[str, str, str, bool]:
+) -> Tuple[str, str, str, bool, bool]:
     return (
         context.op_config["study_id"],
         context.op_config["study_type"],
         context.op_config["gold_nmdc_instrument_mapping_file_url"],
         context.op_config["include_field_site_info"],
+        context.op_config["enable_biosample_filtering"],
     )
 
 
@@ -659,6 +618,7 @@ def nmdc_schema_database_from_gold_study(
     analysis_projects: List[Dict[str, Any]],
     gold_nmdc_instrument_map_df: pd.DataFrame,
     include_field_site_info: bool,
+    enable_biosample_filtering: bool,
 ) -> nmdc.Database:
     client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
 
@@ -674,6 +634,7 @@ def nmdc_schema_database_from_gold_study(
         analysis_projects,
         gold_nmdc_instrument_map_df,
         include_field_site_info,
+        enable_biosample_filtering,
         id_minter=id_minter,
     )
     database = translator.get_database()
@@ -1043,18 +1004,249 @@ def site_code_mapping() -> dict:
         )
 
 
-@op(required_resource_keys={"mongo"})
+@op(
+    required_resource_keys={"mongo"},
+    config_schema={
+        "source_ontology": str,
+        "output_directory": Field(Noneable(str), default_value=None, is_required=False),
+        "generate_reports": Field(bool, default_value=True, is_required=False),
+    },
+)
+def load_ontology(context: OpExecutionContext):
+    cfg = context.op_config
+    source_ontology = cfg["source_ontology"]
+    output_directory = cfg.get("output_directory")
+    generate_reports = cfg.get("generate_reports", True)
+
+    if output_directory is None:
+        output_directory = os.path.join(os.getcwd(), "ontology_reports")
+
+    # Redirect Python logging to Dagster context
+    handler = logging.Handler()
+    handler.emit = lambda record: context.log.info(record.getMessage())
+
+    # Get logger from ontology-loader package
+    controller_logger = logging.getLogger("ontology_loader.ontology_load_controller")
+    controller_logger.setLevel(logging.INFO)
+    controller_logger.addHandler(handler)
+
+    context.log.info(f"Running Ontology Loader for ontology: {source_ontology}")
+    loader = OntologyLoaderController(
+        source_ontology=source_ontology,
+        output_directory=output_directory,
+        generate_reports=generate_reports,
+        mongo_client=context.resources.mongo.client,
+        db_name=context.resources.mongo.db.name,
+    )
+
+    loader.run_ontology_loader()
+    context.log.info(f"Ontology load for {source_ontology} completed successfully!")
+
+
+def _add_related_ids_to_alldocs(
+    temp_collection, context, document_reference_ranged_slots_by_type
+) -> None:
+    """
+    Adds {`_inbound`,`_outbound`} fields to each document in the temporary alldocs collection.
+
+    The {`_inbound`,`_outbound`} fields each contain an array of subdocuments, each with fields `id` and `type`.
+    Each subdocument represents a link to any other document that either links to or is linked from
+    the document via document-reference-ranged slots.
+
+    Args:
+        temp_collection: The temporary MongoDB collection to process
+        context: The Dagster execution context for logging
+        document_reference_ranged_slots_by_type: Dictionary mapping document types to their reference-ranged slot names
+
+    Returns:
+        None (modifies the documents in place)
+    """
+
+    context.log.info(
+        "Building relationships and adding `_inbound` and `_outbound` fields..."
+    )
+
+    # document ID -> type (with "nmdc:" prefix preserved)
+    id_to_type_map: Dict[str, str] = {}
+
+    # set of (<referencing document ID>, <slot>, <referenced document ID>) 3-tuples.
+    relationship_triples: Set[Tuple[str, str, str]] = set()
+
+    # Collect relationship triples.
+    for doc in temp_collection.find():
+        doc_id = doc["id"]
+        # Store the full type with prefix intact
+        doc_type = doc["type"]
+        # For looking up reference slots, we still need the type without prefix
+        doc_type_no_prefix = doc_type[5:] if doc_type.startswith("nmdc:") else doc_type
+
+        # Record ID to type mapping - preserve the original type with prefix
+        id_to_type_map[doc_id] = doc_type
+
+        # Find all document references from this document
+        reference_slots = document_reference_ranged_slots_by_type.get(
+            doc_type_no_prefix, []
+        )
+        for slot in reference_slots:
+            if slot in doc:
+                # Handle both single-value and array references
+                refs = doc[slot] if isinstance(doc[slot], list) else [doc[slot]]
+                for ref_doc in temp_collection.find(
+                    {"id": {"$in": refs}}, ["id", "type"]
+                ):
+                    id_to_type_map[ref_doc["id"]] = ref_doc["type"]
+                for ref_id in refs:
+                    relationship_triples.add((doc_id, slot, ref_id))
+
+    context.log.info(
+        f"Found {len(id_to_type_map)} documents, with "
+        f"{len({d for (d, _, _) in relationship_triples})} containing references"
+    )
+
+    # The bifurcation of document-reference-ranged slots as "inbound" and "outbound" is essential
+    # in order to perform graph traversal and collect all entities "related" to a given entity without
+    # recursion "exploding".
+    #
+    # Note: We are hard-coding this "direction" information here in the Runtime
+    #       because the NMDC schema does not currently contain or expose it.
+    #
+    # An "inbound" slot is one for which an entity in the domain "was influenced by" (formally,
+    # <https://www.w3.org/ns/prov#wasInfluencedBy>, with typical CURIE prov:wasInfluencedBy) an entity in the range.
+    inbound_document_reference_ranged_slots = [
+        "collected_from",  # a `nmdc:Biosample` was influenced by the `nmdc:Site` from which it was collected.
+        "has_chromatography_configuration",  # a `nmdc:PlannedProcess` was influenced by its `nmdc:Configuration`.
+        "has_input",  # a `nmdc:PlannedProcess` was influenced by a `nmdc:NamedThing`.
+        "has_mass_spectrometry_configuration",  # a `nmdc:PlannedProcess` was influenced by its `nmdc:Configuration`.
+        "instrument_used",  # a `nmdc:PlannedProcess` was influenced by a used `nmdc:Instrument`.
+        "uses_calibration",  # a `nmdc:PlannedProcess` was influenced by `nmdc:CalibrationInformation`.
+        "was_generated_by",  # prov:wasGeneratedBy rdfs:subPropertyOf prov:wasInfluencedBy.
+        "was_informed_by",  # prov:wasInformedBy rdfs:subPropertyOf prov:wasInfluencedBy.
+    ]
+    # An "outbound" slot is one for which an entity in the domain "influences"
+    # (i.e., [owl:inverseOf prov:wasInfluencedBy]) an entity in the range.
+    outbound_document_reference_ranged_slots = [
+        "associated_studies",  # a `nmdc:Biosample` influences a `nmdc:Study`.
+        "calibration_object",  # `nmdc:CalibrationInformation` generates a `nmdc:DataObject`.
+        "generates_calibration",  # a `nmdc:PlannedProcess` generates `nmdc:CalibrationInformation`.
+        "has_output",  # a `nmdc:PlannedProcess` generates a `nmdc:NamedThing`.
+        "in_manifest",  # a `nmdc:DataObject` becomes associated with `nmdc:Manifest`.
+        "part_of",  # a "contained" `nmdc:NamedThing` influences its "container" `nmdc:NamedThing`,
+    ]
+
+    unique_document_reference_ranged_slot_names = set()
+    for slot_names in document_reference_ranged_slots_by_type.values():
+        for slot_name in slot_names:
+            unique_document_reference_ranged_slot_names.add(slot_name)
+    context.log.info(f"{unique_document_reference_ranged_slot_names=}")
+    if len(inbound_document_reference_ranged_slots) + len(
+        outbound_document_reference_ranged_slots
+    ) != len(unique_document_reference_ranged_slot_names):
+        raise Failure(
+            "Number of detected unique document-reference-ranged slot names does not match "
+            "sum of accounted-for inbound and outbound document-reference-ranged slot names."
+        )
+
+    # Construct, and update documents with, `_incoming` and `_outgoing` field values.
+    #
+    # manage batching of MongoDB `bulk_write` operations
+    bulk_operations, update_count = [], 0
+    for doc_id, slot, ref_id in relationship_triples:
+
+        # Determine in which respective fields to push this relationship
+        # for the subject (doc) and object (ref) of this triple.
+        if slot in inbound_document_reference_ranged_slots:
+            field_for_doc, field_for_ref = "_inbound", "_outbound"
+        elif slot in outbound_document_reference_ranged_slots:
+            field_for_doc, field_for_ref = "_outbound", "_inbound"
+        else:
+            raise Failure(f"Unknown slot {slot} for document {doc_id}")
+
+        updates = [
+            {
+                "filter": {"id": doc_id},
+                "update": {
+                    "$push": {
+                        field_for_doc: {
+                            "id": ref_id,
+                            # TODO existing tests are failing due to `KeyError`s for `id_to_type_map.get[ref_id]` here,
+                            #   which acts as an implicit referential integrity checker (!). Using `.get` with
+                            #   "nmdc:NamedThing" as default in order to (for now) allow such tests to continue to pass.
+                            "type": id_to_type_map.get(ref_id, "nmdc:NamedThing"),
+                        }
+                    }
+                },
+            },
+            {
+                "filter": {"id": ref_id},
+                "update": {
+                    "$push": {
+                        field_for_ref: {"id": doc_id, "type": id_to_type_map[doc_id]}
+                    }
+                },
+            },
+        ]
+        for update in updates:
+            bulk_operations.append(UpdateOne(**update))
+
+        # Execute in batches for efficiency
+        if len(bulk_operations) >= BULK_WRITE_BATCH_SIZE:
+            temp_collection.bulk_write(bulk_operations)
+            update_count += len(bulk_operations)
+            context.log.info(
+                f"Pushed {update_count/(2*len(relationship_triples)):.1%} of updates so far..."
+            )
+            bulk_operations = []
+
+    # Execute any remaining operations
+    if bulk_operations:
+        temp_collection.bulk_write(bulk_operations)
+        update_count += len(bulk_operations)
+
+    context.log.info(f"Pushed {update_count} updates in total")
+
+    context.log.info("Creating {`_inbound`,`_outbound`} indexes...")
+    temp_collection.create_index("_inbound.id")
+    temp_collection.create_index("_outbound.id")
+    # Create compound indexes to ensure index-covered queries
+    temp_collection.create_index([("_inbound.type", 1), ("_inbound.id", 1)])
+    temp_collection.create_index([("_outbound.type", 1), ("_outbound.id", 1)])
+    context.log.info("Successfully created {`_inbound`,`_outbound`} indexes")
+
+
+# Note: Here, we define a so-called "Nothing dependency," which allows us to (in a graph)
+#       pass an argument to the op (in order to specify the order of the ops in the graph)
+#       while also telling Dagster that this op doesn't need the _value_ of that argument.
+#       This is the approach shown on: https://docs.dagster.io/api/dagster/types#dagster.Nothing
+#       Reference: https://docs.dagster.io/guides/build/ops/graphs#defining-nothing-dependencies
+#
+@op(required_resource_keys={"mongo"}, ins={"waits_for": In(dagster_type=Nothing)})
 def materialize_alldocs(context) -> int:
     """
-    This function re-creates the alldocs collection to reflect the current state of the Mongo database.
-    See nmdc-runtime/docs/nb/bulk_validation_referential_integrity_check.ipynb for more details.
+    This function (re)builds the `alldocs` collection to reflect the current state of the MongoDB database by:
+
+    1. Getting all populated schema collection names with an `id` field.
+    2. Create a temporary collection to build the new alldocs collection.
+    3. For each document in schema collections, extract `id`, `type`, and document-reference-ranged slot values.
+    4. Add a special `_type_and_ancestors` field that contains the class hierarchy for the document's type.
+    5. Add special `_inbound` and `_outbound` fields with subdocuments containing ID and type of related entities.
+    6. Add indexes for `id`, relationship fields, and `{_inbound,_outbound}.type`/`.id` compound indexes.
+    7. Finally, atomically replace the existing `alldocs` collection with the temporary one.
+
+    The `alldocs` collection is scheduled to be updated daily via a scheduled job defined as
+    `nmdc_runtime.site.repository.ensure_alldocs_daily`. The collection is also updated as part of various workflows,
+    such as when applying a changesheet or metadata updates (see `nmdc_runtime.site.graphs`).
+
+    The `alldocs` collection is used primarily by API endpoints like `/data_objects/study/{study_id}` and
+    `/workflow_executions/{workflow_execution_id}/related_resources` that need to perform graph traversal to find
+    related documents. It serves as a denormalized view of the database to make these complex queries more efficient.
+
+    The {`_inbound`,`_outbound`} fields enable efficient index-covered queries to find all entities of specific types
+    that are related to a given set of source entities, leveraging the `_type_and_ancestors` field for subtype
+    expansions.
     """
     mdb = context.resources.mongo.db
     schema_view = nmdc_schema_view()
 
-    # batch size for writing documents to alldocs
-    BULK_WRITE_BATCH_SIZE = 2000
-
     # TODO include functional_annotation_agg  for "real-time" ref integrity checking.
     #   For now, production use cases for materialized `alldocs` are limited to `id`-having collections.
     collection_names = populated_schema_collection_names_with_id_field(mdb)
@@ -1100,7 +1292,14 @@ def materialize_alldocs(context) -> int:
         documents_processed_counter = 0
         for doc in mdb[coll_name].find():
             try:
-                doc_type = doc["type"][5:]  # lop off "nmdc:" prefix
+                # Keep the full type with prefix for document
+                doc_type_full = doc["type"]
+                # Remove prefix for slot lookup and ancestor lookup
+                doc_type = (
+                    doc_type_full[5:]
+                    if doc_type_full.startswith("nmdc:")
+                    else doc_type_full
+                )
             except KeyError:
                 raise Exception(
                     f"doc {doc['id']} in collection {coll_name} has no 'type'!"
@@ -1109,13 +1308,21 @@ def materialize_alldocs(context) -> int:
                 doc_type
             ]
             new_doc = keyfilter(lambda slot: slot in slots_to_include, doc)
+
             new_doc["_type_and_ancestors"] = schema_view.class_ancestors(doc_type)
+            # InsertOne is a method on the py-mongo Client class.
+            # Get ancestors without the prefix, but add prefix to each one in the output
+            ancestors = schema_view.class_ancestors(doc_type)
+            new_doc["_type_and_ancestors"] = [
+                "nmdc:" + a if not a.startswith("nmdc:") else a for a in ancestors
+            ]
             write_operations.append(InsertOne(new_doc))
             if len(write_operations) == BULK_WRITE_BATCH_SIZE:
                 _ = temp_alldocs_collection.bulk_write(write_operations, ordered=False)
                 write_operations.clear()
                 documents_processed_counter += BULK_WRITE_BATCH_SIZE
         if len(write_operations) > 0:
+            # here bulk_write is a method on the py-mongo db Client class
             _ = temp_alldocs_collection.bulk_write(write_operations, ordered=False)
             documents_processed_counter += len(write_operations)
         context.log.info(
@@ -1136,10 +1343,20 @@ def materialize_alldocs(context) -> int:
     [temp_alldocs_collection.create_index(slot) for slot in slots_to_index]
     context.log.info(f"created indexes on id, {slots_to_index}.")
 
+    # Add related-ids fields to enable efficient relationship traversal
+    context.log.info("Adding fields for related ids to documents...")
+    _add_related_ids_to_alldocs(
+        temp_alldocs_collection, context, document_reference_ranged_slots
+    )
+
     context.log.info(f"renaming `{temp_alldocs_collection.name}` to `alldocs`...")
     temp_alldocs_collection.rename("alldocs", dropTarget=True)
 
-    return mdb.alldocs.estimated_document_count()
+    n_alldocs_documents = mdb.alldocs.estimated_document_count()
+    context.log.info(
+        f"Rebuilt `alldocs` collection with {n_alldocs_documents} documents."
+    )
+    return n_alldocs_documents
 
 
 @op(config_schema={"nmdc_study_id": str}, required_resource_keys={"mongo"})
@@ -1282,16 +1499,24 @@ def post_submission_portal_biosample_ingest_record_stitching_filename(
     config_schema={
         "nmdc_study_id": str,
         "gold_nmdc_instrument_mapping_file_url": str,
+        "include_field_site_info": bool,
+        "enable_biosample_filtering": bool,
     },
     out={
         "nmdc_study_id": Out(str),
         "gold_nmdc_instrument_mapping_file_url": Out(str),
+        "include_field_site_info": Out(bool),
+        "enable_biosample_filtering": Out(bool),
     },
 )
-def get_database_updater_inputs(context: OpExecutionContext) -> Tuple[str, str]:
+def get_database_updater_inputs(
+    context: OpExecutionContext,
+) -> Tuple[str, str, bool, bool]:
     return (
         context.op_config["nmdc_study_id"],
         context.op_config["gold_nmdc_instrument_mapping_file_url"],
+        context.op_config["include_field_site_info"],
+        context.op_config["enable_biosample_filtering"],
     )
 
 
@@ -1306,6 +1531,8 @@ def generate_data_generation_set_post_biosample_ingest(
     context: OpExecutionContext,
     nmdc_study_id: str,
     gold_nmdc_instrument_map_df: pd.DataFrame,
+    include_field_site_info: bool,
+    enable_biosample_filtering: bool,
 ) -> nmdc.Database:
     runtime_api_user_client: RuntimeApiUserClient = (
         context.resources.runtime_api_user_client
@@ -1321,6 +1548,8 @@ def generate_data_generation_set_post_biosample_ingest(
         gold_api_client,
         nmdc_study_id,
         gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
     database = (
         database_updater.generate_data_generation_set_records_from_gold_api_for_study()
@@ -1340,6 +1569,8 @@ def generate_biosample_set_for_nmdc_study_from_gold(
     context: OpExecutionContext,
     nmdc_study_id: str,
     gold_nmdc_instrument_map_df: pd.DataFrame,
+    include_field_site_info: bool = False,
+    enable_biosample_filtering: bool = False,
 ) -> nmdc.Database:
     runtime_api_user_client: RuntimeApiUserClient = (
         context.resources.runtime_api_user_client
@@ -1355,12 +1586,72 @@ def generate_biosample_set_for_nmdc_study_from_gold(
         gold_api_client,
         nmdc_study_id,
         gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
     database = database_updater.generate_biosample_set_from_gold_api_for_study()
 
     return database
 
 
+@op(
+    required_resource_keys={
+        "runtime_api_user_client",
+        "runtime_api_site_client",
+        "gold_api_client",
+    },
+    out=Out(Any),
+)
+def run_script_to_update_insdc_biosample_identifiers(
+    context: OpExecutionContext,
+    nmdc_study_id: str,
+    gold_nmdc_instrument_map_df: pd.DataFrame,
+    include_field_site_info: bool,
+    enable_biosample_filtering: bool,
+):
+    """Generates a MongoDB update script to add INSDC biosample identifiers to biosamples.
+
+    This op uses the DatabaseUpdater to generate a script that can be used to update biosample
+    records with INSDC identifiers obtained from GOLD.
+
+    Args:
+        context: The execution context
+        nmdc_study_id: The NMDC study ID for which to generate the update script
+        gold_nmdc_instrument_map_df: A dataframe mapping GOLD instrument IDs to NMDC instrument set records
+
+    Returns:
+        A dictionary or list of dictionaries containing the MongoDB update script(s)
+    """
+    runtime_api_user_client: RuntimeApiUserClient = (
+        context.resources.runtime_api_user_client
+    )
+    runtime_api_site_client: RuntimeApiSiteClient = (
+        context.resources.runtime_api_site_client
+    )
+    gold_api_client: GoldApiClient = context.resources.gold_api_client
+
+    database_updater = DatabaseUpdater(
+        runtime_api_user_client,
+        runtime_api_site_client,
+        gold_api_client,
+        nmdc_study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
+    )
+    update_script = database_updater.queries_run_script_to_update_insdc_identifiers()
+
+    if isinstance(update_script, list):
+        total_updates = sum(len(item.get("updates", [])) for item in update_script)
+    else:
+        total_updates = len(update_script.get("updates", []))
+    context.log.info(
+        f"Generated update script for study {nmdc_study_id} with {total_updates} updates"
+    )
+
+    return update_script
+
+
 @op
 def log_database_ids(
     context: OpExecutionContext,
@@ -1382,3 +1673,55 @@ def log_database_ids(
         message += "\n"
     if message:
         context.log.info(message)
+
+
+@op(
+    description="Render free text through the Dagit UI",
+    out=Out(description="Text content rendered through Dagit UI"),
+)
+def render_text(context: OpExecutionContext, text: Any):
+    """
+    Renders content as a Dagster Asset in the Dagit UI.
+
+    This operation creates a Dagster Asset with the provided content, making it
+    visible in the Dagit UI for easy viewing and sharing.
+
+    Args:
+        context: The execution context
+        text: The content to render (can be a string or a dictionary that will be converted to JSON)
+
+    Returns:
+        The same content that was provided as input
+    """
+    # Convert dictionary to formatted JSON string if needed
+    if isinstance(text, dict):
+        import json
+
+        content = json.dumps(text, indent=2)
+        file_extension = "json"
+        hash_text = json.dumps(text, sort_keys=True)[:20]  # For consistent hashing
+    else:
+        content = str(text)  # Convert to string in case it's not already
+        file_extension = "txt"
+        hash_text = content[:20]
+
+    filename = f"rendered_text_{context.run_id}.{file_extension}"
+    file_path = os.path.join(context.instance.storage_directory(), filename)
+
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+    with open(file_path, "w") as f:
+        f.write(content)
+
+    context.log_event(
+        AssetMaterialization(
+            asset_key=f"rendered_text_{hash_from_str(hash_text, 'md5')[:8]}",
+            description="Rendered Content",
+            metadata={
+                "file_path": MetadataValue.path(file_path),
+                "content": MetadataValue.text(content),
+            },
+        )
+    )
+
+    return Output(text)