mlrun 1.10.0rc24__py3-none-any.whl → 1.10.0rc26__py3-none-any.whl

mlrun/serving/server.py

@@ -33,9 +33,10 @@ from nuclio import Context as NuclioContext
 from nuclio.request import Logger as NuclioLogger
 
 import mlrun
-import mlrun.common.constants
 import mlrun.common.helpers
 import mlrun.common.schemas
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.datastore.datastore_profile as ds_profile
 import mlrun.model_monitoring
 import mlrun.utils
 from mlrun.config import config
@@ -82,7 +83,6 @@ class _StreamContext:
         self.hostname = socket.gethostname()
         self.function_uri = function_uri
         self.output_stream = None
-        stream_uri = None
         log_stream = parameters.get(FileTargetKind.LOG_STREAM, "")
 
         if (enabled or log_stream) and function_uri:
@@ -93,20 +93,16 @@ class _StreamContext:
 
             stream_args = parameters.get("stream_args", {})
 
-            if log_stream == DUMMY_STREAM:
-                # Dummy stream used for testing, see tests/serving/test_serving.py
-                stream_uri = DUMMY_STREAM
-            elif not stream_args.get("mock"):  # if not a mock: `context.is_mock = True`
-                stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
-
             if log_stream:
-                # Update the stream path to the log stream value
-                stream_uri = log_stream.format(project=project)
-                self.output_stream = get_stream_pusher(stream_uri, **stream_args)
+                # Get the output stream from the log stream path
+                stream_path = log_stream.format(project=project)
+                self.output_stream = get_stream_pusher(stream_path, **stream_args)
             else:
                 # Get the output stream from the profile
                 self.output_stream = mlrun.model_monitoring.helpers.get_output_stream(
-                    project=project, mock=stream_args.get("mock", False)
+                    project=project,
+                    profile=parameters.get("stream_profile"),
+                    mock=stream_args.get("mock", False),
                 )
 
 
@@ -184,11 +180,12 @@ class GraphServer(ModelObj):
         self,
         context,
         namespace,
-        resource_cache: ResourceCache = None,
+        resource_cache: Optional[ResourceCache] = None,
         logger=None,
         is_mock=False,
         monitoring_mock=False,
-    ):
+        stream_profile: Optional[ds_profile.DatastoreProfile] = None,
+    ) -> None:
         """for internal use, initialize all steps (recursively)"""
 
         if self.secret_sources:
@@ -203,6 +200,20 @@ class GraphServer(ModelObj):
         context.monitoring_mock = monitoring_mock
         context.root = self.graph
 
+        if is_mock and monitoring_mock:
+            if stream_profile:
+                # Add the user-defined stream profile to the parameters
+                self.parameters["stream_profile"] = stream_profile
+            elif not (
+                self.parameters.get(FileTargetKind.LOG_STREAM)
+                or mlrun.get_secret_or_env(
+                    mm_constants.ProjectSecretKeys.STREAM_PROFILE_NAME
+                )
+            ):
+                # Set a dummy log stream for mocking purposes if there is no direct
+                # user-defined stream profile and no information in the environment
+                self.parameters[FileTargetKind.LOG_STREAM] = DUMMY_STREAM
+
         context.stream = _StreamContext(
             self.track_models, self.parameters, self.function_uri
         )
@@ -406,6 +417,7 @@ def add_monitoring_general_steps(
             "mlrun.serving.system_steps.BackgroundTaskStatus",
             "background_task_status_step",
             model_endpoint_creation_strategy=mlrun.common.schemas.ModelEndpointCreationStrategy.SKIP,
+            full_event=True,
         )
     monitor_flow_step = graph.add_step(
         "storey.Filter",

mlrun/serving/states.py

@@ -39,7 +39,7 @@ import mlrun.common.schemas as schemas
 from mlrun.artifacts.llm_prompt import LLMPromptArtifact, PlaceholderDefaultDict
 from mlrun.artifacts.model import ModelArtifact
 from mlrun.datastore.datastore_profile import (
-    DatastoreProfileKafkaSource,
+    DatastoreProfileKafkaStream,
     DatastoreProfileKafkaTarget,
     DatastoreProfileV3io,
     datastore_profile_read,
@@ -3398,7 +3398,7 @@ def _init_async_objects(context, steps):
                         datastore_profile = datastore_profile_read(stream_path)
                         if isinstance(
                             datastore_profile,
-                            (DatastoreProfileKafkaTarget, DatastoreProfileKafkaSource),
+                            (DatastoreProfileKafkaTarget, DatastoreProfileKafkaStream),
                         ):
                             step._async_object = KafkaStoreyTarget(
                                 path=stream_path,
@@ -3414,7 +3414,7 @@ def _init_async_objects(context, steps):
                         else:
                             raise mlrun.errors.MLRunValueError(
                                 f"Received an unexpected stream profile type: {type(datastore_profile)}\n"
-                                "Expects `DatastoreProfileV3io` or `DatastoreProfileKafkaSource`."
+                                "Expects `DatastoreProfileV3io` or `DatastoreProfileKafkaStream`."
                             )
                     elif stream_path.startswith("kafka://") or kafka_brokers:
                         topic, brokers = parse_kafka_url(stream_path, kafka_brokers)

mlrun/serving/system_steps.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import random
 from copy import copy
 from datetime import timedelta
@@ -25,10 +24,27 @@ import mlrun.artifacts
 import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.feature_store
 import mlrun.serving
+from mlrun.common.model_monitoring.helpers import (
+    get_model_endpoints_creation_task_status,
+)
 from mlrun.common.schemas import MonitoringData
 from mlrun.utils import get_data_from_path, logger
 
 
+class MatchingEndpointsState(mlrun.common.types.StrEnum):
+    all_matched = "all_matched"
+    not_all_matched = "not_all_matched"
+    no_check_needed = "no_check_needed"
+    not_yet_checked = "not_yet_matched"
+
+    @staticmethod
+    def success_states() -> list[str]:
+        return [
+            MatchingEndpointsState.all_matched,
+            MatchingEndpointsState.no_check_needed,
+        ]
+
+
 class MonitoringPreProcessor(storey.MapClass):
     """preprocess step, reconstructs the serving output event body to StreamProcessingEvent schema"""
 
@@ -317,6 +333,9 @@ class BackgroundTaskStatus(storey.MapClass):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self.matching_endpoints = MatchingEndpointsState.not_yet_checked
+        self.graph_model_endpoint_uids: set = set()
+        self.listed_model_endpoint_uids: set = set()
         self.server: mlrun.serving.GraphServer = (
             getattr(self.context, "server", None) if self.context else None
         )
@@ -337,43 +356,47 @@ class BackgroundTaskStatus(storey.MapClass):
                 )
             )
         ):
-            background_task = mlrun.get_run_db().get_project_background_task(
-                self.server.project, self.server.model_endpoint_creation_task_name
-            )
-            self._background_task_check_timestamp = mlrun.utils.now_date()
-            self._log_background_task_state(background_task.status.state)
-            self._background_task_state = background_task.status.state
+            (
+                self._background_task_state,
+                self._background_task_check_timestamp,
+                self.listed_model_endpoint_uids,
+            ) = get_model_endpoints_creation_task_status(self.server)
+        if (
+            self.listed_model_endpoint_uids
+            and self.matching_endpoints == MatchingEndpointsState.not_yet_checked
+        ):
+            if not self.graph_model_endpoint_uids:
+                self.graph_model_endpoint_uids = collect_model_endpoint_uids(
+                    self.server
+                )
+
+            if self.graph_model_endpoint_uids.issubset(self.listed_model_endpoint_uids):
+                self.matching_endpoints = MatchingEndpointsState.all_matched
+        elif self.listed_model_endpoint_uids is None:
+            self.matching_endpoints = MatchingEndpointsState.no_check_needed
 
         if (
             self._background_task_state
             == mlrun.common.schemas.BackgroundTaskState.succeeded
+            and self.matching_endpoints in MatchingEndpointsState.success_states()
         ):
             return event
         else:
             return None
 
-    def _log_background_task_state(
-        self, background_task_state: mlrun.common.schemas.BackgroundTaskState
-    ):
-        logger.info(
-            "Checking model endpoint creation task status",
-            task_name=self.server.model_endpoint_creation_task_name,
-        )
-        if (
-            background_task_state
-            in mlrun.common.schemas.BackgroundTaskState.terminal_states()
-        ):
-            logger.info(
-                f"Model endpoint creation task completed with state {background_task_state}"
-            )
-        else:  # in progress
-            logger.info(
-                f"Model endpoint creation task is still in progress with the current state: "
-                f"{background_task_state}. Events will not be monitored for the next "
-                f"{mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period} seconds",
-                name=self.name,
-                background_task_check_timestamp=self._background_task_check_timestamp.isoformat(),
-            )
+
+def collect_model_endpoint_uids(server: mlrun.serving.GraphServer) -> set[str]:
+    """Collects all model endpoint UIDs from the server's graph steps."""
+    model_endpoint_uids = set()
+    for step in server.graph.steps.values():
+        if hasattr(step, "monitoring_data"):
+            for model in step.monitoring_data.keys():
+                uid = step.monitoring_data[model].get(
+                    mlrun.common.schemas.MonitoringData.MODEL_ENDPOINT_UID
+                )
+                if uid:
+                    model_endpoint_uids.add(uid)
+    return model_endpoint_uids
 
 
 class SamplingStep(storey.MapClass):

mlrun/serving/v2_serving.py

@@ -24,6 +24,9 @@ import mlrun.common.schemas.model_monitoring
 import mlrun.model_monitoring
 from mlrun.utils import logger, now_date
 
+from ..common.model_monitoring.helpers import (
+    get_model_endpoints_creation_task_status,
+)
 from .utils import StepToDict, _extract_input_data, _update_result_body
 
 
@@ -474,22 +477,18 @@ class V2ModelServer(StepToDict):
         ) or getattr(self.context, "server", None)
         if not self.context.is_mock or self.context.monitoring_mock:
             if server.model_endpoint_creation_task_name:
-                background_task = mlrun.get_run_db().get_project_background_task(
-                    server.project, server.model_endpoint_creation_task_name
-                )
-                logger.debug(
-                    "Checking model endpoint creation task status",
-                    task_name=server.model_endpoint_creation_task_name,
+                background_task_state, _, _ = get_model_endpoints_creation_task_status(
+                    server
                 )
                 if (
-                    background_task.status.state
+                    background_task_state
                     in mlrun.common.schemas.BackgroundTaskState.terminal_states()
                 ):
                     logger.debug(
-                        f"Model endpoint creation task completed with state {background_task.status.state}"
+                        f"Model endpoint creation task completed with state {background_task_state}"
                     )
                     if (
-                        background_task.status.state
+                        background_task_state
                         == mlrun.common.schemas.BackgroundTaskState.succeeded
                     ):
                         self._model_logger = (
@@ -504,7 +503,7 @@ class V2ModelServer(StepToDict):
                 else:  # in progress
                     logger.debug(
                         f"Model endpoint creation task is still in progress with the current state: "
-                        f"{background_task.status.state}.",
+                        f"{background_task_state}.",
                         name=self.name,
                     )
             else:

mlrun/utils/helpers.py

@@ -804,7 +804,7 @@ def remove_tag_from_artifact_uri(uri: str) -> Optional[str]:
 
 def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
     """
-    Retrieve the full uri of the item's yaml in the hub.
+    Retrieve the full uri of the function's yaml in the hub.
 
     :param uri: structure: "hub://[<source>/]<item-name>[:<tag>]"
 
@@ -845,7 +845,10 @@ def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
     # hub function directory name are with underscores instead of hyphens
     name = name.replace("-", "_")
     function_suffix = f"{name}/{tag}/src/function.yaml"
-    return indexed_source.source.get_full_uri(function_suffix), is_hub_uri
+    function_type = mlrun.common.schemas.hub.HubSourceType.functions
+    return indexed_source.source.get_full_uri(
+        function_suffix, function_type
+    ), is_hub_uri
 
 
 def gen_md_table(header, rows=None):

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "cc74f73a862905d1eb2adcbb80c8c7fc72550f00",
-  "version": "1.10.0-rc24"
+  "git_commit": "fc51af08faf3c93220de4d619b679cf1950ba5ed",
+  "version": "1.10.0-rc26"
 }

{mlrun-1.10.0rc24.dist-info → mlrun-1.10.0rc26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc24
+Version: 1.10.0rc26
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -43,9 +43,9 @@ Requires-Dist: v3io-frames~=0.10.15; python_version < "3.11"
 Requires-Dist: v3io-frames>=0.13.0; python_version >= "3.11"
 Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
-Requires-Dist: fsspec<2024.7,>=2023.9.2
+Requires-Dist: fsspec<=2025.7.0,>=2025.5.1
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.10.11
+Requires-Dist: storey~=1.10.13
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
@@ -56,14 +56,15 @@ Requires-Dist: mlrun-pipelines-kfp-common~=0.5.8
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
+Requires-Dist: deepdiff<9.0.0,>=8.6.1
 Provides-Extra: s3
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "s3"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "s3"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "s3"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "s3"
 Provides-Extra: azure-blob-storage
 Requires-Dist: msrest~=0.6.21; extra == "azure-blob-storage"
 Requires-Dist: azure-core~=1.24; extra == "azure-blob-storage"
-Requires-Dist: adlfs==2023.9.0; extra == "azure-blob-storage"
+Requires-Dist: adlfs==2024.12.0; extra == "azure-blob-storage"
 Requires-Dist: pyopenssl>=23; extra == "azure-blob-storage"
 Provides-Extra: azure-key-vault
 Requires-Dist: azure-identity~=1.5; extra == "azure-key-vault"
@@ -78,7 +79,7 @@ Requires-Dist: google-cloud-storage==2.14.0; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "google-cloud"
 Requires-Dist: google-cloud==0.34; extra == "google-cloud"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "google-cloud"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "google-cloud"
 Provides-Extra: kafka
 Requires-Dist: kafka-python~=2.1.0; extra == "kafka"
 Requires-Dist: avro~=1.11; extra == "kafka"
@@ -96,8 +97,8 @@ Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "da
 Requires-Dist: dask~=2023.12.1; python_version < "3.11" and extra == "dask"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "dask"
 Provides-Extra: alibaba-oss
-Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
-Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
+Requires-Dist: ossfs==2025.5.0; extra == "alibaba-oss"
+Requires-Dist: oss2==2.18.4; extra == "alibaba-oss"
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
 Provides-Extra: snowflake
@@ -126,7 +127,7 @@ Requires-Dist: pydantic<2,>=1; extra == "api"
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
-Requires-Dist: adlfs==2023.9.0; extra == "all"
+Requires-Dist: adlfs==2024.12.0; extra == "all"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "all"
 Requires-Dist: avro~=1.11; extra == "all"
 Requires-Dist: azure-core~=1.24; extra == "all"
@@ -138,7 +139,7 @@ Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "all"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "all"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "all"
 Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "all"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "all"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "all"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "all"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "all"
 Requires-Dist: google-cloud-storage==2.14.0; extra == "all"
@@ -147,17 +148,17 @@ Requires-Dist: graphviz~=0.20.0; extra == "all"
 Requires-Dist: kafka-python~=2.1.0; extra == "all"
 Requires-Dist: mlflow~=2.22; extra == "all"
 Requires-Dist: msrest~=0.6.21; extra == "all"
-Requires-Dist: oss2==2.18.1; extra == "all"
-Requires-Dist: ossfs==2023.12.0; extra == "all"
+Requires-Dist: oss2==2.18.4; extra == "all"
+Requires-Dist: ossfs==2025.5.0; extra == "all"
 Requires-Dist: plotly~=5.23; extra == "all"
 Requires-Dist: pyopenssl>=23; extra == "all"
 Requires-Dist: redis~=4.3; extra == "all"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "all"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
 Requires-Dist: sqlalchemy~=2.0; extra == "all"
 Requires-Dist: taos-ws-py==0.3.2; extra == "all"
 Provides-Extra: complete
-Requires-Dist: adlfs==2023.9.0; extra == "complete"
+Requires-Dist: adlfs==2024.12.0; extra == "complete"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
 Requires-Dist: avro~=1.11; extra == "complete"
 Requires-Dist: azure-core~=1.24; extra == "complete"
@@ -169,7 +170,7 @@ Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "complete"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "complete"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "complete"
 Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "complete"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "complete"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "complete"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "complete"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "complete"
 Requires-Dist: google-cloud-storage==2.14.0; extra == "complete"
@@ -178,17 +179,17 @@ Requires-Dist: graphviz~=0.20.0; extra == "complete"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete"
 Requires-Dist: mlflow~=2.22; extra == "complete"
 Requires-Dist: msrest~=0.6.21; extra == "complete"
-Requires-Dist: oss2==2.18.1; extra == "complete"
-Requires-Dist: ossfs==2023.12.0; extra == "complete"
+Requires-Dist: oss2==2.18.4; extra == "complete"
+Requires-Dist: ossfs==2025.5.0; extra == "complete"
 Requires-Dist: plotly~=5.23; extra == "complete"
 Requires-Dist: pyopenssl>=23; extra == "complete"
 Requires-Dist: redis~=4.3; extra == "complete"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "complete"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
 Requires-Dist: sqlalchemy~=2.0; extra == "complete"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
 Provides-Extra: complete-api
-Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
+Requires-Dist: adlfs==2024.12.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
 Requires-Dist: aiosmtplib~=3.0; extra == "complete-api"
 Requires-Dist: alembic~=1.14; extra == "complete-api"
@@ -205,7 +206,7 @@ Requires-Dist: databricks-sdk~=0.20.0; extra == "complete-api"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "complete-api"
 Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "complete-api"
 Requires-Dist: fastapi~=0.116.0; extra == "complete-api"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "complete-api"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "complete-api"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "complete-api"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "complete-api"
 Requires-Dist: google-cloud-storage==2.14.0; extra == "complete-api"
@@ -220,15 +221,15 @@ Requires-Dist: mlflow~=2.22; extra == "complete-api"
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
-Requires-Dist: oss2==2.18.1; extra == "complete-api"
-Requires-Dist: ossfs==2023.12.0; extra == "complete-api"
+Requires-Dist: oss2==2.18.4; extra == "complete-api"
+Requires-Dist: ossfs==2025.5.0; extra == "complete-api"
 Requires-Dist: plotly~=5.23; extra == "complete-api"
 Requires-Dist: psycopg2-binary~=2.9; extra == "complete-api"
 Requires-Dist: pydantic<2,>=1; extra == "complete-api"
 Requires-Dist: pymysql~=1.1; extra == "complete-api"
 Requires-Dist: pyopenssl>=23; extra == "complete-api"
 Requires-Dist: redis~=4.3; extra == "complete-api"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "complete-api"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
 Requires-Dist: sqlalchemy-utils~=0.41.2; extra == "complete-api"
 Requires-Dist: sqlalchemy~=2.0; extra == "complete-api"