clarifai 11.7.5__py3-none-any.whl → 11.8.0__py3-none-any.whl

clarifai/client/pipeline.py

@@ -193,6 +193,7 @@ class Pipeline(Lister, BaseClient):
         """
         start_time = time.time()
         seen_logs = set()
+        current_page = 1  # Track current page for log pagination.
 
         while time.time() - start_time < timeout:
             # Get run status
@@ -217,8 +218,8 @@ class Pipeline(Lister, BaseClient):
                     pipeline_run, preserving_proto_field_name=True
                 )
 
-                # Display new log entries
-                self._display_new_logs(run_id, seen_logs)
+                # Display new log entries and update current page
+                current_page = self._display_new_logs(run_id, seen_logs, current_page)
 
                 elapsed_time = time.time() - start_time
                 logger.info(f"Pipeline run monitoring... (elapsed {elapsed_time:.1f}s)")
@@ -276,12 +277,16 @@ class Pipeline(Lister, BaseClient):
         logger.error(f"Pipeline run timed out after {timeout} seconds")
         return {"status": "timeout"}
 
-    def _display_new_logs(self, run_id: str, seen_logs: set):
+    def _display_new_logs(self, run_id: str, seen_logs: set, current_page: int = 1) -> int:
         """Display new log entries for a pipeline version run.
 
         Args:
             run_id (str): The pipeline version run ID.
             seen_logs (set): Set of already seen log entry IDs.
+            current_page (int): The current page to fetch logs from.
+
+        Returns:
+            int: The next page number to fetch from in subsequent calls.
         """
         try:
             logs_request = service_pb2.ListLogEntriesRequest()
@@ -290,7 +295,7 @@ class Pipeline(Lister, BaseClient):
             logs_request.pipeline_version_id = self.pipeline_version_id or ""
             logs_request.pipeline_version_run_id = run_id
             logs_request.log_type = "pipeline.version.run"  # Set required log type
-            logs_request.page = 1
+            logs_request.page = current_page
             logs_request.per_page = 50
 
             logs_response = self.STUB.ListLogEntries(
@@ -298,7 +303,9 @@ class Pipeline(Lister, BaseClient):
             )
 
             if logs_response.status.code == status_code_pb2.StatusCode.SUCCESS:
+                entries_count = 0
                 for log_entry in logs_response.log_entries:
+                    entries_count += 1
                     # Use log entry URL or timestamp as unique identifier
                     log_id = log_entry.url or f"{log_entry.created_at.seconds}_{log_entry.message}"
                     if log_id not in seen_logs:
@@ -312,5 +319,14 @@ class Pipeline(Lister, BaseClient):
                         else:
                             logger.info(log_message)
 
+                # If we got a full page (50 entries), there might be more logs on the next page
+                # If we got fewer than 50 entries, we've reached the end and should stay on current page
+                if entries_count == 50:
+                    return current_page + 1
+                else:
+                    return current_page
+
         except Exception as e:
             logger.debug(f"Error fetching logs: {e}")
+            # Return current page on error to retry the same page next fetch
+            return current_page

clarifai/runners/models/model_builder.py

@@ -10,7 +10,7 @@ import tarfile
 import time
 import webbrowser
 from string import Template
-from typing import Literal
+from typing import Any, Dict, Literal, Optional
 from unittest.mock import MagicMock
 
 import yaml
@@ -1107,7 +1107,7 @@ class ModelBuilder:
                 logger.error(f"Failed to download checkpoints for model {repo_id}")
                 sys.exit(1)
             else:
-                logger.info(f"Downloaded checkpoints for model {repo_id}")
+                logger.info(f"Downloaded checkpoints for model {repo_id} successfully to {path}")
         return path
 
     def _concepts_protos_from_concepts(self, concepts):
@@ -1140,7 +1140,109 @@ class ModelBuilder:
         concepts = config.get('concepts')
         logger.info(f"Updated config.yaml with {len(concepts)} concepts.")
 
-    def get_model_version_proto(self):
+    def _get_git_info(self) -> Optional[Dict[str, Any]]:
+        """
+        Get git repository information for the model path.
+
+        Returns:
+            Dict with git info (url, commit, branch) or None if not a git repository
+        """
+        try:
+            # Check if the folder is within a git repository
+            result = subprocess.run(
+                ['git', 'rev-parse', '--git-dir'],
+                cwd=self.folder,
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+
+            # Get git remote URL
+            remote_result = subprocess.run(
+                ['git', 'config', '--get', 'remote.origin.url'],
+                cwd=self.folder,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+
+            # Get current commit hash
+            commit_result = subprocess.run(
+                ['git', 'rev-parse', 'HEAD'],
+                cwd=self.folder,
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+
+            # Get current branch
+            branch_result = subprocess.run(
+                ['git', 'branch', '--show-current'],
+                cwd=self.folder,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+
+            git_info = {
+                'commit': commit_result.stdout.strip(),
+                'branch': branch_result.stdout.strip()
+                if branch_result.returncode == 0
+                else 'HEAD',
+            }
+
+            if remote_result.returncode == 0:
+                git_info['url'] = remote_result.stdout.strip()
+
+            return git_info
+
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            # Not a git repository or git not available
+            return None
+
+    def _check_git_status_and_prompt(self) -> bool:
+        """
+        Check for uncommitted changes in git repository within the model path and prompt user.
+
+        Returns:
+            True if should continue with upload, False if should abort
+        """
+        try:
+            # Check for uncommitted changes within the model path only
+            status_result = subprocess.run(
+                ['git', 'status', '--porcelain', '.'],
+                cwd=self.folder,
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+
+            if status_result.stdout.strip():
+                logger.warning("Uncommitted changes detected in model path:")
+                logger.warning(status_result.stdout)
+
+                response = input(
+                    "\nDo you want to continue upload with uncommitted changes? (y/N): "
+                )
+                return response.lower() in ['y', 'yes']
+            else:
+                logger.info("Model path has no uncommitted changes.")
+                return True
+
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            # Error checking git status, but we already know it's a git repo
+            logger.warning("Could not check git status, continuing with upload.")
+            return True
+
+    def get_model_version_proto(self, git_info: Optional[Dict[str, Any]] = None):
+        """
+        Create a ModelVersion protobuf message for the model.
+        Args:
+          git_info (Optional[Dict[str, Any]]): Git repository information to include in metadata.
+        Returns:
+          resources_pb2.ModelVersion: The ModelVersion protobuf message.
+        """
+
         signatures = self.get_method_signatures()
         model_version_proto = resources_pb2.ModelVersion(
             pretrained_model_config=resources_pb2.PretrainedModelConfig(),
@@ -1148,6 +1250,14 @@ class ModelBuilder:
             method_signatures=signatures,
         )
 
+        # Add git information to metadata if available
+        if git_info:
+            from google.protobuf.struct_pb2 import Struct
+
+            metadata_struct = Struct()
+            metadata_struct.update({'git_registry': git_info})
+            model_version_proto.metadata.CopyFrom(metadata_struct)
+
         model_type_id = self.config.get('model').get('model_type_id')
         if model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE:
             if 'concepts' in self.config:
@@ -1175,7 +1285,7 @@ class ModelBuilder:
                 )
         return model_version_proto
 
-    def upload_model_version(self):
+    def upload_model_version(self, git_info=None):
         file_path = f"{self.folder}.tar.gz"
         logger.debug(f"Will tar it into file: {file_path}")
 
@@ -1208,7 +1318,7 @@ class ModelBuilder:
                 )
                 return
 
-        model_version_proto = self.get_model_version_proto()
+        model_version_proto = self.get_model_version_proto(git_info)
 
         def filter_func(tarinfo):
             name = tarinfo.name
@@ -1228,7 +1338,7 @@ class ModelBuilder:
         if when != "upload" and self.config.get("checkpoints"):
             # Get the checkpoint size to add to the storage request.
             # First check for the env variable, then try querying huggingface. If all else fails, use the default.
-            checkpoint_size = os.environ.get('CHECKPOINT_SIZE_BYTES', 0)
+            checkpoint_size = int(os.environ.get('CHECKPOINT_SIZE_BYTES', 0))
             if not checkpoint_size:
                 _, repo_id, _, _, _, _ = self._validate_config_checkpoints()
                 checkpoint_size = HuggingFaceLoader.get_huggingface_checkpoint_total_size(repo_id)
@@ -1332,13 +1442,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
         )
         return result
 
-    def get_model_build_logs(self):
+    def get_model_build_logs(self, current_page=1):
         logs_request = service_pb2.ListLogEntriesRequest(
             log_type="builder",
             user_app_id=self.client.user_app_id,
             model_id=self.model_proto.id,
             model_version_id=self.model_version_id,
-            page=1,
+            page=current_page,
             per_page=50,
         )
         response = self.client.STUB.ListLogEntries(logs_request)
@@ -1347,6 +1457,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     def monitor_model_build(self):
         st = time.time()
         seen_logs = set()  # To avoid duplicate log messages
+        current_page = 1  # Track current page for log pagination
         while True:
             resp = self.client.STUB.GetModelVersion(
                 service_pb2.GetModelVersionRequest(
@@ -1357,8 +1468,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
             )
 
             status_code = resp.model_version.status.code
-            logs = self.get_model_build_logs()
+            logs = self.get_model_build_logs(current_page)
+            entries_count = 0
             for log_entry in logs.log_entries:
+                entries_count += 1
                 if log_entry.url not in seen_logs:
                     seen_logs.add(log_entry.url)
                     log_entry_msg = re.sub(
@@ -1367,6 +1480,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
                         log_entry.message.strip(),
                     )
                     logger.info(log_entry_msg)
+
+            # If we got a full page (50 entries), there might be more logs on the next page
+            # If we got fewer than 50 entries, we've reached the end and should stay on current page
+            if entries_count == 50:
+                current_page += 1
+            # else: stay on current_page
             if status_code == status_code_pb2.MODEL_BUILDING:
                 print(
                     f"Model is building... (elapsed {time.time() - st:.1f}s)", end='\r', flush=True
@@ -1414,8 +1533,20 @@ def upload_model(folder, stage, skip_dockerfile, pat=None, base_url=None):
             f"New model will be created at {builder.model_ui_url} with it's first version."
         )
 
+    # Check for git repository information
+    git_info = builder._get_git_info()
+    if git_info:
+        logger.info(f"Detected git repository: {git_info.get('url', 'local repository')}")
+        logger.info(f"Current commit: {git_info['commit']}")
+        logger.info(f"Current branch: {git_info['branch']}")
+
+        # Check for uncommitted changes and prompt user
+        if not builder._check_git_status_and_prompt():
+            logger.info("Upload cancelled by user due to uncommitted changes.")
+            return
     input("Press Enter to continue...")
-    model_version = builder.upload_model_version()
+
+    model_version = builder.upload_model_version(git_info)
 
     # Ask user if they want to deploy the model
     if model_version is not None:  # if it comes back None then it failed.

clarifai/runners/models/model_runner.py

@@ -9,7 +9,7 @@ from clarifai_protocol.utils.health import HealthProbeRequestHandler
 from clarifai.client.auth.helper import ClarifaiAuthHelper
 from clarifai.utils.constants import STATUS_FAIL, STATUS_MIXED, STATUS_OK, STATUS_UNKNOWN
 from clarifai.utils.logging import get_req_id_from_context, logger
-from clarifai.utils.secrets import inject_secrets
+from clarifai.utils.secrets import inject_secrets, req_secrets_context
 
 from ..utils.url_fetcher import ensure_urls_downloaded
 from .model_class import ModelClass
@@ -126,7 +126,10 @@ class ModelRunner(BaseRunner, HealthProbeRequestHandler):
         if method_name == '_GET_SIGNATURES':
             logging = False
 
-        resp = self.model.predict_wrapper(request)
+        # Use req_secrets_context to temporarily set request-type secrets as environment variables
+        with req_secrets_context(request):
+            resp = self.model.predict_wrapper(request)
+
         # if we have any non-successful code already it's an error we can return.
         if (
             resp.status.code != status_code_pb2.SUCCESS
@@ -185,45 +188,49 @@ class ModelRunner(BaseRunner, HealthProbeRequestHandler):
         status_str = STATUS_UNKNOWN
         endpoint = "POST /v2/.../outputs/generate"
 
-        for resp in self.model.generate_wrapper(request):
-            # if we have any non-successful code already it's an error we can return.
-            if (
-                resp.status.code != status_code_pb2.SUCCESS
-                and resp.status.code != status_code_pb2.ZERO
-            ):
-                status_str = f"{resp.status.code} ERROR"
-                duration_ms = (time.time() - start_time) * 1000
-                logger.info(f"{endpoint} | {status_str} | {duration_ms:.2f}ms | req_id={req_id}")
-                yield service_pb2.RunnerItemOutput(multi_output_response=resp)
-                continue
-            successes = []
-            for output in resp.outputs:
-                if not output.HasField('status') or not output.status.code:
-                    raise Exception(
-                        "Output must have a status code, please check the model implementation."
+        # Use req_secrets_context to temporarily set request-type secrets as environment variables
+        with req_secrets_context(request):
+            for resp in self.model.generate_wrapper(request):
+                # if we have any non-successful code already it's an error we can return.
+                if (
+                    resp.status.code != status_code_pb2.SUCCESS
+                    and resp.status.code != status_code_pb2.ZERO
+                ):
+                    status_str = f"{resp.status.code} ERROR"
+                    duration_ms = (time.time() - start_time) * 1000
+                    logger.info(
+                        f"{endpoint} | {status_str} | {duration_ms:.2f}ms | req_id={req_id}"
                     )
-                successes.append(output.status.code == status_code_pb2.SUCCESS)
-            if all(successes):
-                status = status_pb2.Status(
-                    code=status_code_pb2.SUCCESS,
-                    description="Success",
-                )
-                status_str = STATUS_OK
-            elif any(successes):
-                status = status_pb2.Status(
-                    code=status_code_pb2.MIXED_STATUS,
-                    description="Mixed Status",
-                )
-                status_str = STATUS_MIXED
-            else:
-                status = status_pb2.Status(
-                    code=status_code_pb2.FAILURE,
-                    description="Failed",
-                )
-                status_str = STATUS_FAIL
-            resp.status.CopyFrom(status)
+                    yield service_pb2.RunnerItemOutput(multi_output_response=resp)
+                    continue
+                successes = []
+                for output in resp.outputs:
+                    if not output.HasField('status') or not output.status.code:
+                        raise Exception(
+                            "Output must have a status code, please check the model implementation."
+                        )
+                    successes.append(output.status.code == status_code_pb2.SUCCESS)
+                if all(successes):
+                    status = status_pb2.Status(
+                        code=status_code_pb2.SUCCESS,
+                        description="Success",
+                    )
+                    status_str = STATUS_OK
+                elif any(successes):
+                    status = status_pb2.Status(
+                        code=status_code_pb2.MIXED_STATUS,
+                        description="Mixed Status",
+                    )
+                    status_str = STATUS_MIXED
+                else:
+                    status = status_pb2.Status(
+                        code=status_code_pb2.FAILURE,
+                        description="Failed",
+                    )
+                    status_str = STATUS_FAIL
+                resp.status.CopyFrom(status)
 
-            yield service_pb2.RunnerItemOutput(multi_output_response=resp)
+                yield service_pb2.RunnerItemOutput(multi_output_response=resp)
 
         duration_ms = (time.time() - start_time) * 1000
         logger.info(f"{endpoint} | {status_str} | {duration_ms:.2f}ms | req_id={req_id}")
@@ -237,45 +244,55 @@ class ModelRunner(BaseRunner, HealthProbeRequestHandler):
         status_str = STATUS_UNKNOWN
         endpoint = "POST /v2/.../outputs/stream  "
 
-        for resp in self.model.stream_wrapper(pmo_iterator(runner_item_iterator)):
-            # if we have any non-successful code already it's an error we can return.
-            if (
-                resp.status.code != status_code_pb2.SUCCESS
-                and resp.status.code != status_code_pb2.ZERO
-            ):
-                status_str = f"{resp.status.code} ERROR"
-                duration_ms = (time.time() - start_time) * 1000
-                logger.info(f"{endpoint} | {status_str} | {duration_ms:.2f}ms | req_id={req_id}")
-                yield service_pb2.RunnerItemOutput(multi_output_response=resp)
-                continue
-            successes = []
-            for output in resp.outputs:
-                if not output.HasField('status') or not output.status.code:
-                    raise Exception(
-                        "Output must have a status code, please check the model implementation."
+        # Get the first request to establish secrets context
+        first_request = None
+        runner_items = list(runner_item_iterator)  # Convert to list to avoid consuming iterator
+        if runner_items:
+            first_request = runner_items[0].post_model_outputs_request
+
+        # Use req_secrets_context based on the first request (secrets should be consistent across stream)
+        with req_secrets_context(first_request):
+            for resp in self.model.stream_wrapper(pmo_iterator(iter(runner_items))):
+                # if we have any non-successful code already it's an error we can return.
+                if (
+                    resp.status.code != status_code_pb2.SUCCESS
+                    and resp.status.code != status_code_pb2.ZERO
+                ):
+                    status_str = f"{resp.status.code} ERROR"
+                    duration_ms = (time.time() - start_time) * 1000
+                    logger.info(
+                        f"{endpoint} | {status_str} | {duration_ms:.2f}ms | req_id={req_id}"
                     )
-                successes.append(output.status.code == status_code_pb2.SUCCESS)
-            if all(successes):
-                status = status_pb2.Status(
-                    code=status_code_pb2.SUCCESS,
-                    description="Success",
-                )
-                status_str = STATUS_OK
-            elif any(successes):
-                status = status_pb2.Status(
-                    code=status_code_pb2.MIXED_STATUS,
-                    description="Mixed Status",
-                )
-                status_str = STATUS_MIXED
-            else:
-                status = status_pb2.Status(
-                    code=status_code_pb2.FAILURE,
-                    description="Failed",
-                )
-                status_str = STATUS_FAIL
-            resp.status.CopyFrom(status)
+                    yield service_pb2.RunnerItemOutput(multi_output_response=resp)
+                    continue
+                successes = []
+                for output in resp.outputs:
+                    if not output.HasField('status') or not output.status.code:
+                        raise Exception(
+                            "Output must have a status code, please check the model implementation."
+                        )
+                    successes.append(output.status.code == status_code_pb2.SUCCESS)
+                if all(successes):
+                    status = status_pb2.Status(
+                        code=status_code_pb2.SUCCESS,
+                        description="Success",
+                    )
+                    status_str = STATUS_OK
+                elif any(successes):
+                    status = status_pb2.Status(
+                        code=status_code_pb2.MIXED_STATUS,
+                        description="Mixed Status",
+                    )
+                    status_str = STATUS_MIXED
+                else:
+                    status = status_pb2.Status(
+                        code=status_code_pb2.FAILURE,
+                        description="Failed",
+                    )
+                    status_str = STATUS_FAIL
+                resp.status.CopyFrom(status)
 
-            yield service_pb2.RunnerItemOutput(multi_output_response=resp)
+                yield service_pb2.RunnerItemOutput(multi_output_response=resp)
 
         duration_ms = (time.time() - start_time) * 1000
         logger.info(f"{endpoint} | {status_str} | {duration_ms:.2f}ms | req_id={req_id}")
@@ -284,6 +301,27 @@ class ModelRunner(BaseRunner, HealthProbeRequestHandler):
         """Set the model for this runner."""
         self.model = model
 
+    def handle_liveness_probe(self):
+        # if the model has a handle_liveness_probe method, call it to determine liveness
+        # otherwise rely on HealthProbeRequestHandler.is_alive from the protocol
+        if hasattr(self.model, 'handle_liveness_probe'):
+            HealthProbeRequestHandler.is_alive = self.model.handle_liveness_probe()
+        return super().handle_liveness_probe()
+
+    def handle_readiness_probe(self):
+        # if the model has a handle_readiness_probe method, call it to determine readiness
+        # otherwise rely on HealthProbeRequestHandler.is_ready from the protocol
+        if hasattr(self.model, 'handle_readiness_probe'):
+            HealthProbeRequestHandler.is_ready = self.model.handle_readiness_probe()
+        return super().handle_readiness_probe()
+
+    def handle_startup_probe(self):
+        # if the model has a handle_startup_probe method, call it to determine startup
+        # otherwise rely on HealthProbeRequestHandler.is_startup from the protocol
+        if hasattr(self.model, 'handle_startup_probe'):
+            HealthProbeRequestHandler.is_startup = self.model.handle_startup_probe()
+        return super().handle_startup_probe()
+
 
 def pmo_iterator(runner_item_iterator, auth_helper=None):
     for runner_item in runner_item_iterator:

clarifai/runners/models/openai_class.py

@@ -69,6 +69,24 @@ class OpenAIModelClass(ModelClass):
 
         return completion_args
 
+    def handle_liveness_probe(self) -> bool:
+        """Handle liveness probe by checking if the client can list models."""
+        try:
+            _ = self.client.models.list()
+            return True
+        except Exception as e:
+            logger.error(f"Liveness probe failed: {e}", exc_info=True)
+            return False
+
+    def handle_readiness_probe(self) -> bool:
+        """Handle readiness probe by checking if the client can list models."""
+        try:
+            _ = self.client.models.list()
+            return True
+        except Exception as e:
+            logger.error(f"Readiness probe failed: {e}", exc_info=True)
+            return False
+
     def _set_usage(self, resp):
         if resp.usage and resp.usage.prompt_tokens and resp.usage.completion_tokens:
             self.set_output_context(

clarifai/runners/pipeline_steps/pipeline_step_builder.py

@@ -408,6 +408,7 @@ COPY --link=true requirements.txt config.yaml /home/nonroot/main/
         """
         max_checks = timeout_sec // interval_sec
         seen_logs = set()  # To avoid duplicate log messages
+        current_page = 1  # Track current page for log pagination
         st = time.time()
 
         for _ in range(max_checks):
@@ -434,14 +435,16 @@ COPY --link=true requirements.txt config.yaml /home/nonroot/main/
                     user_app_id=self.client.user_app_id,
                     pipeline_step_id=self.pipeline_step_id,
                     pipeline_step_version_id=self.pipeline_step_version_id,
-                    page=1,
+                    page=current_page,
                     per_page=50,
                 )
                 logs = self.client.STUB.ListLogEntries(
                     logs_request, metadata=self.client.auth_helper.metadata
                 )
 
+                entries_count = 0
                 for log_entry in logs.log_entries:
+                    entries_count += 1
                     if log_entry.url not in seen_logs:
                         seen_logs.add(log_entry.url)
                         log_entry_msg = re.sub(
@@ -451,6 +454,12 @@ COPY --link=true requirements.txt config.yaml /home/nonroot/main/
                         )
                         logger.info(log_entry_msg)
 
+                # If we got a full page (50 entries), there might be more logs on the next page
+                # If we got fewer than 50 entries, we've reached the end and should stay on current page
+                if entries_count == 50:
+                    current_page += 1
+                # else: stay on current_page
+
                 status = response.pipeline_step_version.status.code
                 if status in {
                     status_code_pb2.StatusCode.PIPELINE_STEP_READY,

clarifai/runners/server.py

@@ -301,7 +301,7 @@ class ModelServer:
                 f"> Playground:   To chat with your model, visit: {context.ui}/playground?model={context.model_id}__{context.model_version_id}&user_id={context.user_id}&app_id={context.app_id}\n"
             )
             logger.info(
-                f"> API URL:      To call your model via the API, use this model URL: {context.ui}/users/{context.user_id}/apps/{context.app_id}/models/{context.model_id}\n"
+                f"> API URL:      To call your model via the API, use this model URL: {context.ui}/{context.user_id}/{context.app_id}/models/{context.model_id}\n"
             )
             logger.info("Press CTRL+C to stop the runner.\n")
         self._runner.start()  # start the runner to fetch work from the API.