inference-cli 0.9.0__1-py3-none-any.whl

cli/inference_cli/actions.py

@@ -0,0 +1,31 @@
+import base64
+import requests
+
+def is_url(string: str):
+    return string.startswith("http")
+
+def infer(image, project_id, model_version, api_key, host):
+    endpoint = f"{host}/{project_id}/{model_version}"
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    if is_url(image):
+        image_type = "url"
+        params = {
+            "api_key": api_key,
+            "image": image,
+            "image_type": image_type
+        }
+        response = requests.post(endpoint, headers=headers, params=params).json()
+    else:
+        with open(image, 'rb') as image_file:
+            image = base64.b64encode(image_file.read()).decode('utf-8')
+        image_type = "base64"
+        params = {
+            "api_key": api_key,
+            "image_type": image_type
+        }
+        response = requests.post(endpoint, headers=headers, params=params, data=image).json()
+
+    print(response)

cli/inference_cli/lib/__init__.py

@@ -0,0 +1 @@
+from inference_cli.lib.container_adapter import start_inference_container

cli/inference_cli/lib/container_adapter.py

@@ -0,0 +1,112 @@
+import subprocess
+
+import docker
+import typer
+
+docker_client = docker.from_env()
+
+
+def ask_user_to_kill_container(c):
+    name = c.attrs.get("Name", "")
+    env_vars = c.attrs.get("Config", {}).get("Env", {})
+    port = 9001
+    for var in env_vars:
+        if var.startswith("PORT="):
+            port = var.split("=")[1]
+    should_delete = typer.confirm(
+        f" An inference server is already running in container {name} on port {port}. Are you sure you want to delete it?"
+    )
+    return should_delete
+
+
+def is_inference_server_container(container):
+    image_tags = container.image.tags
+    for t in image_tags:
+        if t.startswith("roboflow/roboflow-inference-server"):
+            return True
+    return False
+
+
+def handle_existing_containers(containers):
+    has_existing_containers = False
+    for c in containers:
+        if is_inference_server_container(c):
+            has_existing_containers = True
+            if c.attrs.get("State", {}).get("Status", "").lower() == "running":
+                should_kill = ask_user_to_kill_container(c)
+                if should_kill:
+                    c.kill()
+                    has_existing_containers = False
+    return has_existing_containers
+
+
+def find_existing_containers():
+    containers = []
+    for c in docker_client.containers.list():
+        if is_inference_server_container(c):
+            if c.attrs.get("State", {}).get("Status", "").lower() == "running":
+                containers.append(c)
+    return containers
+
+
+def get_image():
+    try:
+        subprocess.check_output("nvidia-smi")
+        print("GPU detected. Using a GPU image.")
+        return "roboflow/roboflow-inference-server-gpu:latest"
+    except:
+        print("No GPU detected. Using a CPU image.")
+        return "roboflow/roboflow-inference-server-cpu:latest"
+
+
+def start_inference_container(
+    api_key,
+    image=None,
+    port=9001,
+    labels=None,
+    project="roboflow-platform",
+    metrics_enabled=True,
+    device_id=None,
+    num_workers=1,
+):
+    containers = find_existing_containers()
+    if len(containers) > 0:
+        still_has_containers = handle_existing_containers(containers)
+        if still_has_containers:
+            print("Please kill the existing containers and try again.")
+            return
+
+    if image is None:
+        image = get_image()
+
+    device_requests = None
+    privileged = False
+    if "gpu" in image:
+        privileged = True
+        device_requests = (
+            [docker.types.DeviceRequest(device_ids=["all"], capabilities=[["gpu"]])],
+        )
+
+    print(f"Starting inference server container...")
+    docker_client.containers.run(
+        image=image,
+        privileged=privileged,
+        detach=True,
+        labels=labels,
+        ports={"9001": port},
+        # network="host",
+        device_requests=device_requests,
+        environment=[
+            "HOST=0.0.0.0",
+            f"PORT={port}",
+            f"PROJECT={project}",
+            f"METRICS_ENABLED={metrics_enabled}",
+            f"DEVICE_ID={device_id}",
+            f"API_KEY={api_key}",
+            f"NUM_WORKERS={num_workers}",
+        ],
+    )
+
+
+if __name__ == "__main__":
+    start_inference_container("my_api_key")

cli/inference_cli/main.py

@@ -0,0 +1,46 @@
+import typer
+from typing_extensions import Annotated
+from inference_cli.server import server_app
+import inference_cli.lib
+
+app = typer.Typer()
+
+app.add_typer(server_app, name="server")
+
+
+@app.command()
+def infer(
+    image: Annotated[
+        str, typer.Argument(help="URL or local path of image to run inference on.")
+    ],
+    project_id: Annotated[
+        str,
+        typer.Option(
+            "--project-id", "-p", help="Roboflow project to run inference with."
+        ),
+    ],
+    model_version: Annotated[
+        str,
+        typer.Option(
+            "--model-version",
+            "-v",
+            help="Version of model to run inference with.",
+        ),
+    ],
+    api_key: Annotated[
+        str,
+        typer.Option("--api-key", "-a", help="Roboflow API key for your workspace."),
+    ],
+    host: Annotated[
+        str,
+        typer.Option("--host", "-h", help="Host to run inference on."),
+    ] = "http://localhost:9001",
+):
+    typer.echo(
+        f"Running inference on image {image}, using model ({project_id}/{model_version}), and host ({host})"
+    )
+    inference_cli.lib.infer(image, project_id, model_version, api_key, host)
+
+
+if __name__ == "__main__":
+    app()

cli/inference_cli/server.py

@@ -0,0 +1,40 @@
+import typer
+from typing_extensions import Annotated
+from inference_cli.lib import start_inference_container, check_inference_server_status
+
+server_app = typer.Typer(
+    help="""Commands for running the inference server locally. \n 
+    Supported devices targets are x86 CPU, ARM64 CPU, and NVIDIA GPU."""
+)
+
+
+@server_app.command()
+def start(
+    port: Annotated[
+        int,
+        typer.Option(
+            "--port",
+            "-p",
+            help="Port to run the inference server on (default is 9001).",
+        ),
+    ] = 9001,
+    rf_env: Annotated[
+        str,
+        typer.Option(
+            "--rf-env",
+            "-rfe",
+            help="Roboflow environment to run the inference server with (default is roboflow-platform).",
+        ),
+    ] = "roboflow-platform",
+):
+    start_inference_container("", port=port, project=rf_env)
+
+
+@server_app.command()
+def status():
+    print("Checking status of inference server.")
+    check_inference_server_status()
+
+
+if __name__ == "__main__":
+    server_app()

cli/inference_cli/version.py

@@ -0,0 +1,5 @@
+__version__ = "0.0.11"
+
+
+if __name__ == "__main__":
+    print(__version__)

cli/setup.py

@@ -0,0 +1,49 @@
+import os
+import setuptools
+from setuptools import find_packages
+import sys
+import shutil
+
+root = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+sys.path.append(root)
+
+shutil.copyfile(
+    os.path.join(root, "../inference/core/version.py"),
+    os.path.join(root, "inference_cli/version.py"),
+)
+from inference_cli.version import __version__
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+
+def read_requirements(path):
+    with open(os.path.join(root, path)) as fh:
+        return [line.strip() for line in fh]
+
+
+setuptools.setup(
+    name="inference-cli",
+    version=__version__,
+    author="Roboflow",
+    author_email="help@roboflow.com",
+    description="With no prior knowledge of machine learning or device-specific deployment, you can deploy a computer vision model to a range of devices and environments using Roboflow Inference CLI.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/roboflow/inference",
+    packages=find_packages(
+        where="./",
+    ),
+    entry_points={
+        "console_scripts": [
+            "inference=inference_cli.main:app",
+        ],
+    },
+    install_requires=read_requirements("requirements.txt"),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.7",
+)

inference/cli/inference_cli/actions.py

@@ -0,0 +1,31 @@
+import base64
+import requests
+
+def is_url(string: str):
+    return string.startswith("http")
+
+def infer(image, project_id, model_version, api_key, host):
+    endpoint = f"{host}/{project_id}/{model_version}"
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    if is_url(image):
+        image_type = "url"
+        params = {
+            "api_key": api_key,
+            "image": image,
+            "image_type": image_type
+        }
+        response = requests.post(endpoint, headers=headers, params=params).json()
+    else:
+        with open(image, 'rb') as image_file:
+            image = base64.b64encode(image_file.read()).decode('utf-8')
+        image_type = "base64"
+        params = {
+            "api_key": api_key,
+            "image_type": image_type
+        }
+        response = requests.post(endpoint, headers=headers, params=params, data=image).json()
+
+    print(response)

inference/cli/inference_cli/lib/__init__.py

@@ -0,0 +1 @@
+from inference_cli.lib.container_adapter import start_inference_container

inference/cli/inference_cli/lib/container_adapter.py

@@ -0,0 +1,112 @@
+import subprocess
+
+import docker
+import typer
+
+docker_client = docker.from_env()
+
+
+def ask_user_to_kill_container(c):
+    name = c.attrs.get("Name", "")
+    env_vars = c.attrs.get("Config", {}).get("Env", {})
+    port = 9001
+    for var in env_vars:
+        if var.startswith("PORT="):
+            port = var.split("=")[1]
+    should_delete = typer.confirm(
+        f" An inference server is already running in container {name} on port {port}. Are you sure you want to delete it?"
+    )
+    return should_delete
+
+
+def is_inference_server_container(container):
+    image_tags = container.image.tags
+    for t in image_tags:
+        if t.startswith("roboflow/roboflow-inference-server"):
+            return True
+    return False
+
+
+def handle_existing_containers(containers):
+    has_existing_containers = False
+    for c in containers:
+        if is_inference_server_container(c):
+            has_existing_containers = True
+            if c.attrs.get("State", {}).get("Status", "").lower() == "running":
+                should_kill = ask_user_to_kill_container(c)
+                if should_kill:
+                    c.kill()
+                    has_existing_containers = False
+    return has_existing_containers
+
+
+def find_existing_containers():
+    containers = []
+    for c in docker_client.containers.list():
+        if is_inference_server_container(c):
+            if c.attrs.get("State", {}).get("Status", "").lower() == "running":
+                containers.append(c)
+    return containers
+
+
+def get_image():
+    try:
+        subprocess.check_output("nvidia-smi")
+        print("GPU detected. Using a GPU image.")
+        return "roboflow/roboflow-inference-server-gpu:latest"
+    except:
+        print("No GPU detected. Using a CPU image.")
+        return "roboflow/roboflow-inference-server-cpu:latest"
+
+
+def start_inference_container(
+    api_key,
+    image=None,
+    port=9001,
+    labels=None,
+    project="roboflow-platform",
+    metrics_enabled=True,
+    device_id=None,
+    num_workers=1,
+):
+    containers = find_existing_containers()
+    if len(containers) > 0:
+        still_has_containers = handle_existing_containers(containers)
+        if still_has_containers:
+            print("Please kill the existing containers and try again.")
+            return
+
+    if image is None:
+        image = get_image()
+
+    device_requests = None
+    privileged = False
+    if "gpu" in image:
+        privileged = True
+        device_requests = (
+            [docker.types.DeviceRequest(device_ids=["all"], capabilities=[["gpu"]])],
+        )
+
+    print(f"Starting inference server container...")
+    docker_client.containers.run(
+        image=image,
+        privileged=privileged,
+        detach=True,
+        labels=labels,
+        ports={"9001": port},
+        # network="host",
+        device_requests=device_requests,
+        environment=[
+            "HOST=0.0.0.0",
+            f"PORT={port}",
+            f"PROJECT={project}",
+            f"METRICS_ENABLED={metrics_enabled}",
+            f"DEVICE_ID={device_id}",
+            f"API_KEY={api_key}",
+            f"NUM_WORKERS={num_workers}",
+        ],
+    )
+
+
+if __name__ == "__main__":
+    start_inference_container("my_api_key")

inference/cli/inference_cli/main.py

@@ -0,0 +1,51 @@
+import typer
+import inference_cli.actions
+from inference_cli.lib import start_inference_container
+
+app = typer.Typer()
+
+
+@app.command()
+def serve(
+    port: int = typer.Option(
+        9001,
+        "-p",
+        "--port",
+        help="Port to run the inference server on (default is 9001).",
+    ),
+    rf_env: str = typer.Option(
+        "roboflow-platform",
+        "-rfe",
+        "--rf-env",
+        help="Roboflow environment to run the inference server with (default is roboflow-platform).",
+    ),
+):
+    start_inference_container("", port=port, project=rf_env)
+
+
+@app.command()
+def infer(
+    image: str = typer.Option(
+        None, "-i", "--image", help="URL or local path of image to run inference on."
+    ),
+    project_id: str = typer.Option(
+        None, "-p", "--project-id", help="Roboflow project to run inference with."
+    ),
+    model_version: str = typer.Option(
+        None, "-v", "--model-version", help="Version of model to run inference with."
+    ),
+    api_key: str = typer.Option(
+        None, "-a", "--api-key", help="Roboflow API key for your workspace."
+    ),
+    host: str = typer.Option(
+        "http://localhost:9001", "-h", "--host", help="Host to run inference on."
+    ),
+):
+    typer.echo(
+        f"Running inference on image {image}, using model ({project_id}/{model_version}), and host ({host})"
+    )
+    inference_cli.actions.infer(image, project_id, model_version, api_key, host)
+
+
+if __name__ == "__main__":
+    app()

inference/cli/inference_cli/version.py

@@ -0,0 +1,5 @@
+__version__ = "0.8.9"
+
+
+if __name__ == "__main__":
+    print(__version__)

inference/cli/setup.py

@@ -0,0 +1,49 @@
+import os
+import setuptools
+from setuptools import find_packages
+import sys
+import shutil
+
+root = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+sys.path.append(root)
+
+shutil.copyfile(
+    os.path.join(root, "../inference/core/version.py"),
+    os.path.join(root, "inference_cli/version.py"),
+)
+from inference_cli.version import __version__
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+
+def read_requirements(path):
+    with open(os.path.join(root, path)) as fh:
+        return [line.strip() for line in fh]
+
+
+setuptools.setup(
+    name="inference-cli",
+    version=__version__,
+    author="Roboflow",
+    author_email="help@roboflow.com",
+    description="With no prior knowledge of machine learning or device-specific deployment, you can deploy a computer vision model to a range of devices and environments using Roboflow Inference CLI.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/roboflow/inference",
+    packages=find_packages(
+        where="./",
+    ),
+    entry_points={
+        "console_scripts": [
+            "inference=inference_cli.main:app",
+        ],
+    },
+    install_requires=read_requirements("requirements.txt"),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.7",
+)

inference/core/__init__.py

@@ -0,0 +1,52 @@
+import threading
+import time
+
+import requests
+
+from inference.core.env import DISABLE_VERSION_CHECK, VERSION_CHECK_MODE
+from inference.core.logger import logger
+from inference.core.version import __version__
+
+latest_release = None
+last_checked = 0
+cache_duration = 86400  # 24 hours
+log_frequency = 300  # 5 minutes
+
+
+def get_latest_release_version():
+    global latest_release, last_checked
+    now = time.time()
+    if latest_release is None or now - last_checked > cache_duration:
+        try:
+            logger.debug("Checking for latest inference release version...")
+            response = requests.get(
+                "https://api.github.com/repos/roboflow/inference/releases/latest"
+            )
+            response.raise_for_status()
+            latest_release = response.json()["tag_name"].lstrip("v")
+            last_checked = now
+        except requests.exceptions.RequestException:
+            pass
+
+
+def check_latest_release_against_current():
+    get_latest_release_version()
+    if latest_release is not None and latest_release != __version__:
+        logger.warning(
+            f"Your inference package version {__version__} is out of date! Please upgrade to version {latest_release} of inference for the latest features and bug fixes by running `pip install --upgrade roboflow-inference`."
+        )
+
+
+def check_latest_release_against_current_continuous():
+    while True:
+        check_latest_release_against_current()
+        time.sleep(log_frequency)
+
+
+if not DISABLE_VERSION_CHECK:
+    if VERSION_CHECK_MODE == "continuous":
+        t = threading.Thread(target=check_latest_release_against_current_continuous)
+        t.daemon = True
+        t.start()
+    else:
+        check_latest_release_against_current()

inference/core/cache/__init__.py

@@ -0,0 +1,8 @@
+from inference.core.cache.memory import MemoryCache
+from inference.core.cache.redis import RedisCache
+from inference.core.env import REDIS_HOST, REDIS_PORT
+
+if REDIS_HOST is not None:
+    cache = RedisCache(host=REDIS_HOST, port=REDIS_PORT)
+else:
+    cache = MemoryCache()