clarifai 11.0.5__py3-none-any.whl → 11.0.6rc2__py3-none-any.whl

clarifai/runners/matt_example.py~

@@ -0,0 +1,87 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    model = request.model
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          model.model_version.output_info, preserving_proto_field_name=True)
+
+    for i in range(10):  # fake something iterating generating 10 times.
+
+      outputs = []
+      for input in request.inputs:
+        # output = self.run_input(input, output_info, extra=f" {i}")
+        output = resources_pb2.Output()
+        output.data.text.raw = f"Generate Hello World {i}"
+        outputs.append(output)
+      resp = service_pb2.MultiOutputResponse(outputs=outputs,)
+      yield resp
+
+  def stream(self, request: service_pb2.PostModelOutputsRequest
+            ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    model = request.model
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          model.model_version.output_info, preserving_proto_field_name=True)
+
+    for i in range(10):  # fake something iterating generating 10 times.
+
+      outputs = []
+      for input in request.inputs:
+        # output = self.run_input(input, output_info, extra=f" {i}")
+        output = resources_pb2.Output()
+        output.data.text.raw = input.data.text.raw + f"Stream Hello World {i}"
+        outputs.append(output)
+      resp = service_pb2.MultiOutputResponse(outputs=outputs,)
+      yield resp
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/matt_llm_example.py

@@ -0,0 +1,129 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+import time
+from threading import Thread
+
+import grpc
+import requests
+
+from transformers import (AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer)
+
+model_name_or_path = "TheBloke/Llama-2-7B-chat-GPTQ"
+model_basename = "model"
+use_triton = False
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map='auto')
+streamer = TextIteratorStreamer(tokenizer)
+print("Model loaded")
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          request.model.model_version.output_info, preserving_proto_field_name=True)
+
+    for inp in request.inputs:
+      data = inp.data
+      print('start')
+      if data.text.raw != "":
+        input_text = data.text.raw
+      elif data.text.url != "":
+        input_text = str(requests.get(data.text.url).text)
+      else:
+        raise Exception("Need to include data.text.raw or data.text.url in your inputs.")
+
+      st = time.time()
+      max_tokens = 1024
+      # # Method 1
+      inputs = tokenizer(input_text, return_tensors='pt') #.input_ids.cuda()
+      generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_tokens)
+      thread = Thread(target=model.generate, kwargs=generation_kwargs)
+      thread.start()
+      times = []
+      st = time.time()
+      total_start = st
+      for new_text in streamer:
+        duration = time.time() - st
+        st = time.time()
+        print(f"Duration: {duration}")
+        times.append(duration)
+        # for new_text in ["hello", "world", "i'm", "streaming"]:
+
+        # out = model.generate(inputs=input_ids, temperature=0.7, max_new_tokens=max_tokens)
+        # out_text = tokenizer.decode(out[0], skip_special_tokens=True)
+        # output.data.text.raw = out_text.replace(input_text, '')
+
+        # # # Method 2
+        # print('before')
+        # pipe = pipeline(
+        #     "text-generation",
+        #     model=model,
+        #     tokenizer=tokenizer,
+        #     streamer=streamer,
+        #     max_new_tokens=max_tokens,
+        #     temperature=0.7,
+        #     top_p=0.95,
+        #     repetition_penalty=1.15,
+        #     return_full_text=False)
+        # print('pipe')
+        # a = pipe(input_text)
+        # print(a)
+        print("Posting: ", new_text)
+        output = resources_pb2.Output()
+        output.data.text.raw = new_text
+        result = service_pb2.MultiOutputResponse(
+            status=status_pb2.Status(
+                code=status_code_pb2.SUCCESS,
+                description="Success",
+            ),
+            outputs=[output],
+        )
+        yield result
+      print(f"Total time: {time.time() - total_start}")
+      print(f"Average time: {sum(times) / len(times)}")
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/matt_llm_example.py~

@@ -0,0 +1,128 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+import time
+from threading import Thread
+
+import grpc
+import requests
+
+from transformers import (AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer)
+
+model_name_or_path = "TheBloke/Llama-2-7B-chat-GPTQ"
+model_basename = "model"
+use_triton = False
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map='auto')
+streamer = TextIteratorStreamer(tokenizer)
+print("Model loaded")
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          request.model.model_version.output_info, preserving_proto_field_name=True)
+
+    for inp in request.inputs:
+      data = inp.data
+      print('start')
+      if data.text.raw != "":
+        input_text = data.text.raw
+      elif data.text.url != "":
+        input_text = str(requests.get(data.text.url).text)
+      else:
+        raise Exception("Need to include data.text.raw or data.text.url in your inputs.")
+
+      st = time.time()
+      max_tokens = 1024
+      # # Method 1
+      inputs = tokenizer(input_text, return_tensors='pt') #.input_ids.cuda()
+      generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_tokens)
+      thread = Thread(target=model.generate, kwargs=generation_kwargs)
+      thread.start()
+      times = []
+      st = time.time()
+      for new_text in streamer:
+        duration = time.time() - st
+        st = time.time()
+        print(f"Duration: {duration}")
+        times.append(duration)
+        # for new_text in ["hello", "world", "i'm", "streaming"]:
+
+        # out = model.generate(inputs=input_ids, temperature=0.7, max_new_tokens=max_tokens)
+        # out_text = tokenizer.decode(out[0], skip_special_tokens=True)
+        # output.data.text.raw = out_text.replace(input_text, '')
+
+        # # # Method 2
+        # print('before')
+        # pipe = pipeline(
+        #     "text-generation",
+        #     model=model,
+        #     tokenizer=tokenizer,
+        #     streamer=streamer,
+        #     max_new_tokens=max_tokens,
+        #     temperature=0.7,
+        #     top_p=0.95,
+        #     repetition_penalty=1.15,
+        #     return_full_text=False)
+        # print('pipe')
+        # a = pipe(input_text)
+        # print(a)
+        print("Posting: ", new_text)
+        output = resources_pb2.Output()
+        output.data.text.raw = new_text
+        result = service_pb2.MultiOutputResponse(
+            status=status_pb2.Status(
+                code=status_code_pb2.SUCCESS,
+                description="Success",
+            ),
+            outputs=[output],
+        )
+        yield result
+      print(f"Total time: {time.time() - st}")
+      print(f"Average time: {sum(times) / len(times)}")
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/models/model_upload.py

@@ -278,14 +278,15 @@ class ModelUploader:
       )
       python_version = DEFAULT_PYTHON_VERSION
 
-    base_image = PYTHON_BASE_IMAGE.format(python_version=python_version)
+    download_image = PYTHON_BASE_IMAGE.format(python_version=python_version)
 
     # Parse the requirements.txt file to determine the base image
     dependencies = self._parse_requirements()
     if 'torch' in dependencies and dependencies['torch']:
       torch_version = dependencies['torch']
 
-      for image in AVAILABLE_TORCH_IMAGES:
+      # Sort in reverse so that newer cuda versions come first and are preferred.
+      for image in sorted(AVAILABLE_TORCH_IMAGES, reverse=True):
         if torch_version in image and f'py{python_version}' in image:
           cuda_version = image.split('-')[-1].replace('cuda', '')
           base_image = TORCH_BASE_IMAGE.format(
@@ -295,11 +296,13 @@ class ModelUploader:
           )
           logger.info(f"Using Torch version {torch_version} base image to build the Docker image")
           break
-
+    else:  # if not torch then use the download image for the base image too
+      base_image = download_image
     # Replace placeholders with actual values
     dockerfile_content = dockerfile_template.safe_substitute(
         name='main',
-        BASE_IMAGE=base_image,
+        DOWNLOAD_IMAGE=download_image,  # for downloading
+        BASE_IMAGE=base_image,  # for runtime
     )
 
     # Write Dockerfile

clarifai/runners/utils/const.py

@@ -11,23 +11,24 @@ AVAILABLE_PYTHON_IMAGES = ['3.11', '3.12', '3.13']
 DEFAULT_PYTHON_VERSION = 3.12
 
 # List of available torch images
+# Keep sorted by most recent cuda version.
 AVAILABLE_TORCH_IMAGES = [
     '2.2.2-py3.11-cuda121',
     '2.3.1-py3.11-cuda121',
-    '2.4.0-py3.11-cuda121',
     '2.4.0-py3.11-cuda124',
-    '2.4.1-py3.11-cuda121',
+    '2.4.0-py3.11-cuda121',
     '2.4.1-py3.11-cuda124',
-    '2.5.1-py3.11-cuda121',
+    '2.4.1-py3.11-cuda121',
     '2.5.1-py3.11-cuda124',
+    '2.5.1-py3.11-cuda121',
     '2.2.2-py3.12-cuda121',
     '2.3.1-py3.12-cuda121',
-    '2.4.0-py3.12-cuda121',
     '2.4.0-py3.12-cuda124',
-    '2.4.1-py3.12-cuda121',
+    '2.4.0-py3.12-cuda121',
     '2.4.1-py3.12-cuda124',
-    '2.5.1-py3.12-cuda121',
+    '2.4.1-py3.12-cuda121',
     '2.5.1-py3.12-cuda124',
+    '2.5.1-py3.12-cuda121',
     # '2.2.2-py3.13-cuda121',
     # '2.3.1-py3.13-cuda121',
     # '2.4.0-py3.13-cuda121',

clarifai/runners/utils/loader.py

@@ -44,6 +44,7 @@ class HuggingFaceLoader:
       from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
+    os.environ['HF_HUB_ENABLE_HF_TRANSFER'] = '1'
     if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
       logger.info("Checkpoints already exist")
       return True

clarifai/runners/utils/logging.py

@@ -0,0 +1,6 @@
+import os
+
+from clarifai.utils.logging import get_logger
+
+logger_level = os.environ.get("LOG_LEVEL", "INFO")
+logger = get_logger(logger_level, __name__)

clarifai/utils/logging.py

@@ -143,7 +143,7 @@ def _configure_logger(name: str, logger_level: Union[int, str] = logging.NOTSET)
   else:
     # Add the new rich handler and formatter
     handler = RichHandler(
-        rich_tracebacks=True, log_time_format="%Y-%m-%d %H:%M:%S", console=Console(width=255))
+        rich_tracebacks=True, log_time_format="%Y-%m-%d %H:%M:%S.%f", console=Console(width=255))
     formatter = logging.Formatter('%(name)s:  %(message)s')
     handler.setFormatter(formatter)
     logger.addHandler(handler)

{clarifai-11.0.5.dist-info → clarifai-11.0.6rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: clarifai
-Version: 11.0.5
+Version: 11.0.6rc2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,32 +20,21 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=11.0.0
-Requires-Dist: clarifai-protocol>=0.0.14
-Requires-Dist: numpy>=1.22.0
-Requires-Dist: tqdm>=4.65.0
-Requires-Dist: tritonclient>=2.34.0
-Requires-Dist: rich>=13.4.2
-Requires-Dist: PyYAML>=6.0.1
-Requires-Dist: schema==0.7.5
-Requires-Dist: Pillow>=9.5.0
-Requires-Dist: inquirerpy==0.3.4
-Requires-Dist: tabulate>=0.9.0
-Requires-Dist: fsspec==2024.6.1
-Requires-Dist: click==8.1.7
+Requires-Dist: clarifai-grpc >=11.0.0
+Requires-Dist: clarifai-protocol >=0.0.14
+Requires-Dist: numpy >=1.22.0
+Requires-Dist: tqdm >=4.65.0
+Requires-Dist: tritonclient >=2.34.0
+Requires-Dist: rich >=13.4.2
+Requires-Dist: PyYAML >=6.0.1
+Requires-Dist: schema ==0.7.5
+Requires-Dist: Pillow >=9.5.0
+Requires-Dist: inquirerpy ==0.3.4
+Requires-Dist: tabulate >=0.9.0
+Requires-Dist: fsspec ==2024.6.1
+Requires-Dist: click ==8.1.7
 Provides-Extra: all
-Requires-Dist: pycocotools==2.0.6; extra == "all"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
+Requires-Dist: pycocotools ==2.0.6 ; extra == 'all'
 
 <h1 align="center">
   <a href="https://www.clarifai.com/"><img alt="Clarifai" title="Clarifai" src="https://github.com/user-attachments/assets/623b883b-7fe5-4b95-bbfa-8691f5779af4"></a>