clarifai 10.5.1__py3-none-any.whl → 10.5.3__py3-none-any.whl

clarifai/runners/deepgram_runner.py~

@@ -0,0 +1,130 @@
+import os
+import time
+import httpx
+from dotenv import load_dotenv
+import threading
+import queue
+import hashlib
+
+from deepgram import (
+    DeepgramClient,
+    LiveTranscriptionEvents,
+    LiveOptions,
+)
+
+load_dotenv()
+
+os.environ["DG_API_KEY"] = "1d5666523f0f2fbaf3e2db6ae7717f6f87280b5e"
+API_KEY = os.getenv("DG_API_KEY")
+
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def setup_connection(self):
+    print("ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZz")
+    # STEP 2: Create a websocket connection to Deepgram
+    self.dg_connection = self.deepgram.listen.live.v("1")
+
+    output_q = self.output_q
+
+    # STEP 3: Define the event handlers for the connection
+    def on_message(self, result, **kwargs):
+      if result.is_final:
+        sentence = result.channel.alternatives[0].transcript
+        if len(sentence) == 0:
+          return
+        print("SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS")
+        print(f"speaker: {sentence}")
+        # put it on a queue as we get responses from deepgram.
+        output_q.put(sentence)
+
+    def on_metadata(self, metadata, **kwargs):
+      print(f"\n\n{metadata}\n\n")
+
+    def on_error(self, error, **kwargs):
+      print(f"\n\n{error}\n\n")
+
+    # STEP 4: Register the event handlers
+    self.dg_connection.on(LiveTranscriptionEvents.Transcript, on_message)
+    self.dg_connection.on(LiveTranscriptionEvents.Metadata, on_metadata)
+    self.dg_connection.on(LiveTranscriptionEvents.Error, on_error)
+
+    # STEP 5: Configure Deepgram options for live transcription
+    self.options = LiveOptions(
+        model="nova-2",
+        language="en-US",
+        smart_format=True,
+    )
+
+    # STEP 6: Start the connection
+    self.dg_connection.start(self.options)
+
+    print("DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD")
+    print(self.dg_connection._socket)
+
+  def __init__(self, *args, **kwargs):
+    print("MyRunner init")
+    # STEP 1: Create a Deepgram client using the API key
+    self.deepgram = DeepgramClient(API_KEY)
+    self.output_q = queue.Queue()
+
+    self.setup_connection()
+
+    super().__init__(*args, **kwargs)
+
+  def stream(self, request: service_pb2.PostModelOutputsRequest
+            ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    assert len(request.inputs) == 1, "This runner only supports one input at a time."
+
+    # Get the next chunk of data from the incoming stream.
+
+    print("Got some audio data")
+
+    data = request.inputs[0].data.audio.base64
+    print(hashlib.md5(data).hexdigest())
+
+    if not self.dg_connection._socket:
+      self.dg_connection.finish()
+      self.dg_connection.start(self.options)
+      #self.setup_connection()
+    self.dg_connection.send(data)
+    print("Sent it to deepgram")
+
+    while True:
+      try:
+        item = self.output_q.get(timeout=0.1)
+        output = resources_pb2.Output()
+        output.data.text.raw = item
+        output.status.code = 10000
+        print("YYYYYYYYYYYYYYYYYYYYYYYYYYYYYY")
+        print("Yielding: ", output.data.text.raw)
+        yield service_pb2.MultiOutputResponse(outputs=[
+            output,
+        ])
+      except queue.Empty:
+        print("Queue is empty, sleeping then breaking")
+        break
+
+  # # STEP 13: Close the connection to Deepgram
+  # dg_connection.finish()
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/example_llama2.py~

@@ -0,0 +1,72 @@
+from clarifai.client.runner import Runner
+from clarifai_grpc.grpc.api import resources_pb2
+
+# This example requires to run the following before running this example:
+# pip install transformers
+
+# https://huggingface.co/TheBloke/Llama-2-70B-chat-GPTQ
+model_name_or_path = "TheBloke/Llama-2-7B-chat-GPTQ"
+model_basename = "model"
+
+use_triton = False
+
+
+class Llama2Runner(Runner):
+  """A custom runner that runs the LLama2 LLM.
+  """
+
+  def __init__(self, *args, **kwargs):
+    print("Starting to load the model...")
+    st = time.time()
+    self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+    self.model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map='auto')
+
+    self.logger.info("Loading model complete in (%f seconds), ready to loop for requests." %
+                     (time.time() - st))
+    super(MyRunner, self).__init__(*args, **kwargs)
+
+  def run_input(self, input: resources_pb2.Input,
+                output_info: resources_pb2.OutputInfo) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+    data = input.data
+    if data.text.raw != "":
+      input_text = data.text.raw
+    elif data.text.url != "":
+      input_text = str(requests.get(data.text.url).text)
+    else:
+      raise Exception("Need to include data.text.raw or data.text.url in your inputs.")
+
+    st = time.time()
+    max_tokens = 4096
+    # # Method 1
+    # input_ids = self.tokenizer(input_text, return_tensors='pt').input_ids.cuda()
+    # out = self.model.generate(inputs=input_ids, temperature=0.7, max_new_tokens=max_tokens)
+    # out_text = self.tokenizer.decode(out[0], skip_special_tokens=True)
+    # output.data.text.raw = out_text.replace(input_text, '')
+
+    # # Method 2
+    pipe = pipeline(
+        "text-generation",
+        model=self.model,
+        tokenizer=self.tokenizer,
+        max_new_tokens=max_tokens,
+        temperature=0.7,
+        top_p=0.95,
+        repetition_penalty=1.15,
+        return_full_text=False)
+    a = pipe(input_text)
+    output.data.text.raw = a[0]['generated_text']
+    return output
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  Llama2Runner(runner_id="sdk-llama2-runner").start()

clarifai/runners/matt_example.py

@@ -0,0 +1,89 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    model = request.model
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          model.model_version.output_info, preserving_proto_field_name=True)
+
+    for i in range(10):  # fake something iterating generating 10 times.
+
+      outputs = []
+      for input in request.inputs:
+        # output = self.run_input(input, output_info, extra=f" {i}")
+        output = resources_pb2.Output()
+        output.data.text.raw = f"Generate Hello World {i}"
+        outputs.append(output)
+      resp = service_pb2.MultiOutputResponse(outputs=outputs,)
+      yield resp
+
+  def stream(self, request: service_pb2.PostModelOutputsRequest
+            ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    model = request.model
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          model.model_version.output_info, preserving_proto_field_name=True)
+
+    for i in range(10):  # fake something iterating generating 10 times.
+
+      outputs = []
+      for input in request.inputs:
+        # output = self.run_input(input, output_info, extra=f" {i}")
+        output = resources_pb2.Output()
+        out_text = input.data.text.raw + f"Stream Hello World {i}"
+        print(out_text)
+        output.data.text.raw = out_text
+        outputs.append(output)
+      resp = service_pb2.MultiOutputResponse(outputs=outputs,)
+      yield resp
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/matt_example.py~

@@ -0,0 +1,87 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    model = request.model
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          model.model_version.output_info, preserving_proto_field_name=True)
+
+    for i in range(10):  # fake something iterating generating 10 times.
+
+      outputs = []
+      for input in request.inputs:
+        # output = self.run_input(input, output_info, extra=f" {i}")
+        output = resources_pb2.Output()
+        output.data.text.raw = f"Generate Hello World {i}"
+        outputs.append(output)
+      resp = service_pb2.MultiOutputResponse(outputs=outputs,)
+      yield resp
+
+  def stream(self, request: service_pb2.PostModelOutputsRequest
+            ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    model = request.model
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          model.model_version.output_info, preserving_proto_field_name=True)
+
+    for i in range(10):  # fake something iterating generating 10 times.
+
+      outputs = []
+      for input in request.inputs:
+        # output = self.run_input(input, output_info, extra=f" {i}")
+        output = resources_pb2.Output()
+        output.data.text.raw = input.data.text.raw + f"Stream Hello World {i}"
+        outputs.append(output)
+      resp = service_pb2.MultiOutputResponse(outputs=outputs,)
+      yield resp
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/matt_llm_example.py

@@ -0,0 +1,129 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+import time
+from threading import Thread
+
+import grpc
+import requests
+
+from transformers import (AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer)
+
+model_name_or_path = "TheBloke/Llama-2-7B-chat-GPTQ"
+model_basename = "model"
+use_triton = False
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map='auto')
+streamer = TextIteratorStreamer(tokenizer)
+print("Model loaded")
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          request.model.model_version.output_info, preserving_proto_field_name=True)
+
+    for inp in request.inputs:
+      data = inp.data
+      print('start')
+      if data.text.raw != "":
+        input_text = data.text.raw
+      elif data.text.url != "":
+        input_text = str(requests.get(data.text.url).text)
+      else:
+        raise Exception("Need to include data.text.raw or data.text.url in your inputs.")
+
+      st = time.time()
+      max_tokens = 1024
+      # # Method 1
+      inputs = tokenizer(input_text, return_tensors='pt') #.input_ids.cuda()
+      generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_tokens)
+      thread = Thread(target=model.generate, kwargs=generation_kwargs)
+      thread.start()
+      times = []
+      st = time.time()
+      total_start = st
+      for new_text in streamer:
+        duration = time.time() - st
+        st = time.time()
+        print(f"Duration: {duration}")
+        times.append(duration)
+        # for new_text in ["hello", "world", "i'm", "streaming"]:
+
+        # out = model.generate(inputs=input_ids, temperature=0.7, max_new_tokens=max_tokens)
+        # out_text = tokenizer.decode(out[0], skip_special_tokens=True)
+        # output.data.text.raw = out_text.replace(input_text, '')
+
+        # # # Method 2
+        # print('before')
+        # pipe = pipeline(
+        #     "text-generation",
+        #     model=model,
+        #     tokenizer=tokenizer,
+        #     streamer=streamer,
+        #     max_new_tokens=max_tokens,
+        #     temperature=0.7,
+        #     top_p=0.95,
+        #     repetition_penalty=1.15,
+        #     return_full_text=False)
+        # print('pipe')
+        # a = pipe(input_text)
+        # print(a)
+        print("Posting: ", new_text)
+        output = resources_pb2.Output()
+        output.data.text.raw = new_text
+        result = service_pb2.MultiOutputResponse(
+            status=status_pb2.Status(
+                code=status_code_pb2.SUCCESS,
+                description="Success",
+            ),
+            outputs=[output],
+        )
+        yield result
+      print(f"Total time: {time.time() - total_start}")
+      print(f"Average time: {sum(times) / len(times)}")
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/runners/matt_llm_example.py~

@@ -0,0 +1,128 @@
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
+from collections.abc import Iterator
+from google.protobuf import json_format
+
+from clarifai.client.runner import Runner
+import time
+from threading import Thread
+
+import grpc
+import requests
+
+from transformers import (AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer)
+
+model_name_or_path = "TheBloke/Llama-2-7B-chat-GPTQ"
+model_basename = "model"
+use_triton = False
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map='auto')
+streamer = TextIteratorStreamer(tokenizer)
+print("Model loaded")
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input, output_info: resources_pb2.OutputInfo,
+                **kwargs) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    # Optional use of output_info
+    params_dict = {}
+    if "params" in output_info:
+      params_dict = output_info["params"]
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World" + params_dict.get(
+          "hello", "") + kwargs.get("extra", "")
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com",
+                                                    "newdomain.com" + params_dict.get("domain",))
+    return output
+
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    """Example yielding a whole batch of streamed stuff back.
+    """
+
+    output_info = None
+    if request.model.model_version.id != "":
+      output_info = json_format.MessageToDict(
+          request.model.model_version.output_info, preserving_proto_field_name=True)
+
+    for inp in request.inputs:
+      data = inp.data
+      print('start')
+      if data.text.raw != "":
+        input_text = data.text.raw
+      elif data.text.url != "":
+        input_text = str(requests.get(data.text.url).text)
+      else:
+        raise Exception("Need to include data.text.raw or data.text.url in your inputs.")
+
+      st = time.time()
+      max_tokens = 1024
+      # # Method 1
+      inputs = tokenizer(input_text, return_tensors='pt') #.input_ids.cuda()
+      generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_tokens)
+      thread = Thread(target=model.generate, kwargs=generation_kwargs)
+      thread.start()
+      times = []
+      st = time.time()
+      for new_text in streamer:
+        duration = time.time() - st
+        st = time.time()
+        print(f"Duration: {duration}")
+        times.append(duration)
+        # for new_text in ["hello", "world", "i'm", "streaming"]:
+
+        # out = model.generate(inputs=input_ids, temperature=0.7, max_new_tokens=max_tokens)
+        # out_text = tokenizer.decode(out[0], skip_special_tokens=True)
+        # output.data.text.raw = out_text.replace(input_text, '')
+
+        # # # Method 2
+        # print('before')
+        # pipe = pipeline(
+        #     "text-generation",
+        #     model=model,
+        #     tokenizer=tokenizer,
+        #     streamer=streamer,
+        #     max_new_tokens=max_tokens,
+        #     temperature=0.7,
+        #     top_p=0.95,
+        #     repetition_penalty=1.15,
+        #     return_full_text=False)
+        # print('pipe')
+        # a = pipe(input_text)
+        # print(a)
+        print("Posting: ", new_text)
+        output = resources_pb2.Output()
+        output.data.text.raw = new_text
+        result = service_pb2.MultiOutputResponse(
+            status=status_pb2.Status(
+                code=status_code_pb2.SUCCESS,
+                description="Success",
+            ),
+            outputs=[output],
+        )
+        yield result
+      print(f"Total time: {time.time() - st}")
+      print(f"Average time: {sum(times) / len(times)}")
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="matt-test-runner", base_url="http://q6:32013", num_parallel_polls=1).start()

clarifai/utils/constants.py

@@ -1,12 +1,2 @@
-import os
-
-USER_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache")
-CLARIFAI_HOME = os.path.expanduser(
-    os.getenv(
-        "CLARIFAI_HOME",
-        os.path.join(os.getenv("XDG_CACHE_HOME", USER_CACHE_DIR), "clarifai"),
-    ))
-os.makedirs(CLARIFAI_HOME, exist_ok=True)
-CLARIFAI_PAT_PATH = os.path.join(CLARIFAI_HOME, "pat")
 CLARIFAI_PAT_ENV_VAR = "CLARIFAI_PAT"
 CLARIFAI_SESSION_TOKEN_ENV_VAR = "CLARIFAI_SESSION_TOKEN"

clarifai/versions.py

@@ -1,6 +1,6 @@
 import os
 
-CLIENT_VERSION = "10.5.1"
+CLIENT_VERSION = "10.5.3"
 OS_VER = os.sys.platform
 PYTHON_VERSION = '.'.join(
     map(str, [os.sys.version_info.major, os.sys.version_info.minor, os.sys.version_info.micro]))