sciveo 0.1.39__tar.gz → 0.1.40__tar.gz

{sciveo-0.1.39 → sciveo-0.1.40}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.39
+Version: 0.1.40
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net
 Provides-Extra: media
 Provides-Extra: media-server
 Provides-Extra: media-ml
+Provides-Extra: media-ml-all
 Provides-Extra: all
 Provides-Extra: media-all
 

{sciveo-0.1.39 → sciveo-0.1.40}/sciveo/media/pipelines/pipeline.py

@@ -81,6 +81,7 @@ class MediaPipeline:
       "image-diffusion": ImageDiffusionText,
       "image-diffusion-image-text": ImageDiffusionImageText,
       "image-to-text": ImageToTextProcessor,
+      "image-query": ImageQueryProcessor,
       "image-fgbg-filter": ImageFGBGFilter,
       "image-segmentation": ImageSegmentation,
       "image-depth-estimation": ImageDepthEstimation,

{sciveo-0.1.39 → sciveo-0.1.40}/sciveo/media/pipelines/processors/image/generators.py

@@ -24,7 +24,7 @@ from sciveo.tools.logger import *
 from sciveo.tools.common import *
 from sciveo.media.pipelines.processors.tpu_base import *
 from sciveo.media.pipelines.base import ApiContent
-from sciveo.ml.images.description import ImageToText
+from sciveo.ml.images.description import ImageToText, ImageQuery
 
 
 class ImageDiffusionText(TPUBaseProcessor):
@@ -202,6 +202,56 @@ class ImageToTextProcessor(TPUBaseProcessor):
     return True
 
 
+class ImageQueryProcessor(TPUBaseProcessor):
+  def __init__(self, processor_config, max_progress) -> None:
+    super().__init__(processor_config, max_progress)
+
+    self.api = ApiContent()
+
+    self.cache_dir = os.path.join(os.environ['MEDIA_MODELS_BASE_PATH'], "models/")
+    self.device = os.environ.get("MEDIA_PROCESSING_BACKEND", "cpu")
+
+    self.default.update({
+      "max_length": 64,
+      "model_id": 0,
+      "query": "describe the image",
+      "output": False
+    })
+
+    self.predictor = None
+
+  def process(self, media):
+    debug("process", media['guid'])
+    if self.predictor is None:
+      self.predictor = ImageQuery(self['query'], self['model_id'], self['max_length'], self.cache_dir, self.device)
+    local_path = media["local_path"]
+    predict = self.predictor.predict_one(local_path)
+    return self.set_media(media, predict)
+
+  def set_media(self, media, predict):
+    media.setdefault("next", [])
+    media["next"].append({
+      "guid": f"TXT-{media['guid']}",
+      "parent": media['guid'],
+      "content_type": "comment",
+      "content_text": predict,
+      "owner": media["owner"],
+      "name": f"{predict} [{self['max_length']}]",
+      "processor": self.name()
+    })
+
+    return media
+
+  def content_type(self):
+    return "image"
+
+  def name(self):
+    return "image-query"
+
+  def is_append_processor(self):
+    return True
+
+
 class ImageDiffusionImageText(TPUBaseProcessor):
   def __init__(self, processor_config, max_progress) -> None:
     super().__init__(processor_config, max_progress)

{sciveo-0.1.39 → sciveo-0.1.40}/sciveo/ml/images/description.py

@@ -33,12 +33,14 @@ class ImageToText(BaseImageML):
     self.models = [
       ["GIT", "softel/git-base-v1.0", "auto"],
       ["GIT", "softel/git-large-v1.0", "auto"],
+      ["CAPTION", "softel/image-instruct-v2.0", "auto"],
       ["BLIP2", "softel/blip2-opt-2.7b-v1.0", torch.float16],
       # ["BLIP2", "softel/blip2-opt-6.7b-v1.0", torch.float16],
     ]
 
     model_config = self.models[model_id]
     self.dtype = model_config[2]
+    self.model_type = model_config[0]
 
     if model_config[0] == "GIT":
       self.pipe = AutoProcessor.from_pretrained(model_config[1], cache_dir=self.cache_dir)
@@ -46,6 +48,9 @@ class ImageToText(BaseImageML):
     elif model_config[0] == "BLIP2":
       self.pipe = Blip2Processor.from_pretrained(model_config[1], cache_dir=self.cache_dir)
       self.model = Blip2ForConditionalGeneration.from_pretrained(model_config[1], torch_dtype=self.dtype, device_map="auto", cache_dir=self.cache_dir)
+    elif model_config[0] == "CAPTION":
+      self.model = AutoModelForCausalLM.from_pretrained(model_config[1], torch_dtype=self.dtype, device_map="auto", cache_dir=self.cache_dir, trust_remote_code=True)
+      self.pipe = self.model
 
     debug("model name", model_config[1], "on device", self.device, "dtype", self.dtype, self.model.dtype)
     self.dtype = self.model.dtype
@@ -53,12 +58,49 @@ class ImageToText(BaseImageML):
   def predict(self, images):
     images = self.load(images)
 
-    pixel_values = self.pipe(images=images, return_tensors="pt").pixel_values.to(self.device, self.dtype)
-    ids = self.model.generate(pixel_values=pixel_values, max_length=self.max_length)
-    prediction = self.pipe.batch_decode(ids, skip_special_tokens=True)
+    if self.model_type == "CAPTION":
+      prediction = []
+      if self.max_length < 64:
+        predict_length = "short"
+      else:
+        predict_length = "normal"
+      for image in images:
+        prediction.append(self.model.caption(image, length=predict_length))
+    else:
+      pixel_values = self.pipe(images=images, return_tensors="pt").pixel_values.to(self.device, self.dtype)
+      ids = self.model.generate(pixel_values=pixel_values, max_length=self.max_length)
+      prediction = self.pipe.batch_decode(ids, skip_special_tokens=True)
 
-    del ids
-    del pixel_values
+      del ids
+      del pixel_values
 
     # debug("image description", prediction)
     return prediction
+
+
+class ImageQuery(BaseImageML):
+  def __init__(self, query, model_id, max_length=64, cache_dir=None, device=None) -> None:
+    super().__init__(model_id, cache_dir, device)
+    self.query = query
+    self.max_length = max_length
+
+    self.models = [
+      ["CAPTION", "softel/image-instruct-v2.0", "auto"],
+      # ["BLIP2", "softel/blip2-opt-2.7b-v1.0", torch.float16],
+      # ["BLIP2", "softel/blip2-opt-6.7b-v1.0", torch.float16],
+    ]
+
+    model_config = self.models[model_id]
+    self.dtype = model_config[2]
+    self.model_type = model_config[0]
+
+    if model_config[0] == "CAPTION":
+      self.model = AutoModelForCausalLM.from_pretrained(model_config[1], torch_dtype=self.dtype, device_map="auto", cache_dir=self.cache_dir, trust_remote_code=True)
+      self.pipe = self.model
+
+    debug("model name", model_config[1], "on device", self.device, "dtype", self.dtype, self.model.dtype)
+    self.dtype = self.model.dtype
+
+  def predict_one(self, x):
+    image = self.load_image(x)
+    return self.model.query(image, self.query)["answer"]

sciveo-0.1.40/sciveo/ml/nlp/tokenizers/bpe.py

@@ -0,0 +1,76 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import re
+
+from sciveo.tools.logger import *
+
+
+class BPETokenizer:
+  def __init__(self, max_size):
+    self.initial_tokens = 256
+    self.max_size = max_size
+    self.max_merges = max_size - self.initial_tokens
+    self.vocab = {}
+    self.merges = {}
+
+  def encode(self, text):
+    tokens = list(map(int, text.encode("utf-8")))
+    l1 = len(tokens)
+    for k, v in self.merges.items():
+      self.merge(tokens, k, v)
+    debug(f"encoded ratio {len(tokens) / l1:.2}x")
+    return tokens
+
+  def decode_token(self, token):
+    if token not in self.vocab:
+      return [token]
+
+    bigram = self.vocab[token]
+    return self.decode_token(bigram[0]) + self.decode_token(bigram[1])
+
+  def decode(self, tokens):
+    decoded = []
+    for token in tokens:
+      decoded += self.decode_token(token)
+    return bytes(decoded).decode("utf-8", errors="replace")
+
+  def train(self, text, debug_step=100):
+    tokens = list(map(int, text.encode("utf-8")))
+    token_id = self.initial_tokens
+    debug("max_merges", self.max_merges)
+    while(len(self.merges) < self.max_merges):
+      current_counts = self.counts(tokens)
+      bigram = max(current_counts, key=current_counts.get)
+      self.merge(tokens, bigram, token_id)
+      self.merges[bigram] = token_id
+      self.vocab[token_id] = bigram
+      token_id += 1
+      if len(self.merges) % debug_step == 0:
+        debug("train", f"{len(self.merges)}/{self.max_merges}")
+
+  def fit(self, x):
+    return self.train(x)
+
+  def counts(self, tokens):
+    result = {}
+    for bigram in zip(tokens, tokens[1:]):
+      result.setdefault(bigram, 0)
+      result[bigram] += 1
+    return result
+
+  def merge(self, tokens, bigram, token_id):
+    i = 0
+    while i < len(tokens) - 1:
+      if tokens[i] == bigram[0] and tokens[i + 1] == bigram[1]:
+        tokens[i] = token_id
+        del tokens[i + 1]
+      i += 1

sciveo-0.1.40/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.40'

{sciveo-0.1.39 → sciveo-0.1.40}/sciveo.egg-info/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.39
+Version: 0.1.40
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net
 Provides-Extra: media
 Provides-Extra: media-server
 Provides-Extra: media-ml
+Provides-Extra: media-ml-all
 Provides-Extra: all
 Provides-Extra: media-all
 

{sciveo-0.1.39 → sciveo-0.1.40}/sciveo.egg-info/SOURCES.txt

@@ -100,6 +100,8 @@ sciveo/ml/images/tools.py
 sciveo/ml/images/transformers.py
 sciveo/ml/nlp/__init__.py
 sciveo/ml/nlp/embeddings.py
+sciveo/ml/nlp/tokenizers/__init__.py
+sciveo/ml/nlp/tokenizers/bpe.py
 sciveo/ml/video/__init__.py
 sciveo/ml/video/description.py
 sciveo/monitoring/__init__.py
@@ -136,4 +138,5 @@ test/test_configuration.py
 test/test_crypto.py
 test/test_monitoring.py
 test/test_runner.py
-test/test_sampling.py
+test/test_sampling.py
+test/test_tokenizers.py

{sciveo-0.1.39 → sciveo-0.1.40}/sciveo.egg-info/requires.txt

@@ -69,6 +69,10 @@ accelerate>=0.0.0
 annoy>=0.0.0
 ultralytics>=0.0.0
 
+[media-ml-all]
+pyvips>=0.0.0
+einops>=0.0.0
+
 [media-server]
 fastapi
 uvicorn[standard]

{sciveo-0.1.39 → sciveo-0.1.40}/setup.py

@@ -30,7 +30,11 @@ extras_require = {
         'tensorflow>=0.0.0', 'keras>=0.0.0',
         'torch>=0.0.0', 'torchvision>=0.0.0',
         'diffusers>=0.0.0', 'transformers>=0.0.0', 'accelerate>=0.0.0', 'annoy>=0.0.0',
-        'ultralytics>=0.0.0'
+        'ultralytics>=0.0.0',
+      ],
+      'media-ml-all': [
+        'pyvips>=0.0.0',
+        'einops>=0.0.0'
       ]
 }
 

sciveo-0.1.40/test/test_tokenizers.py

@@ -0,0 +1,34 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import math
+import unittest
+
+from sciveo.ml.nlp.tokenizers.bpe import *
+
+
+class TestTokenizers(unittest.TestCase):
+  def test_BPE(self):
+    text = "節樂，《漢語大詞典》一則：「《史記．樂書》：凡作樂者，所以節槳。張守義正義：音洛，言不樂至荒淫也, 網站有中、英文版本，也有繁、簡體版，可通過每頁左上角的連結隨時調整 Ｕｎｉｃｏｄｅ! 🅤🅝🅘🅒🅞🅓🅔‽ 🇺‌🇳‌🇮‌🇨‌🇴‌🇩‌🇪! 😄 The very name strikes fear and awe into the hearts of programmers worldwide. We all know we ought to “support Unicode” in our software (whatever that means—like using wchar_t for all the strings, right?)"
+    text += "Using a row in the above table to encode a code point less than 'First code point' (thus using more bytes than necessary) is termed an overlong encoding. These are a security problem because they allow the same code point to be encoded in multiple ways. Overlong encodings (of ../ for example) have been used to bypass security validations in high-profile products including Microsoft's IIS web server[14] and Apache's Tomcat servlet container.[15] Overlong encodings should therefore be considered an error and never decoded. Modified UTF-8 allows an overlong encoding of U+0000."
+
+    T = BPETokenizer(max_size=512)
+    T.train(text)
+
+    t = "你好世界，美好的一天"
+    self.assertTrue(T.decode(T.encode(t)) == t)
+
+    t = "hello world and testing"
+    self.assertTrue(T.decode(T.encode(t)) == t)
+
+
+if __name__ == '__main__':
+  unittest.main()

sciveo-0.1.39/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.39'