MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/integrations/handlers/huggingface_handler/huggingface_handler.py

@@ -0,0 +1,383 @@
+from typing import Dict, Optional
+
+import pandas as pd
+import transformers
+from huggingface_hub import HfApi
+
+from mindsdb.integrations.handlers.huggingface_handler.settings import FINETUNE_MAP
+from mindsdb.integrations.libs.base import BaseMLEngine
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+class HuggingFaceHandler(BaseMLEngine):
+    name = "huggingface"
+
+    @staticmethod
+    def create_validation(target, args=None, **kwargs):
+
+        if "using" in args:
+            args = args["using"]
+
+        hf_api = HfApi()
+
+        # check model is pytorch based
+        metadata = hf_api.model_info(args["model_name"])
+        if "pytorch" not in metadata.tags:
+            raise Exception(
+                "Currently only PyTorch models are supported (https://huggingface.co/models?library=pytorch&sort=downloads). To request another library, please contact us on our community slack (https://mindsdb.com/joincommunity)."
+            )
+
+        # check model task
+        supported_tasks = [
+            "text-classification",
+            "text-generation",
+            "zero-shot-classification",
+            "translation",
+            "summarization",
+            "text2text-generation",
+            "fill-mask",
+        ]
+
+        if metadata.pipeline_tag not in supported_tasks:
+            raise Exception(
+                f'Not supported task for model: {metadata.pipeline_tag}.\
+             Should be one of {", ".join(supported_tasks)}'
+            )
+
+        if "task" not in args:
+            args["task"] = metadata.pipeline_tag
+        elif args["task"] != metadata.pipeline_tag:
+            raise Exception(
+                f'Task mismatch for model: {args["task"]}!={metadata.pipeline_tag}'
+            )
+
+        input_keys = list(args.keys())
+
+        # task, model_name, input_column is essential
+        for key in ["task", "model_name", "input_column"]:
+            if key not in args:
+                raise Exception(f'Parameter "{key}" is required')
+            input_keys.remove(key)
+
+        # check tasks input
+
+        if args["task"] == "zero-shot-classification":
+            key = "candidate_labels"
+            if key not in args:
+                raise Exception(
+                    '"candidate_labels" is required for zero-shot-classification'
+                )
+            input_keys.remove(key)
+
+        if args["task"] == "translation":
+            keys = ["lang_input", "lang_output"]
+            for key in keys:
+                if key not in args:
+                    raise Exception(f"{key} is required for translation")
+                input_keys.remove(key)
+
+        if args["task"] == "summarization":
+            keys = ["min_output_length", "max_output_length"]
+            for key in keys:
+                if key not in args:
+                    raise Exception(f"{key} is required for summarization")
+                input_keys.remove(key)
+
+        # optional keys
+        for key in ["labels", "max_length", "truncation_policy"]:
+            if key in input_keys:
+                input_keys.remove(key)
+
+        if len(input_keys) > 0:
+            raise Exception(f'Not expected parameters: {", ".join(input_keys)}')
+
+    def create(self, target, args=None, **kwargs):
+        # TODO change BaseMLEngine api?
+        if "using" in args:
+            args = args["using"]
+
+        args["target"] = target
+
+        model_name = args["model_name"]
+        hf_model_storage_path = self.engine_storage.folder_get(model_name)  # real
+
+        if args["task"] == "translation":
+            args[
+                "task_proper"
+            ] = f"translation_{args['lang_input']}_to_{args['lang_output']}"
+        else:
+            args["task_proper"] = args["task"]
+
+        logger.debug(f"Checking file system for {model_name}...")
+
+        ####
+        # Check if pipeline has already been downloaded
+        try:
+            pipeline = transformers.pipeline(task=args['task_proper'], model=hf_model_storage_path,
+                                             tokenizer=hf_model_storage_path)
+            logger.debug(f'Model already downloaded!')
+        ####
+        # Otherwise download it
+        except (ValueError, OSError):
+            try:
+                logger.debug(f"Downloading {model_name}...")
+                pipeline = transformers.pipeline(task=args['task_proper'], model=model_name)
+
+                pipeline.save_pretrained(hf_model_storage_path)
+
+                logger.debug(f"Saved to {hf_model_storage_path}")
+            except Exception:
+                raise Exception(
+                    "Error while downloading and setting up the model. Please try a different model. We're working on expanding the list of supported models, so we would appreciate it if you let us know about this in our community slack (https://mindsdb.com/joincommunity)."
+                )  # noqa
+        ####
+
+        if "max_length" in args:
+            pass
+        elif "max_position_embeddings" in pipeline.model.config.to_dict().keys():
+            args["max_length"] = pipeline.model.config.max_position_embeddings
+        elif "max_length" in pipeline.model.config.to_dict().keys():
+            args["max_length"] = pipeline.model.config.max_length
+        else:
+            logger.debug('No max_length found!')
+
+        labels_default = pipeline.model.config.id2label
+        labels_map = {}
+        if "labels" in args:
+            for num in labels_default.keys():
+                labels_map[labels_default[num]] = args["labels"][num]
+            args["labels_map"] = labels_map
+        else:
+            for num in labels_default.keys():
+                labels_map[labels_default[num]] = labels_default[num]
+            args["labels_map"] = labels_map
+
+        ###### store and persist in model folder
+        self.model_storage.json_set("args", args)
+
+        ###### persist changes to handler folder
+        self.engine_storage.folder_sync(model_name)
+
+    # todo move infer tasks to a seperate file
+    def predict_text_classification(self, pipeline, item, args):
+        top_k = args.get("top_k", 1000)
+
+        result = pipeline(
+            [item], top_k=top_k, truncation=True, max_length=args["max_length"]
+        )[0]
+
+        final = {}
+        explain = {}
+        if type(result) == dict:
+            result = [result]
+        final[args["target"]] = args["labels_map"][result[0]["label"]]
+        for elem in result:
+            if args["labels_map"]:
+                explain[args["labels_map"][elem["label"]]] = elem["score"]
+            else:
+                explain[elem["label"]] = elem["score"]
+        final[f"{args['target']}_explain"] = explain
+        return final
+
+    def predict_text_generation(self, pipeline, item, args):
+        result = pipeline([item], max_length=args["max_length"])[0]
+
+        final = {}
+        final[args["target"]] = result["generated_text"]
+
+        return final
+
+    def predict_zero_shot(self, pipeline, item, args):
+        top_k = args.get("top_k", 1000)
+
+        result = pipeline(
+            [item],
+            candidate_labels=args["candidate_labels"],
+            truncation=True,
+            top_k=top_k,
+            max_length=args["max_length"],
+        )[0]
+
+        final = {}
+        final[args["target"]] = result["labels"][0]
+
+        explain = dict(zip(result["labels"], result["scores"]))
+        final[f"{args['target']}_explain"] = explain
+
+        return final
+
+    def predict_translation(self, pipeline, item, args):
+        result = pipeline([item], max_length=args["max_length"])[0]
+
+        final = {}
+        final[args["target"]] = result["translation_text"]
+
+        return final
+
+    def predict_summarization(self, pipeline, item, args):
+        result = pipeline(
+            [item],
+            min_length=args["min_output_length"],
+            max_length=args["max_output_length"],
+        )[0]
+
+        final = {}
+        final[args["target"]] = result["summary_text"]
+
+        return final
+
+    def predict_text2text(self, pipeline, item, args):
+        result = pipeline([item], max_length=args["max_length"])[0]
+
+        final = {}
+        final[args["target"]] = result["generated_text"]
+
+        return final
+
+    def predict_fill_mask(self, pipeline, item, args):
+        result = pipeline([item])[0]
+
+        final = {}
+        final[args["target"]] = result[0]["sequence"]
+        explain = {elem["sequence"]: elem["score"] for elem in result}
+        final[f"{args['target']}_explain"] = explain
+
+        return final
+
+    def predict(self, df, args=None):
+
+        fnc_list = {
+            "text-classification": self.predict_text_classification,
+            "text-generation": self.predict_text_generation,
+            "zero-shot-classification": self.predict_zero_shot,
+            "translation": self.predict_translation,
+            "summarization": self.predict_summarization,
+            "fill-mask": self.predict_fill_mask,
+        }
+
+        ###### get stuff from model folder
+        args = self.model_storage.json_get("args")
+
+        task = args["task"]
+
+        if task not in fnc_list:
+            raise RuntimeError(f"Unknown task: {task}")
+
+        fnc = fnc_list[task]
+
+        try:
+            # load from model storage (finetuned models will use this)
+            hf_model_storage_path = self.model_storage.folder_get(
+                args["model_name"]
+            )
+            pipeline = transformers.pipeline(
+                task=args["task_proper"],
+                model=hf_model_storage_path,
+                tokenizer=hf_model_storage_path,
+            )
+        except (ValueError, OSError):
+            # load from engine storage (i.e. 'common' models)
+            hf_model_storage_path = self.engine_storage.folder_get(
+                args["model_name"]
+            )
+            pipeline = transformers.pipeline(
+                task=args["task_proper"],
+                model=hf_model_storage_path,
+                tokenizer=hf_model_storage_path,
+            )
+
+        input_column = args["input_column"]
+        if input_column not in df.columns:
+            raise RuntimeError(f'Column "{input_column}" not found in input data')
+        input_list = df[input_column]
+
+        max_tokens = pipeline.tokenizer.model_max_length
+
+        results = []
+        for item in input_list:
+            if max_tokens is not None:
+                tokens = pipeline.tokenizer.encode(item)
+                if len(tokens) > max_tokens:
+                    truncation_policy = args.get("truncation_policy", "strict")
+                    if truncation_policy == "strict":
+                        results.append(
+                            {
+                                "error": f"Tokens count exceed model limit: {len(tokens)} > {max_tokens}"
+                            }
+                        )
+                        continue
+                    elif truncation_policy == "left":
+                        tokens = tokens[
+                            -max_tokens + 1 : -1
+                        ]  # cut 2 empty tokens from left and right
+                    else:
+                        tokens = tokens[
+                            1 : max_tokens - 1
+                        ]  # cut 2 empty tokens from left and right
+
+                    item = pipeline.tokenizer.decode(tokens)
+
+            item = str(item)
+            try:
+                result = fnc(pipeline, item, args)
+            except Exception as e:
+                msg = str(e).strip()
+                if msg == "":
+                    msg = e.__class__.__name__
+                result = {"error": msg}
+            results.append(result)
+
+        pred_df = pd.DataFrame(results)
+
+        return pred_df
+
+    def describe(self, attribute: Optional[str] = None) -> pd.DataFrame:
+        args = self.model_storage.json_get("args")
+        if attribute == "args":
+            return pd.DataFrame(args.items(), columns=["key", "value"])
+        elif attribute == "metadata":
+            hf_api = HfApi()
+            metadata = hf_api.model_info(args["model_name"])
+            data = metadata.__dict__
+            return pd.DataFrame(list(data.items()), columns=["key", "value"])
+        else:
+            tables = ["args", "metadata"]
+            return pd.DataFrame(tables, columns=["tables"])
+
+    def finetune(
+        self, df: Optional[pd.DataFrame] = None, args: Optional[Dict] = None
+    ) -> None:
+        finetune_args = args if args else {}
+        args = self.base_model_storage.json_get("args")
+        args.update(finetune_args)
+
+        model_name = args["model_name"]
+        model_folder = self.model_storage.folder_get(model_name)
+        args["model_folder"] = model_folder
+        model_folder_name = model_folder.split("/")[-1]
+        task = args["task"]
+
+        if task not in FINETUNE_MAP:
+            raise KeyError(
+                f"{task} is not currently supported, please choose a supported task - {', '.join(FINETUNE_MAP)}"
+            )
+
+        tokenizer, trainer = FINETUNE_MAP[task](df, args)
+
+        try:
+            trainer.train()
+            trainer.save_model(
+                model_folder
+            )  # TODO: save entire pipeline instead https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.Pipeline.save_pretrained
+            tokenizer.save_pretrained(model_folder)
+
+            # persist changes
+            self.model_storage.json_set("args", args)
+            self.model_storage.folder_sync(model_folder_name)
+
+        except Exception as e:
+            err_str = f"Finetune failed with error: {str(e)}"
+            logger.debug(err_str)
+            raise Exception(err_str)

mindsdb/integrations/handlers/huggingface_handler/icon.svg

@@ -0,0 +1,24 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="40" height="37" viewBox="0 0 40 37" fill="none">
+  <g clip-path="url(#clip0_146_567)">
+    <path d="M19.8851 32.2976C28.6251 32.2976 35.68 25.4517 35.68 17.0325C35.68 8.6132 28.5966 1.76736 19.8851 1.76736C11.145 1.76736 4.09012 8.58559 4.09012 17.0325C4.09012 25.4793 11.145 32.2976 19.8851 32.2976Z" fill="#FFD21E"/>
+    <path d="M35.6814 17.0328C35.6814 8.5859 28.598 1.76767 19.8865 1.76767C11.1464 1.76767 4.09151 8.5859 4.09151 17.0328C4.09151 25.4797 11.175 32.2979 19.8865 32.2979C28.6265 32.2979 35.6814 25.4797 35.6814 17.0328ZM2.26352 17.0328C2.26352 7.61976 10.1467 0.00100708 19.8865 0.00100708C29.6262 0.00100708 37.5094 7.61976 37.5094 17.0328C37.5094 26.4458 29.6262 34.0646 19.8865 34.0646C10.1467 34.0646 2.26352 26.4458 2.26352 17.0328Z" fill="#FF9D0B"/>
+    <path d="M25.0264 12.8656C25.5976 13.0588 25.8261 14.2182 26.4259 13.9145C27.5399 13.3348 27.9683 12.0098 27.3685 10.9333C26.7687 9.8567 25.3977 9.44264 24.2838 10.0223C23.1698 10.602 22.7414 11.927 23.3412 13.0036C23.6268 13.5281 24.5123 12.6999 25.0264 12.8656Z" fill="#3A3B45"/>
+    <path d="M14.3148 12.8656C13.7435 13.0588 13.515 14.2182 12.9152 13.9145C11.8013 13.3348 11.3729 12.0098 11.9727 10.9333C12.5725 9.8567 13.9435 9.44264 15.0574 10.0223C16.1713 10.602 16.5998 11.927 15.9999 13.0036C15.7143 13.5281 14.8289 12.6999 14.3148 12.8656Z" fill="#3A3B45"/>
+    <path d="M19.7716 23.4095C24.2274 23.4095 25.684 19.5449 25.684 17.585C25.684 16.5637 24.97 16.8673 23.8275 17.4194C22.7707 17.9163 21.3426 18.634 19.7716 18.634C16.5155 18.634 13.8593 15.5975 13.8593 17.585C13.8593 19.5725 15.3159 23.4095 19.7716 23.4095Z" fill="#3A3B45"/>
+    <mask id="mask0_146_567" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="13" y="16" width="13" height="8">
+      <path d="M19.7716 23.4095C24.2274 23.4095 25.684 19.5449 25.684 17.585C25.684 16.5637 24.97 16.8673 23.8275 17.4194C22.7707 17.9163 21.3426 18.634 19.7716 18.634C16.5155 18.634 13.8593 15.5975 13.8593 17.585C13.8593 19.5725 15.3159 23.4095 19.7716 23.4095Z" fill="white"/>
+    </mask>
+    <g mask="url(#mask0_146_567)">
+      <path d="M19.8872 27.9086C22.0579 27.9086 23.8288 26.1972 23.8288 24.0992C23.8288 22.4706 22.7434 21.0628 21.2582 20.5107C21.201 20.4831 21.1439 20.4831 21.0868 20.4555C20.7155 20.3451 20.3156 21.5873 19.8872 21.5873C19.4873 21.5873 19.116 20.3175 18.7733 20.4279C17.1452 20.8972 15.9456 22.3602 15.9456 24.0716C15.9456 26.1972 17.7164 27.9086 19.8872 27.9086Z" fill="#F94040"/>
+    </g>
+    <path d="M6.37511 19.7652C5.63249 19.7652 4.97555 20.0688 4.51856 20.5933C4.23293 20.9246 3.94731 21.4491 3.91875 22.2496C3.60456 22.1668 3.31894 22.1116 3.03332 22.1116C2.31926 22.1116 1.69089 22.36 1.23389 22.8569C0.662645 23.4642 0.405585 24.2095 0.519834 24.9548C0.576958 25.3136 0.691207 25.6173 0.891143 25.9209C0.491271 26.2246 0.205649 26.6386 0.0913994 27.1631C-0.0228498 27.5496 -0.137099 28.4053 0.462709 29.2334C0.434147 29.2886 0.405585 29.3438 0.348461 29.3991C0.00571296 30.0339 -0.0228495 30.7517 0.291336 31.4418C0.748332 32.4631 1.91939 33.2636 4.17581 34.1194C5.57536 34.6438 6.86066 35.0027 6.86066 35.0027C8.71721 35.472 10.3738 35.6928 11.8305 35.6928C14.4868 35.6928 16.4005 34.8923 17.4858 33.3465C19.2567 30.8345 18.9996 28.5433 16.7147 26.3626C15.4579 25.148 14.601 23.3537 14.4297 22.9397C14.0869 21.7803 13.1444 20.4829 11.602 20.4829C11.4592 20.4829 11.3449 20.4829 11.2021 20.5105C10.5166 20.6209 9.91683 20.9798 9.51696 21.5595C9.05996 21.0074 8.60296 20.5657 8.20309 20.3173C7.57472 19.9584 6.97491 19.7652 6.37511 19.7652ZM6.37511 21.5319C6.6036 21.5319 6.88923 21.6147 7.20341 21.8079C8.17453 22.4152 10.0311 25.5069 10.7166 26.7215C10.9451 27.1355 11.3449 27.3011 11.6877 27.3011C12.4017 27.3011 12.9444 26.6386 11.7448 25.7829C9.97395 24.4855 10.6023 22.3876 11.4306 22.2496C11.4592 22.2496 11.5163 22.2496 11.5449 22.2496C12.3161 22.2496 12.6588 23.547 12.6588 23.547C12.6588 23.547 13.6585 25.9761 15.3722 27.6324C17.086 29.2886 17.1717 30.6136 15.9149 32.4079C15.058 33.6225 13.43 33.9813 11.7448 33.9813C10.0025 33.9813 8.23165 33.5949 7.23197 33.3465C7.17485 33.3465 1.11964 31.6902 1.89082 30.2548C2.03364 30.0063 2.23357 29.9235 2.49063 29.9235C3.576 29.9235 5.5468 31.4694 6.37511 31.4694C6.57504 31.4694 6.68929 31.3866 6.74641 31.1933C7.11772 29.9511 1.26245 29.4267 1.74801 27.6048C1.8337 27.2735 2.0622 27.1631 2.40495 27.1631C3.83306 27.1631 7.03204 29.5923 7.71753 29.5923C7.77466 29.5923 7.80322 29.5647 7.83178 29.5371C8.17453 29.0126 7.97459 28.6537 5.60392 27.2459C3.23325 25.8657 1.54808 25.01 2.51919 24.0162C2.63344 23.9058 2.77626 23.8506 2.97619 23.8506C4.40431 23.8506 7.83178 26.8595 7.83178 26.8595C7.83178 26.8595 8.74577 27.7704 9.31702 27.7704C9.43127 27.7704 9.54552 27.7152 9.63121 27.6048C10.0311 26.9699 5.97523 23.9886 5.74673 22.7741C5.63248 21.9459 5.88955 21.5319 6.37511 21.5319Z" fill="#FF9D0B"/>
+    <path d="M15.9737 32.38C17.2305 30.6133 17.1448 29.2883 15.4311 27.6044C13.7173 25.9482 12.7176 23.519 12.7176 23.519C12.7176 23.519 12.3463 22.1112 11.4895 22.2492C10.6326 22.3873 10.0042 24.4852 11.8036 25.7826C13.5745 27.08 11.4609 27.9357 10.7754 26.7487C10.0899 25.5617 8.2048 22.4425 7.26224 21.8352C6.29113 21.2279 5.60563 21.5867 5.83413 22.8013C6.06263 24.0159 10.1185 26.9971 9.7186 27.632C9.31873 28.2669 7.91918 26.8867 7.91918 26.8867C7.91918 26.8867 3.57771 23.0498 2.60659 24.0435C1.66403 25.0373 3.32065 25.8654 5.69132 27.2732C8.06199 28.6534 8.26192 29.0399 7.91918 29.5643C7.57643 30.0888 2.34953 25.8102 1.83541 27.632C1.34985 29.4539 7.17656 29.9784 6.83381 31.2206C6.4625 32.4628 2.72084 28.8466 1.94966 30.2544C1.17847 31.6622 7.23368 33.3185 7.29081 33.3185C9.26161 33.8154 14.2314 34.8643 15.9737 32.38Z" fill="#FFD21E"/>
+    <path d="M24.0261 32.38C22.7694 30.6133 22.8551 29.2883 24.5688 27.6044C26.2826 25.9482 27.2822 23.519 27.2822 23.519C27.2822 23.519 27.6536 22.1112 28.5104 22.2492C29.3673 22.3873 29.9957 24.4852 28.1962 25.7826C26.4254 27.08 28.539 27.9357 29.2245 26.7487C29.91 25.5341 31.7951 22.4425 32.7376 21.8352C33.7088 21.2279 34.3943 21.5867 34.1658 22.8013C33.9373 24.0159 29.8814 26.9971 30.2813 27.632C30.6812 28.2669 32.0807 26.8867 32.0807 26.8867C32.0807 26.8867 36.4222 23.0498 37.3933 24.0435C38.3359 25.0373 36.6792 25.8654 34.3086 27.2732C31.9379 28.6534 31.738 29.0399 32.0807 29.5643C32.4235 30.0888 37.6504 25.8102 38.1645 27.632C38.65 29.4539 32.8233 29.9784 33.1661 31.2206C33.5374 32.4628 37.279 28.8466 38.0502 30.2544C38.8214 31.6623 32.7376 33.3185 32.7091 33.3461C30.7383 33.8154 25.7684 34.8643 24.0261 32.38Z" fill="#FFD21E"/>
+    <path d="M33.6244 19.7652C34.3671 19.7652 35.024 20.0688 35.481 20.5933C35.7666 20.9246 36.0522 21.4491 36.0808 22.2496C36.395 22.1668 36.6806 22.1116 36.9662 22.1116C37.6803 22.1116 38.3087 22.36 38.7656 22.8569C39.3369 23.4642 39.594 24.2095 39.4797 24.9548C39.4226 25.3136 39.3083 25.6173 39.1084 25.9209C39.5083 26.2246 39.7939 26.6386 39.9081 27.1631C40.0224 27.5496 40.1366 28.4053 39.5368 29.2334C39.5654 29.2886 39.594 29.3438 39.6511 29.3991C39.9938 30.0339 40.0224 30.7517 39.7082 31.4418C39.2512 32.4631 38.0802 33.2636 35.8237 34.1194C34.4242 34.6438 33.1389 35.0027 33.1389 35.0027C31.2823 35.472 29.6257 35.6928 28.169 35.6928C25.5127 35.6928 23.5991 34.8923 22.5137 33.3465C20.7428 30.8345 20.9999 28.5433 23.2849 26.3626C24.5416 25.148 25.3985 23.3537 25.5699 22.9397C25.9126 21.7803 26.8552 20.4829 28.3975 20.4829C28.5403 20.4829 28.6546 20.4829 28.7974 20.5105C29.4829 20.6209 30.0827 20.9798 30.4826 21.5595C30.9396 21.0074 31.3966 20.5657 31.7965 20.3173C32.4248 19.9584 33.0246 19.7652 33.6244 19.7652ZM33.6244 21.5319C33.3959 21.5319 33.1103 21.6147 32.7961 21.8079C31.825 22.4152 29.9685 25.5069 29.283 26.7215C29.0545 27.1355 28.6546 27.3011 28.3119 27.3011C27.5978 27.3011 27.0551 26.6386 28.2547 25.7829C30.0256 24.4855 29.3972 22.3876 28.5689 22.2496C28.5403 22.2496 28.4832 22.2496 28.4547 22.2496C27.6835 22.2496 27.3407 23.547 27.3407 23.547C27.3407 23.547 26.341 25.9761 24.6273 27.6324C22.9136 29.2886 22.8279 30.6136 24.0846 32.4079C24.9415 33.6225 26.5695 33.9813 28.2547 33.9813C29.997 33.9813 31.7679 33.5949 32.7676 33.3465C32.8247 33.3465 38.8799 31.6902 38.1087 30.2548C37.9659 30.0063 37.766 29.9235 37.5089 29.9235C36.4235 29.9235 34.4527 31.4694 33.6244 31.4694C33.4245 31.4694 33.3102 31.3866 33.2531 31.1933C32.8818 29.9511 38.7371 29.4267 38.2515 27.6048C38.1658 27.2735 37.9373 27.1631 37.5946 27.1631C36.1665 27.1631 32.9675 29.5923 32.282 29.5923C32.2249 29.5923 32.1963 29.5647 32.1678 29.5371C31.825 29.0126 32.0249 28.6537 34.3956 27.2459C36.7663 25.8657 38.4515 25.01 37.4803 24.0162C37.3661 23.9058 37.2233 23.8506 37.0234 23.8506C35.5952 23.8506 32.1678 26.8595 32.1678 26.8595C32.1678 26.8595 31.2538 27.7704 30.6825 27.7704C30.5683 27.7704 30.454 27.7152 30.3683 27.6048C29.9685 26.9699 34.0243 23.9886 34.2528 22.7741C34.3671 21.9459 34.11 21.5319 33.6244 21.5319Z" fill="#FF9D0B"/>
+  </g>
+  <defs>
+    <clipPath id="clip0_146_567">
+      <rect width="40" height="36.3842" fill="white"/>
+    </clipPath>
+  </defs>
+</svg>

mindsdb/integrations/handlers/huggingface_handler/requirements.txt

@@ -0,0 +1,5 @@
+datasets==2.16.1
+evaluate
+torch
+nltk
+huggingface-hub

mindsdb/integrations/handlers/huggingface_handler/requirements_cpu.txt

@@ -0,0 +1,6 @@
+datasets==2.16.1
+evaluate
+nltk
+huggingface-hub
+# Needs to be installed with `pip install --extra-index-url https://download.pytorch.org/whl/ .[huggingface_cpu]`
+torch==2.2.0+cpu

mindsdb/integrations/handlers/huggingface_handler/settings.py

@@ -0,0 +1,27 @@
+from mindsdb.integrations.handlers.huggingface_handler.finetune import (
+    _finetune_cls,
+    _finetune_fill_mask,
+    _finetune_question_answering,
+    _finetune_summarization,
+    _finetune_text_generation,
+    _finetune_translate,
+)
+
+# todo once we have moved predict tasks functions into a separate function
+# PREDICT_MAP = {
+#             'text-classification': self.predict_text_classification,
+#             'zero-shot-classification': self.predict_zero_shot,
+#             'translation': self.predict_translation,
+#             'summarization': self.predict_summarization,
+#             'fill-mask': self.predict_fill_mask
+#         }
+
+FINETUNE_MAP = {
+    "text-classification": _finetune_cls,
+    "zero-shot-classification": _finetune_cls,
+    "translation": _finetune_translate,
+    "summarization": _finetune_summarization,
+    "fill-mask": _finetune_fill_mask,
+    "text-generation": _finetune_text_generation,
+    "question-answering": _finetune_question_answering,
+}

mindsdb/integrations/handlers/ibm_cos_handler/__about__.py

@@ -0,0 +1,9 @@
+__title__ = "MindsDB IBM COS Handler"
+__package_name__ = "mindsdb_ibm_cos_handler"
+__version__ = "0.0.1"
+__description__ = "MindsDB handler for IBM Cloud Object Storage"
+__author__ = "Ton Hoang Nguyen (Bill)"
+__github__ = "'https://github.com/mindsdb/mindsdb'"
+__pypi__ = "'https://github.com/mindsdb/mindsdb'"
+__license__ = "MIT"
+__copyright__ = "Copyright 2024 - mindsdb"

mindsdb/integrations/handlers/ibm_cos_handler/__init__.py

@@ -0,0 +1,30 @@
+from mindsdb.integrations.libs.const import HANDLER_TYPE
+
+from .__about__ import __version__ as version, __description__ as description
+from .connection_args import connection_args, connection_args_example
+
+try:
+    from .ibm_cos_handler import IBMCloudObjectStorageHandler as Handler
+
+    import_error = None
+except Exception as e:
+    Handler = None
+    import_error = e
+
+title = "IBM Cloud Object Storage"
+name = "ibm_cos"
+type = HANDLER_TYPE.DATA
+icon_path = "icon.svg"
+
+__all__ = [
+    "Handler",
+    "version",
+    "name",
+    "type",
+    "title",
+    "description",
+    "connection_args",
+    "connection_args_example",
+    "import_error",
+    "icon_path",
+]

mindsdb/integrations/handlers/ibm_cos_handler/connection_args.py

@@ -0,0 +1,38 @@
+from collections import OrderedDict
+from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE
+
+connection_args = OrderedDict(
+    cos_hmac_access_key_id={
+        "type": ARG_TYPE.PWD,
+        "description": "IBM COS HMAC Access Key ID.",
+        "required": True,
+        "label": "HMAC Access Key ID",
+        "secret": True,
+    },
+    cos_hmac_secret_access_key={
+        "type": ARG_TYPE.PWD,
+        "description": "IBM COS HMAC Secret Access Key.",
+        "required": True,
+        "label": "HMAC Secret Access Key",
+        "secret": True,
+    },
+    cos_endpoint_url={
+        "type": ARG_TYPE.STR,
+        "description": "IBM COS Endpoint URL (e.g., https://s3.eu-gb.cloud-object-storage.appdomain.cloud).",
+        "required": True,
+        "label": "Endpoint URL",
+    },
+    bucket={
+        "type": ARG_TYPE.STR,
+        "description": "IBM COS Bucket Name (Optional).",
+        "required": False,
+        "label": "Bucket Name",
+    },
+)
+
+connection_args_example = OrderedDict(
+    cos_hmac_access_key_id="YOUR_HMAC_ACCESS_KEY_ID",
+    cos_hmac_secret_access_key="YOUR_HMAC_SECRET_ACCESS_KEY",
+    cos_endpoint_url="https://s3.eu-gb.cloud-object-storage.appdomain.cloud",
+    bucket="YOUR_BUCKET_NAME",
+)