spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/training/conll.py

@@ -0,0 +1,150 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for CoNLL."""
+
+import pyspark
+
+from sparknlp.common import ReadAs
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class CoNLL(ExtendedJavaWrapper):
+    """Instantiates the class to read a CoNLL dataset.
+
+    The dataset should be in the format of `CoNLL 2003
+    <https://www.clips.uantwerpen.be/conll2003/ner/>`_ and needs to be specified
+    with :meth:`.readDataset`, which will create a dataframe with the data.
+
+    Can be used to train a :class:`NerDLApproach
+    <sparknlp.annotator.NerDLApproach>`.
+
+    **Input File Format**::
+
+        -DOCSTART- -X- -X- O
+
+        EU NNP B-NP B-ORG
+        rejects VBZ B-VP O
+        German JJ B-NP B-MISC
+        call NN I-NP O
+        to TO B-VP O
+        boycott VB I-VP O
+        British JJ B-NP B-MISC
+        lamb NN I-NP O
+        . . O O
+
+    Parameters
+    ----------
+    documentCol : str, optional
+        Name of the :class:`.DocumentAssembler` column, by default 'document'
+    sentenceCol : str, optional
+        Name of the :class:`.SentenceDetector` column, by default 'sentence'
+    tokenCol : str, optional
+        Name of the :class:`.Tokenizer` column, by default 'token'
+    posCol : str, optional
+        Name of the :class:`.PerceptronModel` column, by default 'pos'
+    conllLabelIndex : int, optional
+        Index of the label column in the dataset, by default 3
+    conllPosIndex : int, optional
+        Index of the POS tags in the dataset, by default 1
+    textCol : str, optional
+        Index of the text column in the dataset, by default 'text'
+    labelCol : str, optional
+        Name of the label column, by default 'label'
+    explodeSentences : bool, optional
+        Whether to explode sentences to separate rows, by default True
+    delimiter: str, optional
+        Delimiter used to separate columns inside CoNLL file
+    includeDocId: bool, optional
+        Whether to try and parse the document id from the third item in the -DOCSTART- line (X if not found)
+
+    Examples
+    --------
+    >>> from sparknlp.training import CoNLL
+    >>> trainingData = CoNLL().readDataset(spark, "src/test/resources/conll2003/eng.train")
+    >>> trainingData.selectExpr(
+    ...     "text",
+    ...     "token.result as tokens",
+    ...     "pos.result as pos",
+    ...     "label.result as label"
+    ... ).show(3, False)
+    +------------------------------------------------+----------------------------------------------------------+-------------------------------------+-----------------------------------------+
+    |text                                            |tokens                                                    |pos                                  |label                                    |
+    +------------------------------------------------+----------------------------------------------------------+-------------------------------------+-----------------------------------------+
+    |EU rejects German call to boycott British lamb .|[EU, rejects, German, call, to, boycott, British, lamb, .]|[NNP, VBZ, JJ, NN, TO, VB, JJ, NN, .]|[B-ORG, O, B-MISC, O, O, O, B-MISC, O, O]|
+    |Peter Blackburn                                 |[Peter, Blackburn]                                        |[NNP, NNP]                           |[B-PER, I-PER]                           |
+    |BRUSSELS 1996-08-22                             |[BRUSSELS, 1996-08-22]                                    |[NNP, CD]                            |[B-LOC, O]                               |
+    +------------------------------------------------+----------------------------------------------------------+-------------------------------------+-----------------------------------------+
+    """
+
+    def __init__(self,
+                 documentCol='document',
+                 sentenceCol='sentence',
+                 tokenCol='token',
+                 posCol='pos',
+                 conllLabelIndex=3,
+                 conllPosIndex=1,
+                 conllDocIdCol="doc_id",
+                 textCol='text',
+                 labelCol='label',
+                 explodeSentences=True,
+                 delimiter=' ',
+                 includeDocId=False
+                 ):
+        super(CoNLL, self).__init__("com.johnsnowlabs.nlp.training.CoNLL",
+                                    documentCol,
+                                    sentenceCol,
+                                    tokenCol,
+                                    posCol,
+                                    conllLabelIndex,
+                                    conllPosIndex,
+                                    conllDocIdCol,
+                                    textCol,
+                                    labelCol,
+                                    explodeSentences,
+                                    delimiter,
+                                    includeDocId)
+
+    def readDataset(self, spark, path, read_as=ReadAs.TEXT, partitions=8, storage_level=pyspark.StorageLevel.DISK_ONLY):
+        # ToDo Replace with std pyspark
+        """Reads the dataset from an external resource.
+
+        Parameters
+        ----------
+        spark : :class:`pyspark.sql.SparkSession`
+            Initiated Spark Session with Spark NLP
+        path : str
+            Path to the resource, it can take two forms; a path to a conll file, or a path to a folder containing multiple CoNLL files.
+            When the path points to a folder, the path must end in '*'.
+            Examples:
+                "/path/to/single/file.conll'
+                "/path/to/folder/containing/multiple/files/*'
+
+        read_as : str, optional
+            How to read the resource, by default ReadAs.TEXT
+        partitions : sets the minimum number of partitions for the case of lifting multiple files in parallel into a single dataframe. Defaults to 8.
+        storage_level : sets the persistence level according to PySpark definitions. Defaults to StorageLevel.DISK_ONLY. Applies only when lifting multiple files.
+        
+
+        Returns
+        -------
+        :class:`pyspark.sql.DataFrame`
+            Spark Dataframe with the data
+        """
+        jSession = spark._jsparkSession
+
+        jdf = self._java_obj.readDataset(jSession, path, read_as, partitions,
+                                         spark.sparkContext._getJavaStorageLevel(storage_level))
+        dataframe = self.getDataFrame(spark, jdf)
+        return dataframe
+

sparknlp/training/conllu.py

@@ -0,0 +1,103 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for CoNLLU."""
+
+from sparknlp.common import ReadAs
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class CoNLLU(ExtendedJavaWrapper):
+    """Instantiates the class to read a CoNLL-U dataset.
+
+    The dataset should be in the format of `CoNLL-U
+    <https://universaldependencies.org/format.html>`_ and needs to be specified
+    with :meth:`.readDataset`, which will create a dataframe with the data.
+
+    Can be used to train a :class:`DependencyParserApproach
+    <sparknlp.annotator.DependencyParserApproach>`
+
+    **Input File Format**::
+
+        # sent_id = 1
+        # text = They buy and sell books.
+        1   They     they    PRON    PRP    Case=Nom|Number=Plur               2   nsubj   2:nsubj|4:nsubj   _
+        2   buy      buy     VERB    VBP    Number=Plur|Person=3|Tense=Pres    0   root    0:root            _
+        3   and      and     CONJ    CC     _                                  4   cc      4:cc              _
+        4   sell     sell    VERB    VBP    Number=Plur|Person=3|Tense=Pres    2   conj    0:root|2:conj     _
+        5   books    book    NOUN    NNS    Number=Plur                        2   obj     2:obj|4:obj       SpaceAfter=No
+        6   .        .       PUNCT   .      _                                  2   punct   2:punct           _
+
+    Examples
+    --------
+    >>> from sparknlp.training import CoNLLU
+    >>> conlluFile = "src/test/resources/conllu/en.test.conllu"
+    >>> conllDataSet = CoNLLU(False).readDataset(spark, conlluFile)
+    >>> conllDataSet.selectExpr(
+    ...     "text",
+    ...     "form.result as form",
+    ...     "upos.result as upos",
+    ...     "xpos.result as xpos",
+    ...     "lemma.result as lemma"
+    ... ).show(1, False)
+    +---------------------------------------+----------------------------------------------+---------------------------------------------+------------------------------+--------------------------------------------+
+    |text                                   |form                                          |upos                                         |xpos                          |lemma                                       |
+    +---------------------------------------+----------------------------------------------+---------------------------------------------+------------------------------+--------------------------------------------+
+    |What if Google Morphed Into GoogleOS?  |[What, if, Google, Morphed, Into, GoogleOS, ?]|[PRON, SCONJ, PROPN, VERB, ADP, PROPN, PUNCT]|[WP, IN, NNP, VBD, IN, NNP, .]|[what, if, Google, morph, into, GoogleOS, ?]|
+    +---------------------------------------+----------------------------------------------+---------------------------------------------+------------------------------+--------------------------------------------+
+    """
+
+    def __init__(self,
+                 textCol='text',
+                 documentCol='document',
+                 sentenceCol='sentence',
+                 formCol='form',
+                 uposCol='upos',
+                 xposCol='xpos',
+                 lemmaCol='lemma',
+                 explodeSentences=True
+                 ):
+        super(CoNLLU, self).__init__("com.johnsnowlabs.nlp.training.CoNLLU",
+                                     textCol,
+                                     documentCol,
+                                     sentenceCol,
+                                     formCol,
+                                     uposCol,
+                                     xposCol,
+                                     lemmaCol,
+                                     explodeSentences)
+
+    def readDataset(self, spark, path, read_as=ReadAs.TEXT):
+        """Reads the dataset from an external resource.
+
+        Parameters
+        ----------
+        spark : :class:`pyspark.sql.SparkSession`
+            Initiated Spark Session with Spark NLP
+        path : str
+            Path to the resource
+        read_as : str, optional
+            How to read the resource, by default ReadAs.TEXT
+
+        Returns
+        -------
+        :class:`pyspark.sql.DataFrame`
+            Spark Dataframe with the data
+        """
+        # ToDo Replace with std pyspark
+        jSession = spark._jsparkSession
+
+        jdf = self._java_obj.readDataset(jSession, path, read_as)
+        dataframe = self.getDataFrame(spark, jdf)
+        return dataframe
+

sparknlp/training/pos.py

@@ -0,0 +1,103 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains helper classes for part-of-speech tagging."""
+
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class POS(ExtendedJavaWrapper):
+    """Helper class for creating DataFrames for training a part-of-speech
+    tagger.
+
+    The dataset needs to consist of sentences on each line, where each word is
+    delimited with its respective tag.
+
+    **Input File Format**::
+
+        A|DT few|JJ months|NNS ago|RB you|PRP received|VBD a|DT letter|NN
+
+
+    The sentence can then be parsed with :meth:`.readDataset` into a column with
+    annotations of type ``POS``.
+
+    Can be used to train a :class:`PerceptronApproach
+    <sparknlp.annotator.PerceptronApproach>`.
+
+    Examples
+    --------
+    In this example, the file ``test-training.txt`` has the content of the
+    sentence above.
+
+    >>> from sparknlp.training import POS
+    >>> pos = POS()
+    >>> path = "src/test/resources/anc-pos-corpus-small/test-training.txt"
+    >>> posDf = pos.readDataset(spark, path, "|", "tags")
+    >>> posDf.selectExpr("explode(tags) as tags").show(truncate=False)
+    +---------------------------------------------+
+    |tags                                         |
+    +---------------------------------------------+
+    |[pos, 0, 5, NNP, [word -> Pierre], []]       |
+    |[pos, 7, 12, NNP, [word -> Vinken], []]      |
+    |[pos, 14, 14, ,, [word -> ,], []]            |
+    |[pos, 16, 17, CD, [word -> 61], []]          |
+    |[pos, 19, 23, NNS, [word -> years], []]      |
+    |[pos, 25, 27, JJ, [word -> old], []]         |
+    |[pos, 29, 29, ,, [word -> ,], []]            |
+    |[pos, 31, 34, MD, [word -> will], []]        |
+    |[pos, 36, 39, VB, [word -> join], []]        |
+    |[pos, 41, 43, DT, [word -> the], []]         |
+    |[pos, 45, 49, NN, [word -> board], []]       |
+    |[pos, 51, 52, IN, [word -> as], []]          |
+    |[pos, 47, 47, DT, [word -> a], []]           |
+    |[pos, 56, 67, JJ, [word -> nonexecutive], []]|
+    |[pos, 69, 76, NN, [word -> director], []]    |
+    |[pos, 78, 81, NNP, [word -> Nov.], []]       |
+    |[pos, 83, 84, CD, [word -> 29], []]          |
+    |[pos, 81, 81, ., [word -> .], []]            |
+    +---------------------------------------------+
+    """
+
+    def __init__(self):
+        super(POS, self).__init__("com.johnsnowlabs.nlp.training.POS")
+
+    def readDataset(self, spark, path, delimiter="|", outputPosCol="tags", outputDocumentCol="document",
+                    outputTextCol="text"):
+        # ToDo Replace with std pyspark
+        """Reads the dataset from an external resource.
+
+        Parameters
+        ----------
+        spark : :class:`pyspark.sql.SparkSession`
+            Initiated Spark Session with Spark NLP
+        path : str
+            Path to the resource
+        delimiter : str, optional
+            Delimiter of word and POS, by default "|"
+        outputPosCol : str, optional
+            Name of the output POS column, by default "tags"
+        outputDocumentCol : str, optional
+            Name of the output document column, by default "document"
+        outputTextCol : str, optional
+            Name of the output text column, by default "text"
+
+        Returns
+        -------
+        :class:`pyspark.sql.DataFrame`
+            Spark Dataframe with the data
+        """
+        jSession = spark._jsparkSession
+
+        jdf = self._java_obj.readDataset(jSession, path, delimiter, outputPosCol, outputDocumentCol, outputTextCol)
+        dataframe = self.getDataFrame(spark, jdf)
+        return dataframe

sparknlp/training/pub_tator.py

@@ -0,0 +1,76 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains helper classes for PubTator datasets."""
+
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class PubTator(ExtendedJavaWrapper):
+    """The PubTator format includes medical papers’ titles, abstracts, and
+    tagged chunks.
+
+    For more information see `PubTator Docs
+    <http://bioportal.bioontology.org/ontologies/EDAM?p=classes&conceptid=format_3783>`_
+    and `MedMentions Docs <http://github.com/chanzuckerberg/MedMentions>`_.
+
+    :meth:`.readDataset` is used to create a Spark DataFrame from a PubTator
+    text file.
+
+    **Input File Format**::
+
+        25763772	0	5	DCTN4	T116,T123	C4308010
+        25763772	23	63	chronic Pseudomonas aeruginosa infection	T047	C0854135
+        25763772	67	82	cystic fibrosis	T047	C0010674
+        25763772	83	120	Pseudomonas aeruginosa (Pa) infection	T047	C0854135
+        25763772	124	139	cystic fibrosis	T047	C0010674
+
+    Examples
+    --------
+    >>> from sparknlp.training import PubTator
+    >>> pubTatorFile = "./src/test/resources/corpus_pubtator_sample.txt"
+    >>> pubTatorDataSet = PubTator().readDataset(spark, pubTatorFile)
+    >>> pubTatorDataSet.show(1)
+    +--------+--------------------+--------------------+--------------------+-----------------------+---------------------+-----------------------+
+    |  doc_id|      finished_token|        finished_pos|        finished_ner|finished_token_metadata|finished_pos_metadata|finished_label_metadata|
+    +--------+--------------------+--------------------+--------------------+-----------------------+---------------------+-----------------------+
+    |25763772|[DCTN4, as, a, mo...|[NNP, IN, DT, NN,...|[B-T116, O, O, O,...|   [[sentence, 0], [...| [[word, DCTN4], [...|   [[word, DCTN4], [...|
+    +--------+--------------------+--------------------+--------------------+-----------------------+---------------------+-----------------------+
+    """
+
+    def __init__(self):
+        super(PubTator, self).__init__("com.johnsnowlabs.nlp.training.PubTator")
+
+    def readDataset(self, spark, path, isPaddedToken=True):
+        # ToDo Replace with std pyspark
+        """Reads the dataset from an external resource.
+
+        Parameters
+        ----------
+        spark : :class:`pyspark.sql.SparkSession`
+            Initiated Spark Session with Spark NLP
+        path : str
+            Path to the resource
+        isPaddedToken : str, optional
+            Whether tokens are padded, by default True
+
+        Returns
+        -------
+        :class:`pyspark.sql.DataFrame`
+            Spark Dataframe with the data
+        """
+        jSession = spark._jsparkSession
+
+        jdf = self._java_obj.readDataset(jSession, path, isPaddedToken)
+        dataframe = self.getDataFrame(spark, jdf)
+        return dataframe

sparknlp/training/spacy_to_annotation.py

@@ -0,0 +1,57 @@
+#  Copyright 2017-2023 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from pyspark.sql import SparkSession
+
+from sparknlp.internal import ExtendedJavaWrapper
+
+
+class SpacyToAnnotation(ExtendedJavaWrapper):
+
+    """Helper class to load a list of tokens/sentences as JSON to Annotation.
+
+    The JSON will be in this format:
+        [
+         {
+            "tokens": ["Hello", "world", "!", "How", "are", "you", "today", "?", "I", "'m", "fine", "thanks", "."],
+            "token_spaces": [true, false, true, true, true, true, false, true, false, true, true, false, false],
+            "sentence_ends": [2, 7, 12]
+         }
+        ]
+
+    Examples
+    --------
+    >>> from sparknlp.training import SpacyToAnnotation
+    >>> result = SpacyToAnnotation().readDataset(spark, "src/test/resources/spacy-to-annotation/multi_doc_tokens.json")
+    >>> result.show(False)
+    +-------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |document                                                                             |sentence                                                                                                                                                                      |token                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+    +-------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[{document, 0, 55, John went to the store last night. He bought some bread., {}, []}]|[{document, 0, 33, John went to the store last night., {sentence -> 0}, []}, {document, 35, 55, He bought some bread., {sentence -> 1}, []}]                                  |[{token, 0, 3, John, {sentence -> 0}, []}, {token, 5, 8, went, {sentence -> 0}, []}, {token, 10, 11, to, {sentence -> 0}, []}, {token, 13, 15, the, {sentence -> 0}, []}, {token, 17, 21, store, {sentence -> 0}, []}, {token, 23, 26, last, {sentence -> 0}, []}, {token, 28, 32, night, {sentence -> 0}, []}, {token, 33, 33, ., {sentence -> 0}, []}, {token, 35, 36, He, {sentence -> 1}, []}, {token, 38, 43, bought, {sentence -> 1}, []}, {token, 45, 48, some, {sentence -> 1}, []}, {token, 50, 54, bread, {sentence -> 1}, []}, {token, 55, 55, ., {sentence -> 1}, []}]|
+    |[{document, 0, 47, Hello world! How are you today? I'm fine thanks., {}, []}]        |[{document, 0, 11, Hello world!, {sentence -> 0}, []}, {document, 13, 30, How are you today?, {sentence -> 1}, []}, {document, 32, 47, I'm fine thanks., {sentence -> 2}, []}]|[{token, 0, 4, Hello, {sentence -> 0}, []}, {token, 6, 10, world, {sentence -> 0}, []}, {token, 11, 11, !, {sentence -> 0}, []}, {token, 13, 15, How, {sentence -> 1}, []}, {token, 17, 19, are, {sentence -> 1}, []}, {token, 21, 23, you, {sentence -> 1}, []}, {token, 25, 29, today, {sentence -> 1}, []}, {token, 30, 30, ?, {sentence -> 1}, []}, {token, 32, 32, I, {sentence -> 2}, []}, {token, 33, 34, 'm, {sentence -> 2}, []}, {token, 36, 39, fine, {sentence -> 2}, []}, {token, 41, 46, thanks, {sentence -> 2}, []}, {token, 47, 47, ., {sentence -> 2}, []}]     |
+    +-------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+
+    """
+
+    def __init__(self):
+        super(SpacyToAnnotation, self).__init__("com.johnsnowlabs.nlp.training.SpacyToAnnotation")
+
+    def readJsonFile(self, spark, jsonFilePath, params=None):
+        if params is None:
+            params = {}
+
+        jSession = spark._jsparkSession
+
+        jdf = self._java_obj.readJsonFileJava(jSession, jsonFilePath, params)
+        annotation_dataset = self.getDataFrame(spark, jdf)
+        return annotation_dataset

sparknlp/training/tfgraphs.py

@@ -0,0 +1,5 @@
+from ._tf_graph_builders.graph_builders import TFGraphBuilderFactory
+from ._tf_graph_builders_1x.graph_builders import TFGraphBuilderFactory as TFGraphBuilderFactory1x
+
+tf_graph = TFGraphBuilderFactory()
+tf_graph_1x = TFGraphBuilderFactory1x()

sparknlp/upload_to_hub.py

@@ -0,0 +1,149 @@
+import requests
+import json
+from typing import List
+import sparknlp
+import os
+import zipfile
+
+
+class PushToHub:
+    list_of_tasks = [  # List  of available tasks in Modelhub
+        "Named Entity Recognition",
+        'Text Classification',
+        'Text Generation',
+        'Sentiment Analysis',
+        'Translation',
+        'Question Answering',
+        'Summarization',
+        'Sentence Detection',
+        'Embeddings',
+        'Language Detection',
+        'Stop Words Removal',
+        'Word Segmentation',
+        'Part of Speech Tagging',
+        'Lemmatization',
+        'Chunk Mapping',
+        'Spell Check',
+        'Dependency Parser',
+        'Pipeline Public']
+
+    def zip_directory(folder_path: str, zip_path: str):
+        """Zips folder for pushing to hub.
+
+        folder_path:Path to the folder to zip.
+        zip_path:Path of the zip file to create."""
+
+        with zipfile.ZipFile(zip_path, mode='w') as zipf:
+            len_dir_path = len(folder_path)
+            for root, _, files in os.walk(folder_path):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    zipf.write(file_path, file_path[len_dir_path:])
+
+    def unzip_directory(zip_path: str):
+        """Unzips Model to check for required files for upload.
+
+        Keyword Arguments:
+        zip_path:Zip Path to unzip.
+        """
+
+    def check_for_required_info(model_data: dict):
+        """Checks if the required fields exist in given dictionary  and fills any remaining fields.
+
+        Keyword Arguments: 
+        model_data: The model data to check.
+        """
+        
+        list_of_required_fields = ['name', 'task', 'language', 'pythonCode', 'model_zip_path']
+
+        if model_data['task'] not in PushToHub.list_of_tasks:
+            list_of_tasks_string_version = "\n".join(PushToHub.list_of_tasks)
+            raise ValueError(
+                f"""Invalid task, please pick one of the following tasks\n{list_of_tasks_string_version}""")
+
+        if model_data['model_zip_path'].endswith(".zip"):
+            with zipfile.ZipFile(model_data['model_zip_path']) as modelfile:
+                if 'metadata/part-00000' not in modelfile.namelist():
+                    raise ValueError("The Model is not a Spark Saved Model.")
+        else:
+            if not os.path.exists(f"{model_data['model_zip_path']}/metadata/part-00000"):
+                raise ValueError("The Model is not a Spark Saved Model.")
+
+    def push_to_hub(name: str,
+                    language: str,
+                    model_zip_path: str,
+                    task: str,
+                    pythonCode: str,
+                    GIT_TOKEN: str,
+                    title: str = None,
+                    tags: List[str] = None,
+                    dependencies: str = None,
+                    description: str = None,
+                    predictedEntities: str = None,
+                    sparknlpVersion: str = None,
+                    howToUse: str = None,
+                    liveDemo: str = None,
+                    runInColab: str = None,
+                    scalaCode: str = None,
+                    nluCode: str = None,
+                    results: str = None,
+                    dataSource: str = None,
+                    includedModels: str = None,
+                    benchmarking: str = None,
+                    ) -> str:
+        """Pushes model to Hub.
+
+        Keyword Arguments:
+        model_data:Dictionary containing info about the model such as Name and Language.
+        GIT_TOKEN: Token required for pushing to hub.
+        """
+
+        model_data = {item: value for (item, value) in locals().items() if value is not None}
+        PushToHub.check_for_required_info(model_data)
+        model_data = PushToHub.create_docs(model_data)
+
+        r1 = requests.post('https://modelshub.johnsnowlabs.com/api/v1/models', data=json.dumps(model_data), headers={
+            'Content-type': 'application/json',
+            'Authorization': f'Bearer {GIT_TOKEN}'
+        })
+
+        if r1.status_code == 201:
+            r2 = requests.post(
+                'https://modelshub.johnsnowlabs.com/api/v1/models/%s/file' % r1.json()['id'],
+                data=open(model_data['model_zip_path'], 'rb'), headers={
+                    'Authorization': f'Bearer {GIT_TOKEN}'
+                })
+            if r2.status_code == 200:
+                print(r2.json()['message'])
+                return r2.json()['message']
+        else:
+            print(f"Something Went Wrong During the Upload. Got Status Code: {r1.status_code}")
+            return f"Something Went Wrong During the Upload. Got Status Code: {r1.status_code}"
+
+    def create_docs(dicionary_for_upload: dict) -> dict:
+        """Adds fields in the dictionary for pushing to hub.
+
+        Keyword Arguments:
+        dictionary_for_upload: The dictionary to add keys to.
+        """
+
+        dicionary_for_upload['sparkVersion'] = "3.0"
+        dicionary_for_upload['license'] = 'Open Source'
+        dicionary_for_upload['supported'] = False
+
+        if 'sparknlpVersion' not in dicionary_for_upload.keys():
+            dicionary_for_upload['sparknlpVersion'] = "Spark NLP " + sparknlp.version()
+
+        if 'description' not in dicionary_for_upload.keys():
+            dicionary_for_upload[
+                'description'] = f"This model is used for {dicionary_for_upload['task']} and this model works with {dicionary_for_upload['language']} language"
+
+        if 'title' not in dicionary_for_upload.keys():
+            dicionary_for_upload[
+                'title'] = f"{dicionary_for_upload['task']} for {dicionary_for_upload['language']} language"
+
+        if os.path.isdir(dicionary_for_upload['model_zip_path']):
+            PushToHub.zip_directory(dicionary_for_upload['model_zip_path'],
+                                    f"{dicionary_for_upload['model_zip_path']}.zip")
+            dicionary_for_upload['model_zip_path'] = dicionary_for_upload['model_zip_path'] + '.zip'
+        return dicionary_for_upload