spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py

@@ -0,0 +1,240 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Contains classes concerning VisionEncoderDecoderForImageCaptioning."""
+
+from sparknlp.common import *
+
+
+class VisionEncoderDecoderForImageCaptioning(AnnotatorModel,
+                                             HasBatchedAnnotateImage,
+                                             HasImageFeatureProperties,
+                                             HasGeneratorProperties,
+                                             HasRescaleFactor,
+                                             HasEngine):
+    """VisionEncoderDecoder model that converts images into text captions. It allows for the use of
+    pretrained vision auto-encoding models, such as ViT, BEiT, or DeiT as the encoder, in
+    combination with pretrained language models, like RoBERTa, GPT2, or BERT as the decoder.
+
+    Pretrained models can be loaded with ``pretrained`` of the companion object:
+
+    .. code-block:: python
+
+        imageClassifier = VisionEncoderDecoderForImageCaptioning.pretrained() \\
+            .setInputCols(["image_assembler"]) \\
+            .setOutputCol("caption")
+
+
+    The default model is ``"image_captioning_vit_gpt2"``, if no name is provided.
+
+    For available pretrained models please see the
+    `Models Hub <https://sparknlp.org/models?task=Image+Captioning>`__.
+
+    Models from the HuggingFace 🤗 Transformers library are also compatible with Spark NLP 🚀. To
+    see which models are compatible and how to import them see
+    https://github.com/JohnSnowLabs/spark-nlp/discussions/5669 and to see more extended
+    examples, see
+    `VisionEncoderDecoderTestSpec <https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/cv/VisionEncoderDecoderForImageCaptioningTestSpec.scala>`__.
+
+    Notes
+    -----
+    This is a very computationally expensive module especially on larger
+    batch sizes. The use of an accelerator such as GPU is recommended.
+
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``IMAGE``              ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    configProtoBytes
+        ConfigProto from tensorflow, serialized into byte array.
+    doResize
+        Whether to resize the input to a certain size
+    doNormalize
+        Whether to normalize the input with mean and standard deviation
+    featureExtractorType
+        Name of model's architecture for feature extraction
+    imageMean
+        The sequence of means for each channel, to be used when normalizing images
+    imageStd
+        The sequence of standard deviations for each channel, to be used when normalizing images
+    resample
+        An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BILINEAR` or
+        `PIL.Image.BICUBIC`. Only has an effect if do_resize is set to True.
+    size
+        Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an integer is
+        provided, then the input will be resized to (size, size). Only has an effect if do_resize is set to True.
+    doRescale
+        Whether to rescale the image values by rescaleFactor
+    rescaleFactor
+        Factor to scale the image values
+    minOutputLength
+        Minimum length of the sequence to be generated
+    maxOutputLength
+        Maximum length of output text
+    doSample
+        Whether or not to use sampling; use greedy decoding otherwise
+    temperature
+        The value used to module the next token probabilities
+    topK
+        The number of highest probability vocabulary tokens to keep for top-k-filtering
+    topP
+        If set to float < 1, only the most probable tokens with probabilities that add up to ``top_p`` or higher are
+        kept for generation
+    repetitionPenalty
+        The parameter for repetition penalty. 1.0 means no penalty.
+        See `this paper <https://arxiv.org/pdf/1909.05858.pdf>`__ for more details
+    noRepeatNgramSize
+        If set to int > 0, all ngrams of that size can only occur once
+    beamSize
+        The Number of beams for beam search
+    nReturnSequences
+        The number of sequences to return from the beam search
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> imageDF = spark.read \\
+    ...     .format("image") \\
+    ...     .option("dropInvalid", value = True) \\
+    ...     .load("src/test/resources/image/")
+    >>> imageAssembler = ImageAssembler() \\
+    ...     .setInputCol("image") \\
+    ...     .setOutputCol("image_assembler")
+    >>> imageCaptioning = VisionEncoderDecoderForImageCaptioning \\
+    ...     .pretrained() \\
+    ...     .setBeamSize(2) \\
+    ...     .setDoSample(False) \\
+    ...     .setInputCols(["image_assembler"]) \\
+    ...     .setOutputCol("caption")
+    >>> pipeline = Pipeline().setStages([imageAssembler, imageCaptioning])
+    >>> pipelineDF = pipeline.fit(imageDF).transform(imageDF)
+    >>> pipelineDF \\
+    ...     .selectExpr("reverse(split(image.origin, '/'))[0] as image_name", "caption.result") \\
+    ...     .show(truncate = False)
+    +-----------------+---------------------------------------------------------+
+    |image_name       |result                                                   |
+    +-----------------+---------------------------------------------------------+
+    |palace.JPEG      |[a large room filled with furniture and a large window]  |
+    |egyptian_cat.jpeg|[a cat laying on a couch next to another cat]            |
+    |hippopotamus.JPEG|[a brown bear in a body of water]                        |
+    |hen.JPEG         |[a flock of chickens standing next to each other]        |
+    |ostrich.JPEG     |[a large bird standing on top of a lush green field]     |
+    |junco.JPEG       |[a small bird standing on a wet ground]                  |
+    |bluetick.jpg     |[a small dog standing on a wooden floor]                 |
+    |chihuahua.jpg    |[a small brown dog wearing a blue sweater]               |
+    |tractor.JPEG     |[a man is standing in a field with a tractor]            |
+    |ox.JPEG          |[a large brown cow standing on top of a lush green field]|
+    +-----------------+---------------------------------------------------------+
+
+    """
+    name = "VisionEncoderDecoderForImageCaptioning"
+
+    inputAnnotatorTypes = [AnnotatorType.IMAGE]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    configProtoBytes = Param(Params._dummy(),
+                             "configProtoBytes",
+                             "ConfigProto from tensorflow, serialized into byte array. Get with "
+                             "config_proto.SerializeToString()",
+                             TypeConverters.toListInt)
+
+    def setConfigProtoBytes(self, b):
+        """Sets configProto from tensorflow, serialized into byte array.
+
+        Parameters
+        ----------
+        b : List[int]
+            ConfigProto from tensorflow, serialized into byte array
+        """
+        return self._set(configProtoBytes=b)
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.cv.VisionEncoderDecoderForImageCaptioning",
+                 java_model=None):
+        super(VisionEncoderDecoderForImageCaptioning, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            batchSize=2,
+            beamSize=1,
+            doNormalize=True,
+            doRescale=True,
+            doResize=True,
+            doSample=True,
+            imageMean=[0.5, 0.5, 0.5],
+            imageStd=[0.5, 0.5, 0.5],
+            maxOutputLength=50,
+            minOutputLength=0,
+            nReturnSequences=1,
+            noRepeatNgramSize=0,
+            repetitionPenalty=1.0,
+            resample=2,
+            rescaleFactor=1 / 255.0,
+            size=224,
+            temperature=1.0,
+            topK=50,
+            topP=1.0)
+
+    @staticmethod
+    def loadSavedModel(folder, spark_session):
+        """Loads a locally saved model.
+
+        Parameters
+        ----------
+        folder : str
+            Folder of the saved model
+        spark_session : pyspark.sql.SparkSession
+            The current SparkSession
+
+        Returns
+        -------
+        VisionEncoderDecoderForImageCaptioning
+            The restored model
+        """
+        from sparknlp.internal import _VisionEncoderDecoderForImageCaptioning
+        jModel = _VisionEncoderDecoderForImageCaptioning(folder, spark_session._jsparkSession)._java_obj
+        return VisionEncoderDecoderForImageCaptioning(java_model=jModel)
+
+    @staticmethod
+    def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default
+            "image_captioning_vit_gpt2"
+        lang : str, optional
+            Language of the pretrained model, by default "en"
+        remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        VisionEncoderDecoderForImageCaptioning
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(VisionEncoderDecoderForImageCaptioning, name, lang, remote_loc)

sparknlp/annotator/cv/vit_for_image_classification.py

@@ -0,0 +1,217 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Contains classes concerning ViTForImageClassification."""
+
+from sparknlp.common import *
+
+
+class ViTForImageClassification(AnnotatorModel,
+                                HasBatchedAnnotateImage,
+                                HasImageFeatureProperties,
+                                HasEngine):
+    """Vision Transformer (ViT) for image classification.
+
+    ViT is a transformer based alternative to the convolutional neural networks usually
+    used for image recognition tasks.
+
+    Pretrained models can be loaded with ``pretrained`` of the companion object:
+
+    .. code-block:: python
+
+        imageClassifier = ViTForImageClassification.pretrained() \\
+            .setInputCols(["image_assembler"]) \\
+            .setOutputCol("class")
+
+
+    The default model is ``"image_classifier_vit_base_patch16_224"``, if no name is
+    provided.
+
+    For available pretrained models please see the
+    `Models Hub <https://sparknlp.org/models?task=Image+Classification>`__.
+
+    Models from the HuggingFace 🤗 Transformers library are also compatible with Spark
+    NLP 🚀. To see which models are compatible and how to import them see
+    https://github.com/JohnSnowLabs/spark-nlp/discussions/5669 and to see more extended
+    examples, see
+    `ViTImageClassificationTestSpec <https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/cv/ViTImageClassificationTestSpec.scala>`__.
+
+    **Paper Abstract:**
+
+    *While the Transformer architecture has become the de-facto standard for natural
+    language processing tasks, its applications to computer vision remain limited. In
+    vision, attention is either applied in conjunction with convolutional networks, or
+    used to replace certain components of convolutional networks while keeping their
+    overall structure in place. We show that this reliance on CNNs is not necessary and
+    a pure transformer applied directly to sequences of image patches can perform very
+    well on image classification tasks. When pre-trained on large amounts of data and
+    transferred to multiple mid-sized or small image recognition benchmarks (ImageNet,
+    CIFAR-100, VTAB, etc.), Vision Transformer (ViT) attains excellent results compared
+    to state-of-the-art convolutional networks while requiring substantially fewer
+    computational resources to train.*
+
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``IMAGE``              ``CATEGORY``
+    ====================== ======================
+
+    References
+    ----------
+
+    `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale
+    <https://arxiv.org/abs/2010.11929>`__
+
+
+    Parameters
+    ----------
+    doResize
+        Whether to resize the input to a certain size
+    doNormalize
+        Whether to normalize the input with mean and standard deviation
+    featureExtractorType
+        Name of model's architecture for feature extraction
+    imageMean
+        The sequence of means for each channel, to be used when normalizing images
+    imageStd
+        The sequence of standard deviations for each channel, to be used when normalizing images
+    resample
+        An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BILINEAR` or
+        `PIL.Image.BICUBIC`. Only has an effect if do_resize is set to True.
+    size
+        Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an integer is
+        provided, then the input will be resized to (size, size). Only has an effect if do_resize is set to True.
+    configProtoBytes
+        ConfigProto from tensorflow, serialized into byte array.
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> imageDF = spark.read \\
+    ...     .format("image") \\
+    ...     .option("dropInvalid", value = True) \\
+    ...     .load("src/test/resources/image/")
+    >>> imageAssembler = ImageAssembler() \\
+    ...     .setInputCol("image") \\
+    ...     .setOutputCol("image_assembler")
+    >>> imageClassifier = ViTForImageClassification \\
+    ...     .pretrained() \\
+    ...     .setInputCols(["image_assembler"]) \\
+    ...     .setOutputCol("class")
+    >>> pipeline = Pipeline().setStages([imageAssembler, imageClassifier])
+    >>> pipelineDF = pipeline.fit(imageDF).transform(imageDF)
+    >>> pipelineDF \\
+    ...   .selectExpr("reverse(split(image.origin, '/'))[0] as image_name", "class.result") \\
+    ...   .show(truncate=False)
+    +-----------------+----------------------------------------------------------+
+    |image_name       |result                                                    |
+    +-----------------+----------------------------------------------------------+
+    |palace.JPEG      |[palace]                                                  |
+    |egyptian_cat.jpeg|[Egyptian cat]                                            |
+    |hippopotamus.JPEG|[hippopotamus, hippo, river horse, Hippopotamus amphibius]|
+    |hen.JPEG         |[hen]                                                     |
+    |ostrich.JPEG     |[ostrich, Struthio camelus]                               |
+    |junco.JPEG       |[junco, snowbird]                                         |
+    |bluetick.jpg     |[bluetick]                                                |
+    |chihuahua.jpg    |[Chihuahua]                                               |
+    |tractor.JPEG     |[tractor]                                                 |
+    |ox.JPEG          |[ox]                                                      |
+    +-----------------+----------------------------------------------------------+
+
+    """
+    name = "ViTForImageClassification"
+
+    inputAnnotatorTypes = [AnnotatorType.IMAGE]
+
+    outputAnnotatorType = AnnotatorType.CATEGORY
+
+    configProtoBytes = Param(Params._dummy(),
+                             "configProtoBytes",
+                             "ConfigProto from tensorflow, serialized into byte array. Get with "
+                             "config_proto.SerializeToString()",
+                             TypeConverters.toListInt)
+
+    def getClasses(self):
+        """
+        Returns labels used to train this model
+        """
+        return self._call_java("getClasses")
+
+    def setConfigProtoBytes(self, b):
+        """Sets configProto from tensorflow, serialized into byte array.
+
+        Parameters
+        ----------
+        b : List[int]
+            ConfigProto from tensorflow, serialized into byte array
+        """
+        return self._set(configProtoBytes=b)
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.cv.ViTForImageClassification",
+                 java_model=None):
+        super(ViTForImageClassification, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            batchSize=2
+        )
+
+    @staticmethod
+    def loadSavedModel(folder, spark_session):
+        """Loads a locally saved model.
+
+        Parameters
+        ----------
+        folder : str
+            Folder of the saved model
+        spark_session : pyspark.sql.SparkSession
+            The current SparkSession
+
+        Returns
+        -------
+        ViTForImageClassification
+            The restored model
+        """
+        from sparknlp.internal import _ViTForImageClassification
+        jModel = _ViTForImageClassification(folder, spark_session._jsparkSession)._java_obj
+        return ViTForImageClassification(java_model=jModel)
+
+    @staticmethod
+    def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default
+            "image_classifier_vit_base_patch16_224"
+        lang : str, optional
+            Language of the pretrained model, by default "en"
+        remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        ViTForImageClassification
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(ViTForImageClassification, name, lang, remote_loc)

sparknlp/annotator/dataframe_optimizer.py

@@ -0,0 +1,216 @@
+#  Copyright 2017-2025 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from pyspark.ml import Transformer
+from pyspark.ml.param.shared import *
+from pyspark.sql import DataFrame
+from typing import Any
+
+# Custom converter for string-to-string dictionaries
+def toStringDict(value):
+    if not isinstance(value, dict):
+        raise TypeError("Expected a dictionary of strings.")
+    return {str(k): str(v) for k, v in value.items()}
+
+class DataFrameOptimizer(Transformer):
+    """
+    Optimizes a Spark DataFrame by repartitioning, optionally caching, and persisting it to disk.
+
+    This transformer is intended to improve performance for Spark NLP pipelines or when preparing
+    data for export. It allows partition tuning via `numPartitions` directly, or indirectly using
+    `executorCores` and `numWorkers`. The DataFrame can also be persisted in a specified format
+    (`csv`, `json`, or `parquet`) with additional writer options.
+
+    Parameters
+    ----------
+    executorCores : int, optional
+        Number of cores per Spark executor (used to compute number of partitions if `numPartitions` is not set).
+
+    numWorkers : int, optional
+        Number of executor nodes (used to compute number of partitions if `numPartitions` is not set).
+
+    numPartitions : int, optional
+        Target number of partitions for the DataFrame (overrides calculation via cores × workers).
+
+    doCache : bool, default False
+        Whether to cache the DataFrame after repartitioning.
+
+    persistPath : str, optional
+        Path to save the DataFrame output (if persistence is enabled).
+
+    persistFormat : str, optional
+        Format to persist the DataFrame in: one of `'csv'`, `'json'`, or `'parquet'`.
+
+    outputOptions : dict, optional
+        Dictionary of options for the DataFrameWriter (e.g., `{"compression": "snappy"}` for parquet).
+
+    Examples
+    --------
+    >>> optimizer = DataFrameOptimizer() \\
+    ...     .setExecutorCores(4) \\
+    ...     .setNumWorkers(5) \\
+    ...     .setDoCache(True) \\
+    ...     .setPersistPath("/tmp/out") \\
+    ...     .setPersistFormat("parquet") \\
+    ...     .setOutputOptions({"compression": "snappy"})
+
+    >>> optimized_df = optimizer.transform(input_df)
+
+    Notes
+    -----
+    - You must specify either `numPartitions`, or both `executorCores` and `numWorkers`.
+    - Schema is preserved; no columns are modified or removed.
+    """
+
+    executorCores = Param(
+        Params._dummy(),
+        "executorCores",
+        "Number of cores per executor",
+        typeConverter = TypeConverters.toInt
+    )
+    numWorkers = Param(
+        Params._dummy(),
+        "numWorkers",
+        "Number of Spark workers",
+        typeConverter = TypeConverters.toInt
+    )
+    numPartitions = Param(
+        Params._dummy(),
+        "numPartitions",
+        "Total number of partitions (overrides executorCores * numWorkers)",
+        typeConverter = TypeConverters.toInt
+    )
+    doCache = Param(
+        Params._dummy(),
+        "doCache",
+        "Whether to cache the DataFrame",
+        typeConverter = TypeConverters.toBoolean
+    )
+
+    persistPath = Param(
+        Params._dummy(),
+        "persistPath",
+        "Optional path to persist the DataFrame",
+        typeConverter = TypeConverters.toString
+    )
+    persistFormat = Param(
+        Params._dummy(),
+        "persistFormat",
+        "Format to persist: parquet, json, csv",
+        typeConverter = TypeConverters.toString
+    )
+
+    outputOptions = Param(
+        Params._dummy(),
+        "outputOptions",
+        "Additional writer options",
+        typeConverter=toStringDict
+    )
+
+    def __init__(self):
+        super().__init__()
+        self._setDefault(
+            doCache=False,
+            persistFormat="none",
+            numPartitions=1,
+            executorCores=1,
+            numWorkers=1
+        )
+
+    # Parameter setters
+    def setExecutorCores(self, value: int):
+        """Set the number of executor cores."""
+        return self._set(executorCores=value)
+
+    def setNumWorkers(self, value: int):
+        """Set the number of Spark workers."""
+        return self._set(numWorkers=value)
+
+    def setNumPartitions(self, value: int):
+        """Set the total number of partitions (overrides cores * workers)."""
+        return self._set(numPartitions=value)
+
+    def setDoCache(self, value: bool):
+        """Set whether to cache the DataFrame."""
+        return self._set(doCache=value)
+
+    def setPersistPath(self, value: str):
+        """Set the path where the DataFrame should be persisted."""
+        return self._set(persistPath=value)
+
+    def setPersistFormat(self, value: str):
+        """Set the format to persist the DataFrame (parquet, json, csv)."""
+        return self._set(persistFormat=value)
+
+    def setOutputOptions(self, value: dict):
+        """Set additional writer options (e.g. for csv headers)."""
+        return self._set(outputOptions=value)
+
+    # Optional bulk setter
+    def setParams(self, **kwargs: Any):
+        for param, value in kwargs.items():
+            self._set(**{param: value})
+        return self
+
+    def _transform(self, dataset: DataFrame) -> DataFrame:
+        self._validate_params()
+        part_count = self.getOrDefault(self.numPartitions)
+        cores = self.getOrDefault(self.executorCores)
+        workers = self.getOrDefault(self.numWorkers)
+        if cores is None or workers is None:
+            raise ValueError("Provide either numPartitions or both executorCores and numWorkers")
+        if part_count == 1:
+            part_count = cores * workers
+
+        optimized_df = dataset.repartition(part_count)
+
+        if self.getOrDefault(self.doCache):
+            optimized_df = optimized_df.cache()
+
+        format = self.getOrDefault(self.persistFormat).lower()
+        if format != "none":
+            path = self.getOrDefault(self.persistPath)
+            if not path:
+                raise ValueError("persistPath must be set when persistFormat is not 'none'")
+            writer = optimized_df.write.mode("overwrite")
+            if self.isDefined(self.outputOptions):
+                writer = writer.options(**self.getOrDefault(self.outputOptions))
+            if format == "parquet":
+                writer.parquet(path)
+            elif format == "json":
+                writer.json(path)
+            elif format == "csv":
+                writer.csv(path)
+            else:
+                raise ValueError(f"Unsupported format: {format}")
+
+        return optimized_df
+
+    def _validate_params(self):
+        if self.isDefined(self.executorCores):
+            val = self.getOrDefault(self.executorCores)
+            if val <= 0:
+                raise ValueError("executorCores must be > 0")
+
+        if self.isDefined(self.numWorkers):
+            val = self.getOrDefault(self.numWorkers)
+            if val <= 0:
+                raise ValueError("numWorkers must be > 0")
+
+        if self.isDefined(self.numPartitions):
+            val = self.getOrDefault(self.numPartitions)
+            if val <= 0:
+                raise ValueError("numPartitions must be > 0")
+
+        if self.isDefined(self.persistPath) and not self.isDefined(self.persistFormat):
+            raise ValueError("persistFormat must be defined when persistPath is set")