pyspark-client 4.2.0.dev1__tar.gz → 4.2.0.dev2__tar.gz

{pyspark_client-4.2.0.dev1/pyspark_client.egg-info → pyspark_client-4.2.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspark-client
-Version: 4.2.0.dev1
+Version: 4.2.0.dev2
 Summary: Python Spark Connect client for Apache Spark
 Home-page: https://github.com/apache/spark/tree/master/python
 Author: Spark Developers
@@ -18,7 +18,7 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: pandas>=2.2.0
-Requires-Dist: pyarrow>=15.0.0
+Requires-Dist: pyarrow>=18.0.0
 Requires-Dist: grpcio>=1.76.0
 Requires-Dist: grpcio-status>=1.76.0
 Requires-Dist: googleapis-common-protos>=1.71.0

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/_typing.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Callable, Iterable, Sized, TypeVar, Union
+from typing import Any, Callable, Iterable, Sized, TypeVar, Union
 from typing_extensions import Literal, Protocol
 
 from numpy import int32, int64, float32, float64, ndarray
@@ -29,10 +29,10 @@ PrimitiveType = Union[bool, float, int, str]
 NonUDFType = Literal[0]
 
 class SupportsIAdd(Protocol):
-    def __iadd__(self, other: SupportsIAdd) -> SupportsIAdd: ...
+    def __iadd__(self, other: Any) -> SupportsIAdd: ...
 
 class SupportsOrdering(Protocol):
-    def __lt__(self, other: SupportsOrdering) -> bool: ...
+    def __lt__(self, other: Any) -> bool: ...
 
 class SizedIterable(Protocol, Sized, Iterable[T_co]): ...
 

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/errors/error-conditions.json

@@ -465,6 +465,11 @@
       "Parameter value <arg_name> must be a valid UUID format: <origin>"
     ]
   },
+  "INVALID_STREAMING_SOURCE_NAME": {
+    "message": [
+      "Invalid streaming source name '<source_name>'. Source names must contain only ASCII letters, digits, and underscores."
+    ]
+  },
   "INVALID_TIMEOUT_TIMESTAMP": {
     "message": [
       "Timeout timestamp (<timestamp>) cannot be earlier than the current watermark (<watermark>)."
@@ -551,6 +556,12 @@
       "<arg1> and <arg2> should be of the same length, got <arg1_length> and <arg2_length>."
     ]
   },
+  "LOCAL_RELATION_SIZE_LIMIT_EXCEEDED": {
+    "message": [
+      "Local relation size (<actualSize> bytes) exceeds the limit (<sizeLimit> bytes)."
+    ],
+    "sqlState": "54000"
+  },
   "MALFORMED_GEOGRAPHY": {
     "message": [
       "Geography binary is malformed. Please check the data source is valid."

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/errors/exceptions/captured.py

@@ -234,25 +234,13 @@ def _convert_exception(e: "Py4JJavaError") -> CapturedException:
         return SparkUpgradeException(origin=e)
     elif is_instance_of(gw, e, "org.apache.spark.SparkNoSuchElementException"):
         return SparkNoSuchElementException(origin=e)
-
-    c: "Py4JJavaError" = e.getCause()
-    stacktrace: str = getattr(jvm, "org.apache.spark.util.Utils").exceptionString(e)
-    if c is not None and (
-        is_instance_of(gw, c, "org.apache.spark.api.python.PythonException")
-        # To make sure this only catches Python UDFs.
-        and any(
-            map(
-                lambda v: "org.apache.spark.sql.execution.python" in v.toString(), c.getStackTrace()
-            )
-        )
-    ):
-        msg = (
-            "\n  An exception was thrown from the Python worker. "
-            "Please see the stack trace below.\n%s" % c.getMessage()
-        )
-        return PythonException(msg, stacktrace)
-
-    return UnknownException(desc=e.toString(), stackTrace=stacktrace, cause=c)
+    elif is_instance_of(gw, e, "org.apache.spark.api.python.PythonException"):
+        return PythonException(origin=e)
+    return UnknownException(
+        desc=e.toString(),
+        stackTrace=getattr(jvm, "org.apache.spark.util.Utils").exceptionString(e),
+        cause=e.getCause(),
+    )
 
 
 def capture_sql_exception(f: Callable[..., Any]) -> Callable[..., Any]:
@@ -348,6 +336,17 @@ class PythonException(CapturedException, BasePythonException):
     Exceptions thrown from Python workers.
     """
 
+    def __str__(self) -> str:
+        messageParameters = self.getMessageParameters()
+
+        if (
+            messageParameters is None
+            or "msg" not in messageParameters
+            or "traceback" not in messageParameters
+        ):
+            return super().__str__()
+        return f"{messageParameters['msg']}:\n{messageParameters['traceback'].strip()}"
+
 
 class ArithmeticException(CapturedException, BaseArithmeticException):
     """

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/errors/utils.py

@@ -271,7 +271,7 @@ def _capture_call_site(depth: int) -> str:
         import IPython
 
         # ipykernel is required for IPython
-        import ipykernel  # type: ignore[import-not-found]
+        import ipykernel
 
         ipython = IPython.get_ipython()
         # Filtering out IPython related frames

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/instrumentation_utils.py

@@ -124,7 +124,7 @@ def _attach(
     logger_module: Union[str, ModuleType],
     modules: List[ModuleType],
     classes: List[Type[Any]],
-    missings: List[Tuple[Type[Any], Type[Any]]],
+    missings: List[Tuple[Union[ModuleType, Type[Any]], Type[Any]]],
 ) -> None:
     if isinstance(logger_module, str):
         logger_module = importlib.import_module(logger_module)

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/ml/_typing.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Any, Dict, List, TypeVar, Tuple, Union
+from typing import Any, Dict, List, TYPE_CHECKING, TypeVar, Tuple, Union
 from typing_extensions import Literal
 
 from numpy import ndarray
@@ -24,10 +24,12 @@ from py4j.java_gateway import JavaObject
 
 import pyspark.ml.base
 import pyspark.ml.param
-import pyspark.ml.util
 from pyspark.ml.linalg import Vector
 import pyspark.ml.wrapper
 
+if TYPE_CHECKING:
+    from scipy.sparse import spmatrix, sparray
+
 ParamMap = Dict[pyspark.ml.param.Param, Any]
 PipelineStage = Union[pyspark.ml.base.Estimator, pyspark.ml.base.Transformer]
 
@@ -81,4 +83,4 @@ RankingEvaluatorMetricType = Union[
     Literal["recallAtK"],
 ]
 
-VectorLike = Union[ndarray, Vector, List[float], Tuple[float, ...]]
+VectorLike = Union[ndarray, Vector, List[float], Tuple[float, ...], "spmatrix", "sparray", range]

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/ml/functions.py

@@ -241,7 +241,7 @@ def _validate_and_transform_single_input(
         # tensor columns
         if len(batch.columns) == 1:
             # one tensor column and one expected input, vstack rows
-            single_input = np.vstack(batch.iloc[:, 0])
+            single_input = np.vstack(batch.iloc[:, 0])  # type: ignore[call-overload]
         else:
             raise ValueError(
                 "Multiple input columns found, but model expected a single "

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/ml/linalg/__init__.py

@@ -70,7 +70,6 @@ __all__ = [
 if TYPE_CHECKING:
     from pyspark.mllib._typing import NormType
     from pyspark.ml._typing import VectorLike
-    from scipy.sparse import spmatrix
 
 
 # Check whether we have SciPy. MLlib works without it too, but if we have it, some methods,
@@ -85,23 +84,25 @@ except BaseException:
     _have_scipy = False
 
 
-def _convert_to_vector(d: Union["VectorLike", "spmatrix", range]) -> "Vector":
+def _convert_to_vector(d: "VectorLike") -> "Vector":
     if isinstance(d, Vector):
         return d
-    elif type(d) in (array.array, np.array, np.ndarray, list, tuple, range):
+    elif isinstance(d, (array.array, np.ndarray, list, tuple, range)):
         return DenseVector(d)
     elif _have_scipy and scipy.sparse.issparse(d):
-        assert cast("spmatrix", d).shape[1] == 1, "Expected column vector"
+        assert hasattr(d, "shape")
+        assert d.shape[1] == 1, "Expected column vector"
         # Make sure the converted csc_matrix has sorted indices.
-        csc = cast("spmatrix", d).tocsc()
+        assert hasattr(d, "tocsc")
+        csc = d.tocsc()
         if not csc.has_sorted_indices:
             csc.sort_indices()
-        return SparseVector(cast("spmatrix", d).shape[0], csc.indices, csc.data)
+        return SparseVector(d.shape[0], csc.indices, csc.data)
     else:
         raise TypeError("Cannot convert type %s into Vector" % type(d))
 
 
-def _vector_size(v: Union["VectorLike", "spmatrix", range]) -> int:
+def _vector_size(v: "VectorLike") -> int:
     """
     Returns the size of the vector.
 
@@ -124,16 +125,17 @@ def _vector_size(v: Union["VectorLike", "spmatrix", range]) -> int:
     """
     if isinstance(v, Vector):
         return len(v)
-    elif type(v) in (array.array, list, tuple, range):
+    elif isinstance(v, (array.array, list, tuple, range)):
         return len(v)
-    elif type(v) == np.ndarray:
+    elif isinstance(v, np.ndarray):
         if v.ndim == 1 or (v.ndim == 2 and v.shape[1] == 1):
             return len(v)
         else:
             raise ValueError("Cannot treat an ndarray of shape %s as a vector" % str(v.shape))
     elif _have_scipy and scipy.sparse.issparse(v):
-        assert cast("spmatrix", v).shape[1] == 1, "Expected column vector"
-        return cast("spmatrix", v).shape[0]
+        assert hasattr(v, "shape")
+        assert v.shape[1] == 1, "Expected column vector"
+        return v.shape[0]
     else:
         raise TypeError("Cannot treat type %s as a vector" % type(v))
 
@@ -337,13 +339,13 @@ class DenseVector(Vector):
     def __reduce__(self) -> Tuple[Type["DenseVector"], Tuple[bytes]]:
         return DenseVector, (self.array.tobytes(),)
 
-    def numNonzeros(self) -> int:
+    def numNonzeros(self) -> Union[int, np.intp]:
         """
         Number of nonzero elements. This scans all active values and count non zeros
         """
         return np.count_nonzero(self.array)
 
-    def norm(self, p: "NormType") -> np.float64:
+    def norm(self, p: "NormType") -> np.floating[Any]:
         """
         Calculates the norm of a DenseVector.
 
@@ -386,21 +388,23 @@ class DenseVector(Vector):
             ...
         AssertionError: dimension mismatch
         """
-        if type(other) == np.ndarray:
+        if isinstance(other, np.ndarray):
             if other.ndim > 1:
                 assert len(self) == other.shape[0], "dimension mismatch"
             return np.dot(self.array, other)
         elif _have_scipy and scipy.sparse.issparse(other):
-            assert len(self) == cast("spmatrix", other).shape[0], "dimension mismatch"
-            return cast("spmatrix", other).transpose().dot(self.toArray())
+            assert hasattr(other, "shape")
+            assert len(self) == other.shape[0], "dimension mismatch"
+            assert hasattr(other, "transpose")
+            return other.transpose().dot(self.toArray())
         else:
-            assert len(self) == _vector_size(other), "dimension mismatch"
+            assert len(self) == _vector_size(other), "dimension mismatch"  # type: ignore[arg-type]
             if isinstance(other, SparseVector):
                 return other.dot(self)
             elif isinstance(other, Vector):
                 return np.dot(self.toArray(), other.toArray())
             else:
-                return np.dot(self.toArray(), other)  # type: ignore[call-overload]
+                return np.dot(self.toArray(), other)
 
     def squared_distance(self, other: Iterable[float]) -> np.float64:
         """
@@ -429,10 +433,11 @@ class DenseVector(Vector):
             ...
         AssertionError: dimension mismatch
         """
-        assert len(self) == _vector_size(other), "dimension mismatch"
+        assert len(self) == _vector_size(other), "dimension mismatch"  # type: ignore[arg-type]
         if isinstance(other, SparseVector):
             return other.squared_distance(self)
         elif _have_scipy and scipy.sparse.issparse(other):
+            assert isinstance(other, scipy.sparse.spmatrix), "other must be a scipy.sparse.spmatrix"
             return _convert_to_vector(other).squared_distance(self)  # type: ignore[attr-defined]
 
         if isinstance(other, Vector):
@@ -636,13 +641,13 @@ class SparseVector(Vector):
             )
             assert np.min(self.indices) >= 0, "Contains negative index %d" % (np.min(self.indices))
 
-    def numNonzeros(self) -> int:
+    def numNonzeros(self) -> Union[int, np.intp]:
         """
         Number of nonzero elements. This scans all active values and count non zeros.
         """
         return np.count_nonzero(self.values)
 
-    def norm(self, p: "NormType") -> np.float64:
+    def norm(self, p: "NormType") -> np.floating[Any]:
         """
         Calculates the norm of a SparseVector.
 
@@ -699,7 +704,7 @@ class SparseVector(Vector):
             assert len(self) == other.shape[0], "dimension mismatch"
             return np.dot(self.values, other[self.indices])
 
-        assert len(self) == _vector_size(other), "dimension mismatch"
+        assert len(self) == _vector_size(other), "dimension mismatch"  # type: ignore[arg-type]
 
         if isinstance(other, DenseVector):
             return np.dot(other.array[self.indices], self.values)
@@ -717,7 +722,7 @@ class SparseVector(Vector):
         else:
             return self.dot(_convert_to_vector(other))  # type: ignore[arg-type]
 
-    def squared_distance(self, other: Iterable[float]) -> np.float64:
+    def squared_distance(self, other: "VectorLike") -> np.float64:
         """
         Squared distance from a SparseVector or 1-dimensional NumPy array.
 
@@ -785,7 +790,7 @@ class SparseVector(Vector):
                 j += 1
             return result
         else:
-            return self.squared_distance(_convert_to_vector(other))  # type: ignore[arg-type]
+            return self.squared_distance(_convert_to_vector(other))
 
     def toArray(self) -> np.ndarray:
         """

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/mllib/_typing.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import List, Tuple, TypeVar, Union
+from typing import List, Tuple, TYPE_CHECKING, TypeVar, Union
 
 from typing_extensions import Literal
 from numpy import ndarray  # noqa: F401
@@ -24,10 +24,14 @@ from py4j.java_gateway import JavaObject
 
 from pyspark.mllib.linalg import Vector
 
-VectorLike = Union[ndarray, Vector, List[float], Tuple[float, ...]]
+if TYPE_CHECKING:
+    from scipy.sparse import spmatrix, sparray
+
 C = TypeVar("C", bound=type)
 JavaObjectOrPickleDump = Union[JavaObject, bytearray, bytes]
 
 CorrMethodType = Union[Literal["spearman"], Literal["pearson"]]
 KolmogorovSmirnovTestDistNameType = Literal["norm"]
 NormType = Union[None, float, Literal["fro"], Literal["nuc"]]
+
+VectorLike = Union[ndarray, Vector, List[float], Tuple[float, ...], "spmatrix", "sparray", range]

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/mllib/linalg/__init__.py

@@ -61,7 +61,6 @@ from pyspark.sql.types import (
 
 if TYPE_CHECKING:
     from pyspark.mllib._typing import VectorLike, NormType
-    from scipy.sparse import spmatrix
     from numpy.typing import ArrayLike
 
 
@@ -94,23 +93,25 @@ except BaseException:
     _have_scipy = False
 
 
-def _convert_to_vector(d: Union["VectorLike", "spmatrix", range]) -> "Vector":
+def _convert_to_vector(d: "VectorLike") -> "Vector":
     if isinstance(d, Vector):
         return d
-    elif type(d) in (array.array, np.array, np.ndarray, list, tuple, range):
+    elif isinstance(d, (array.array, np.ndarray, list, tuple, range)):
         return DenseVector(d)
     elif _have_scipy and scipy.sparse.issparse(d):
-        assert cast("spmatrix", d).shape[1] == 1, "Expected column vector"
+        assert hasattr(d, "shape")
+        assert d.shape[1] == 1, "Expected column vector"
         # Make sure the converted csc_matrix has sorted indices.
-        csc = cast("spmatrix", d).tocsc()
+        assert hasattr(d, "tocsc")
+        csc = d.tocsc()
         if not csc.has_sorted_indices:
             csc.sort_indices()
-        return SparseVector(cast("spmatrix", d).shape[0], csc.indices, csc.data)
+        return SparseVector(d.shape[0], csc.indices, csc.data)
     else:
         raise TypeError("Cannot convert type %s into Vector" % type(d))
 
 
-def _vector_size(v: Union["VectorLike", "spmatrix", range]) -> int:
+def _vector_size(v: "VectorLike") -> int:
     """
     Returns the size of the vector.
 
@@ -133,16 +134,17 @@ def _vector_size(v: Union["VectorLike", "spmatrix", range]) -> int:
     """
     if isinstance(v, Vector):
         return len(v)
-    elif type(v) in (array.array, list, tuple, range):
+    elif isinstance(v, (array.array, list, tuple, range)):
         return len(v)
-    elif type(v) == np.ndarray:
+    elif isinstance(v, np.ndarray):
         if v.ndim == 1 or (v.ndim == 2 and v.shape[1] == 1):
             return len(v)
         else:
             raise ValueError("Cannot treat an ndarray of shape %s as a vector" % str(v.shape))
     elif _have_scipy and scipy.sparse.issparse(v):
-        assert cast("spmatrix", v).shape[1] == 1, "Expected column vector"
-        return cast("spmatrix", v).shape[0]
+        assert hasattr(v, "shape")
+        assert v.shape[1] == 1, "Expected column vector"
+        return v.shape[0]
     else:
         raise TypeError("Cannot treat type %s as a vector" % type(v))
 
@@ -390,13 +392,13 @@ class DenseVector(Vector):
     def __reduce__(self) -> Tuple[Type["DenseVector"], Tuple[bytes]]:
         return DenseVector, (self.array.tobytes(),)
 
-    def numNonzeros(self) -> int:
+    def numNonzeros(self) -> Union[int, np.intp]:
         """
         Number of nonzero elements. This scans all active values and count non zeros
         """
         return np.count_nonzero(self.array)
 
-    def norm(self, p: "NormType") -> np.float64:
+    def norm(self, p: "NormType") -> np.floating[Any]:
         """
         Calculates the norm of a DenseVector.
 
@@ -410,7 +412,7 @@ class DenseVector(Vector):
         """
         return np.linalg.norm(self.array, p)
 
-    def dot(self, other: Iterable[float]) -> np.float64:
+    def dot(self, other: "VectorLike") -> np.float64:
         """
         Compute the dot product of two Vectors. We support
         (Numpy array, list, SparseVector, or SciPy sparse)
@@ -444,8 +446,10 @@ class DenseVector(Vector):
                 assert len(self) == other.shape[0], "dimension mismatch"
             return np.dot(self.array, other)
         elif _have_scipy and scipy.sparse.issparse(other):
-            assert len(self) == cast("spmatrix", other).shape[0], "dimension mismatch"
-            return cast("spmatrix", other).transpose().dot(self.toArray())
+            assert hasattr(other, "shape")
+            assert len(self) == other.shape[0], "dimension mismatch"
+            assert hasattr(other, "transpose")
+            return other.transpose().dot(self.toArray())
         else:
             assert len(self) == _vector_size(other), "dimension mismatch"
             if isinstance(other, SparseVector):
@@ -453,9 +457,9 @@ class DenseVector(Vector):
             elif isinstance(other, Vector):
                 return np.dot(self.toArray(), other.toArray())
             else:
-                return np.dot(self.toArray(), cast("ArrayLike", other))
+                return np.dot(self.toArray(), cast("ArrayLike", other))  # type: ignore[valid-type]
 
-    def squared_distance(self, other: Iterable[float]) -> np.float64:
+    def squared_distance(self, other: "VectorLike") -> np.float64:
         """
         Squared distance of two Vectors.
 
@@ -685,13 +689,13 @@ class SparseVector(Vector):
                         % (self.indices[i], self.indices[i + 1])
                     )
 
-    def numNonzeros(self) -> int:
+    def numNonzeros(self) -> Union[int, np.intp]:
         """
         Number of nonzero elements. This scans all active values and count non zeros.
         """
         return np.count_nonzero(self.values)
 
-    def norm(self, p: "NormType") -> np.float64:
+    def norm(self, p: "NormType") -> np.floating[Any]:
         """
         Calculates the norm of a SparseVector.
 
@@ -766,7 +770,7 @@ class SparseVector(Vector):
             raise ValueError("Unable to parse values from %s." % s)
         return SparseVector(cast(int, size), indices, values)
 
-    def dot(self, other: Iterable[float]) -> np.float64:
+    def dot(self, other: "VectorLike") -> np.float64:
         """
         Dot product with a SparseVector or 1- or 2-dimensional Numpy array.
 
@@ -822,9 +826,9 @@ class SparseVector(Vector):
                 return np.dot(self_values, other.values[other_cmind])
 
         else:
-            return self.dot(_convert_to_vector(other))  # type: ignore[arg-type]
+            return self.dot(_convert_to_vector(other))
 
-    def squared_distance(self, other: Iterable[float]) -> np.float64:
+    def squared_distance(self, other: "VectorLike") -> np.float64:
         """
         Squared distance from a SparseVector or 1-dimensional NumPy array.
 
@@ -892,7 +896,7 @@ class SparseVector(Vector):
                 j += 1
             return result
         else:
-            return self.squared_distance(_convert_to_vector(other))  # type: ignore[arg-type]
+            return self.squared_distance(_convert_to_vector(other))
 
     def toArray(self) -> np.ndarray:
         """

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/mllib/linalg/distributed.py

@@ -35,7 +35,7 @@ UT = TypeVar("UT", bound="DistributedMatrix")
 VT = TypeVar("VT", bound="Matrix")
 
 if TYPE_CHECKING:
-    from pyspark.ml._typing import VectorLike
+    from pyspark.mllib._typing import VectorLike
 
 __all__ = [
     "BlockMatrix",

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/mllib/regression.py

@@ -84,7 +84,7 @@ class LabeledPoint:
     'label' and 'features' are accessible as class attributes.
     """
 
-    def __init__(self, label: float, features: Iterable[float]):
+    def __init__(self, label: float, features: "VectorLike"):
         self.label = float(label)
         self.features = _convert_to_vector(features)
 

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/mllib/stat/_statistics.py

@@ -189,7 +189,8 @@ class Statistics:
 
         if not y:
             return cast(
-                JavaObject, callMLlibFunc("corr", x.map(_convert_to_vector), method)
+                JavaObject,
+                callMLlibFunc("corr", cast(RDD[Vector], x).map(_convert_to_vector), method),
             ).toArray()
         else:
             return cast(

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/mllib/util.py

@@ -145,11 +145,7 @@ class MLUtils:
         if numFeatures <= 0:
             parsed.cache()
             numFeatures = parsed.map(lambda x: -1 if x[1].size == 0 else x[1][-1]).reduce(max) + 1
-        return parsed.map(
-            lambda x: LabeledPoint(
-                x[0], Vectors.sparse(numFeatures, x[1], x[2])  # type: ignore[arg-type]
-            )
-        )
+        return parsed.map(lambda x: LabeledPoint(x[0], Vectors.sparse(numFeatures, x[1], x[2])))
 
     @staticmethod
     def saveAsLibSVMFile(data: RDD["LabeledPoint"], dir: str) -> None:

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/pandas/accessors.py

@@ -579,7 +579,7 @@ class PandasOnSparkFrameMethods:
             return original_func(o, *args, **kwargs)
 
         def apply_func(pdf: pd.DataFrame) -> pd.DataFrame:
-            return new_func(pdf).to_frame()
+            return new_func(pdf).to_frame()  # type: ignore[operator]
 
         def pandas_series_func(
             f: Callable[[pd.DataFrame], pd.DataFrame], return_type: DataType

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/pandas/base.py

@@ -26,7 +26,7 @@ from typing import Any, Callable, Optional, Sequence, Tuple, Union, cast, TYPE_C
 
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_list_like, CategoricalDtype  # type: ignore[attr-defined]
+from pandas.api.types import is_list_like, CategoricalDtype
 
 from pyspark.sql import functions as F, Column, Window
 from pyspark.sql.types import LongType, BooleanType, NumericType

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/pandas/categorical.py

@@ -17,7 +17,7 @@
 from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING, cast
 
 import pandas as pd
-from pandas.api.types import (  # type: ignore[attr-defined]
+from pandas.api.types import (
     CategoricalDtype,
     is_dict_like,
     is_list_like,

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/pandas/data_type_ops/base.py

@@ -116,7 +116,7 @@ def _should_return_all_false(left: IndexOpsLike, right: Any) -> bool:
     based on incompatible dtypes: non-numeric vs. numeric (including bools).
     """
     from pyspark.pandas.base import IndexOpsMixin
-    from pandas.api.types import is_list_like  # type: ignore[attr-defined]
+    from pandas.api.types import is_list_like
 
     def are_both_numeric(left_dtype: Dtype, right_dtype: Dtype) -> bool:
         return is_numeric_dtype(left_dtype) and is_numeric_dtype(right_dtype)

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/pandas/data_type_ops/boolean_ops.py

@@ -19,7 +19,7 @@ import numbers
 from typing import Any, Union
 
 import pandas as pd
-from pandas.api.types import CategoricalDtype, is_integer_dtype  # type: ignore[attr-defined]
+from pandas.api.types import CategoricalDtype, is_integer_dtype
 from pandas.core.dtypes.common import is_numeric_dtype
 
 from pyspark.pandas.base import column_op, IndexOpsMixin

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev2}/pyspark/pandas/data_type_ops/categorical_ops.py

@@ -16,11 +16,11 @@
 #
 
 from itertools import chain
-from typing import cast, Any, Union
+from typing import cast, Any, Sequence, Union
 
 import pandas as pd
 import numpy as np
-from pandas.api.types import is_list_like, CategoricalDtype  # type: ignore[attr-defined]
+from pandas.api.types import is_list_like, CategoricalDtype
 
 from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
 from pyspark.pandas.base import IndexOpsMixin
@@ -43,7 +43,7 @@ class CategoricalOps(DataTypeOps):
         """Restore column when to_pandas."""
         return pd.Series(
             pd.Categorical.from_codes(
-                col.replace(np.nan, -1).astype(int),
+                cast(Sequence[int], col.replace(np.nan, -1).astype(int)),
                 categories=cast(CategoricalDtype, self.dtype).categories,
                 ordered=cast(CategoricalDtype, self.dtype).ordered,
             )