pyspark-client 4.2.0.dev1__tar.gz → 4.2.0.dev3__tar.gz

{pyspark_client-4.2.0.dev1/pyspark_client.egg-info → pyspark_client-4.2.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspark-client
-Version: 4.2.0.dev1
+Version: 4.2.0.dev3
 Summary: Python Spark Connect client for Apache Spark
 Home-page: https://github.com/apache/spark/tree/master/python
 Author: Spark Developers
@@ -18,7 +18,7 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: pandas>=2.2.0
-Requires-Dist: pyarrow>=15.0.0
+Requires-Dist: pyarrow>=18.0.0
 Requires-Dist: grpcio>=1.76.0
 Requires-Dist: grpcio-status>=1.76.0
 Requires-Dist: googleapis-common-protos>=1.71.0

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/_typing.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Callable, Iterable, Sized, TypeVar, Union
+from typing import Any, Callable, Iterable, Sized, TypeVar, Union
 from typing_extensions import Literal, Protocol
 
 from numpy import int32, int64, float32, float64, ndarray
@@ -29,10 +29,10 @@ PrimitiveType = Union[bool, float, int, str]
 NonUDFType = Literal[0]
 
 class SupportsIAdd(Protocol):
-    def __iadd__(self, other: SupportsIAdd) -> SupportsIAdd: ...
+    def __iadd__(self, other: Any) -> SupportsIAdd: ...
 
 class SupportsOrdering(Protocol):
-    def __lt__(self, other: SupportsOrdering) -> bool: ...
+    def __lt__(self, other: Any) -> bool: ...
 
 class SizedIterable(Protocol, Sized, Iterable[T_co]): ...
 

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/accumulators.py

@@ -27,7 +27,7 @@ from pyspark.serializers import read_int, CPickleSerializer
 from pyspark.errors import PySparkRuntimeError
 
 if TYPE_CHECKING:
-    from pyspark._typing import SupportsIAdd  # noqa: F401
+    from pyspark._typing import SupportsIAdd
     import socketserver.BaseRequestHandler  # type: ignore[import-not-found]
 
 

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/daemon.py

@@ -30,6 +30,7 @@ from socket import AF_INET, AF_INET6, SOCK_STREAM, SOMAXCONN
 from signal import SIGHUP, SIGTERM, SIGCHLD, SIG_DFL, SIG_IGN, SIGINT
 
 from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
+from pyspark.util import enable_faulthandler
 from pyspark.errors import PySparkRuntimeError
 
 
@@ -226,53 +227,54 @@ def manager():
 
                 if pid == 0:
                     # in child process
-                    if poller is not None:
-                        poller.unregister(0)
-                        poller.unregister(listen_sock)
-                    listen_sock.close()
+                    with enable_faulthandler():
+                        if poller is not None:
+                            poller.unregister(0)
+                            poller.unregister(listen_sock)
+                        listen_sock.close()
 
-                    # It should close the standard input in the child process so that
-                    # Python native function executions stay intact.
-                    #
-                    # Note that if we just close the standard input (file descriptor 0),
-                    # the lowest file descriptor (file descriptor 0) will be allocated,
-                    # later when other file descriptors should happen to open.
-                    #
-                    # Therefore, here we redirects it to '/dev/null' by duplicating
-                    # another file descriptor for '/dev/null' to the standard input (0).
-                    # See SPARK-26175.
-                    devnull = open(os.devnull, "r")
-                    os.dup2(devnull.fileno(), 0)
-                    devnull.close()
+                        # It should close the standard input in the child process so that
+                        # Python native function executions stay intact.
+                        #
+                        # Note that if we just close the standard input (file descriptor 0),
+                        # the lowest file descriptor (file descriptor 0) will be allocated,
+                        # later when other file descriptors should happen to open.
+                        #
+                        # Therefore, here we redirects it to '/dev/null' by duplicating
+                        # another file descriptor for '/dev/null' to the standard input (0).
+                        # See SPARK-26175.
+                        devnull = open(os.devnull, "r")
+                        os.dup2(devnull.fileno(), 0)
+                        devnull.close()
 
-                    try:
-                        # Acknowledge that the fork was successful
-                        outfile = sock.makefile(mode="wb")
-                        write_int(os.getpid(), outfile)
-                        outfile.flush()
-                        outfile.close()
-                        authenticated = (
-                            os.environ.get("PYTHON_UNIX_DOMAIN_ENABLED", "false").lower() == "true"
-                            or False
-                        )
-                        while True:
-                            code = worker(sock, authenticated)
-                            if code == 0:
-                                authenticated = True
-                            if not reuse or code:
-                                # wait for closing
-                                try:
-                                    while sock.recv(1024):
+                        try:
+                            # Acknowledge that the fork was successful
+                            outfile = sock.makefile(mode="wb")
+                            write_int(os.getpid(), outfile)
+                            outfile.flush()
+                            outfile.close()
+                            authenticated = (
+                                os.environ.get("PYTHON_UNIX_DOMAIN_ENABLED", "false").lower()
+                                == "true"
+                            )
+                            while True:
+                                code = worker(sock, authenticated)
+                                if code == 0:
+                                    authenticated = True
+                                if not reuse or code:
+                                    # wait for closing
+                                    try:
+                                        while sock.recv(1024):
+                                            pass
+                                    except Exception:
                                         pass
-                                except Exception:
-                                    pass
-                                break
-                            gc.collect()
-                    except BaseException:
-                        traceback.print_exc()
-                        os._exit(1)
-                    else:
-                        os._exit(0)
+                                    break
+                                gc.collect()
+                        except BaseException:
+                            traceback.print_exc()
+                            os._exit(1)
+                        else:
+                            os._exit(0)
                 else:
                     sock.close()
 

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors/__init__.py

@@ -18,7 +18,7 @@
 """
 PySpark exceptions.
 """
-from pyspark.errors.exceptions.base import (  # noqa: F401
+from pyspark.errors.exceptions.base import (
     PySparkException,
     AnalysisException,
     SessionNotSameException,

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors/error-conditions.json

@@ -465,6 +465,11 @@
       "Parameter value <arg_name> must be a valid UUID format: <origin>"
     ]
   },
+  "INVALID_STREAMING_SOURCE_NAME": {
+    "message": [
+      "Invalid streaming source name '<source_name>'. Source names must contain only ASCII letters, digits, and underscores."
+    ]
+  },
   "INVALID_TIMEOUT_TIMESTAMP": {
     "message": [
       "Timeout timestamp (<timestamp>) cannot be earlier than the current watermark (<watermark>)."
@@ -551,6 +556,12 @@
       "<arg1> and <arg2> should be of the same length, got <arg1_length> and <arg2_length>."
     ]
   },
+  "LOCAL_RELATION_SIZE_LIMIT_EXCEEDED": {
+    "message": [
+      "Local relation size (<actualSize> bytes) exceeds the limit (<sizeLimit> bytes)."
+    ],
+    "sqlState": "54000"
+  },
   "MALFORMED_GEOGRAPHY": {
     "message": [
       "Geography binary is malformed. Please check the data source is valid."
@@ -1174,6 +1185,11 @@
       "SparkContext or SparkSession should be created first."
     ]
   },
+  "SIMPLE_STREAM_READER_OFFSET_DID_NOT_ADVANCE": {
+    "message": [
+      "SimpleDataSourceStreamReader.read() returned a non-empty batch but the end offset: <end_offset> did not advance past the start offset: <start_offset>. The end offset must represent the position after the last record returned."
+    ]
+  },
   "SLICE_WITH_STEP": {
     "message": [
       "Slice with step is not supported."
@@ -1237,12 +1253,12 @@
       "Return type of the user-defined function should be <expected>, but is <actual>."
     ]
   },
-  "UDTF_ARROW_TYPE_CAST_ERROR": {
+  "UDTF_ARROW_DATA_CONVERSION_ERROR": {
     "message": [
-      "Cannot convert the output value of the column '<col_name>' with type '<col_type>' to the specified return type of the column: '<arrow_type>'. Please check if the data types match and try again."
+      "Cannot convert UDTF output to Arrow. Data: <data>. Schema: <schema>. Arrow Schema: <arrow_schema>."
     ]
   },
-  "UDTF_ARROW_TYPE_CONVERSION_ERROR": {
+"UDTF_ARROW_TYPE_CONVERSION_ERROR": {
     "message": [
       "PyArrow UDTF must return an iterator of pyarrow.Table or pyarrow.RecordBatch objects."
     ]
@@ -1467,6 +1483,12 @@
       "Value for `<arg_name>` must be between <lower_bound> and <upper_bound> (inclusive), got <actual>"
     ]
   },
+  "WKB_PARSE_ERROR" : {
+    "message" : [
+      "Error parsing WKB: <parseError> at position <pos>"
+    ],
+    "sqlState" : "22023"
+  },
   "WRONG_NUM_ARGS_FOR_HIGHER_ORDER_FUNCTION": {
     "message": [
       "Function `<func_name>` should take between 1 and 3 arguments, but the provided function takes <num_args>."

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors/exceptions/captured.py

@@ -234,25 +234,13 @@ def _convert_exception(e: "Py4JJavaError") -> CapturedException:
         return SparkUpgradeException(origin=e)
     elif is_instance_of(gw, e, "org.apache.spark.SparkNoSuchElementException"):
         return SparkNoSuchElementException(origin=e)
-
-    c: "Py4JJavaError" = e.getCause()
-    stacktrace: str = getattr(jvm, "org.apache.spark.util.Utils").exceptionString(e)
-    if c is not None and (
-        is_instance_of(gw, c, "org.apache.spark.api.python.PythonException")
-        # To make sure this only catches Python UDFs.
-        and any(
-            map(
-                lambda v: "org.apache.spark.sql.execution.python" in v.toString(), c.getStackTrace()
-            )
-        )
-    ):
-        msg = (
-            "\n  An exception was thrown from the Python worker. "
-            "Please see the stack trace below.\n%s" % c.getMessage()
-        )
-        return PythonException(msg, stacktrace)
-
-    return UnknownException(desc=e.toString(), stackTrace=stacktrace, cause=c)
+    elif is_instance_of(gw, e, "org.apache.spark.api.python.PythonException"):
+        return PythonException(origin=e)
+    return UnknownException(
+        desc=e.toString(),
+        stackTrace=getattr(jvm, "org.apache.spark.util.Utils").exceptionString(e),
+        cause=e.getCause(),
+    )
 
 
 def capture_sql_exception(f: Callable[..., Any]) -> Callable[..., Any]:
@@ -348,6 +336,17 @@ class PythonException(CapturedException, BasePythonException):
     Exceptions thrown from Python workers.
     """
 
+    def __str__(self) -> str:
+        messageParameters = self.getMessageParameters()
+
+        if (
+            messageParameters is None
+            or "msg" not in messageParameters
+            or "traceback" not in messageParameters
+        ):
+            return super().__str__()
+        return f"{messageParameters['msg']}:\n{messageParameters['traceback'].strip()}"
+
 
 class ArithmeticException(CapturedException, BaseArithmeticException):
     """

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors/exceptions/connect.py

@@ -61,43 +61,89 @@ def convert_exception(
     display_server_stacktrace: bool = False,
     grpc_status_code: grpc.StatusCode = StatusCode.UNKNOWN,
 ) -> SparkConnectException:
+    raw_classes = info.metadata.get("classes")
+    classes: List[str] = json.loads(raw_classes) if raw_classes else []
+    raw_message_parameters = info.metadata.get("messageParameters")
+    message_parameters: Dict[str, str] = (
+        json.loads(raw_message_parameters) if raw_message_parameters else {}
+    )
+    root_error_idx = (
+        resp.root_error_idx if resp is not None and resp.HasField("root_error_idx") else None
+    )
     converted = _convert_exception(
-        info, truncated_message, resp, display_server_stacktrace, grpc_status_code
+        classes=classes,
+        sql_state=info.metadata.get("sqlState"),
+        error_class=info.metadata.get("errorClass"),
+        reason=info.reason,
+        root_error_idx=root_error_idx,
+        errors=list(resp.errors) if resp is not None else None,
+        truncated_message=truncated_message,
+        truncated_message_parameters=message_parameters,
+        truncated_stacktrace=info.metadata.get("stackTrace"),
+        display_server_stacktrace=display_server_stacktrace,
+        grpc_status_code=grpc_status_code,
     )
     return recover_python_exception(converted)
 
 
+def convert_observation_errors(
+    root_error_idx: int,
+    errors: List["pb2.FetchErrorDetailsResponse.Error"],
+) -> SparkConnectException:
+    """
+    Convert observation error payload (root_error_idx + list of Error from ObservedMetrics)
+    to a SparkConnectException.
+    """
+    if root_error_idx < 0 or root_error_idx >= len(errors):
+        return SparkConnectException("Observation error: invalid root_error_idx")
+
+    if len(errors) == 0:
+        return SparkConnectException("Observation error: no errors")
+
+    root_error = errors[root_error_idx]
+
+    return _convert_exception(
+        classes=list(root_error.error_type_hierarchy),
+        sql_state=root_error.spark_throwable.sql_state
+        if root_error.spark_throwable.HasField("sql_state")
+        else None,
+        error_class=root_error.spark_throwable.error_class
+        if root_error.spark_throwable.HasField("error_class")
+        else None,
+        reason=None,
+        root_error_idx=root_error_idx,
+        errors=errors,
+        truncated_message="",
+        truncated_message_parameters=None,
+        truncated_stacktrace=None,
+    )
+
+
 def _convert_exception(
-    info: "ErrorInfo",
+    classes: List[str],
+    sql_state: Optional[str],
+    error_class: Optional[str],
+    reason: Optional[str],
+    root_error_idx: Optional[int],
+    errors: Optional[List["pb2.FetchErrorDetailsResponse.Error"]],
     truncated_message: str,
-    resp: Optional["pb2.FetchErrorDetailsResponse"],
+    truncated_message_parameters: Optional[Dict[str, str]],
+    truncated_stacktrace: Optional[str],
     display_server_stacktrace: bool = False,
     grpc_status_code: grpc.StatusCode = StatusCode.UNKNOWN,
 ) -> SparkConnectException:
     import pyspark.sql.connect.proto as pb2
 
-    raw_classes = info.metadata.get("classes")
-    classes: List[str] = json.loads(raw_classes) if raw_classes else []
-    sql_state = info.metadata.get("sqlState")
-    error_class = info.metadata.get("errorClass")
-    raw_message_parameters = info.metadata.get("messageParameters")
-    message_parameters: Dict[str, str] = (
-        json.loads(raw_message_parameters) if raw_message_parameters else {}
-    )
-    stacktrace: Optional[str] = None
-
-    if resp is not None and resp.HasField("root_error_idx"):
-        message = resp.errors[resp.root_error_idx].message
-        stacktrace = _extract_jvm_stacktrace(resp)
-    else:
-        message = truncated_message
-        stacktrace = info.metadata.get("stackTrace")
-        display_server_stacktrace = display_server_stacktrace if stacktrace else False
-
+    message = truncated_message
+    stacktrace = truncated_stacktrace
+    message_parameters = truncated_message_parameters
     contexts = None
     breaking_change_info = None
-    if resp and resp.HasField("root_error_idx"):
-        root_error = resp.errors[resp.root_error_idx]
+
+    if root_error_idx is not None and errors is not None:
+        root_error = errors[root_error_idx]
+        message = root_error.message
+        stacktrace = _extract_jvm_stacktrace(root_error_idx, errors)
         if hasattr(root_error, "spark_throwable"):
             # Extract errorClass from FetchErrorDetailsResponse if not in metadata
             if error_class is None and root_error.spark_throwable.HasField("error_class"):
@@ -123,6 +169,8 @@ def _convert_exception(
                         "key": bci.mitigation_config.key,
                         "value": bci.mitigation_config.value,
                     }
+    else:
+        display_server_stacktrace = display_server_stacktrace if stacktrace else False
 
     if "org.apache.spark.api.python.PythonException" in classes:
         return PythonException(
@@ -150,7 +198,7 @@ def _convert_exception(
                 sql_state=sql_state,
                 server_stacktrace=stacktrace,
                 display_server_stacktrace=display_server_stacktrace,
-                contexts=contexts,
+                contexts=contexts,  # type: ignore[arg-type]
                 grpc_status_code=grpc_status_code,
                 breaking_change_info=breaking_change_info,
             )
@@ -158,22 +206,21 @@ def _convert_exception(
     # Return UnknownException if there is no matched exception class
     return UnknownException(
         message,
-        reason=info.reason,
+        reason=reason,
         messageParameters=message_parameters,
         errorClass=error_class,
         sql_state=sql_state,
         server_stacktrace=stacktrace,
         display_server_stacktrace=display_server_stacktrace,
-        contexts=contexts,
+        contexts=contexts,  # type: ignore[arg-type]
         grpc_status_code=grpc_status_code,
         breaking_change_info=breaking_change_info,
     )
 
 
-def _extract_jvm_stacktrace(resp: "pb2.FetchErrorDetailsResponse") -> str:
-    if len(resp.errors[resp.root_error_idx].stack_trace) == 0:
-        return ""
-
+def _extract_jvm_stacktrace(
+    root_error_idx: int, errors: List["pb2.FetchErrorDetailsResponse.Error"]
+) -> str:
     lines: List[str] = []
 
     def format_stacktrace(error: "pb2.FetchErrorDetailsResponse.Error") -> None:
@@ -190,9 +237,9 @@ def _extract_jvm_stacktrace(resp: "pb2.FetchErrorDetailsResponse") -> str:
 
         # If this error has a cause, format that recursively
         if error.HasField("cause_idx"):
-            format_stacktrace(resp.errors[error.cause_idx])
+            format_stacktrace(errors[error.cause_idx])
 
-    format_stacktrace(resp.errors[resp.root_error_idx])
+    format_stacktrace(errors[root_error_idx])
 
     return "\n".join(lines)
 

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors/exceptions/tblib.py

@@ -206,9 +206,7 @@ class Traceback:
 
             # noinspection PyBroadException
             try:
-                exec(
-                    code, dict(current.tb_frame.f_globals), dict(current.tb_frame.f_locals)
-                )  # noqa: S102
+                exec(code, dict(current.tb_frame.f_globals), dict(current.tb_frame.f_locals))
             except Exception:
                 next_tb = sys.exc_info()[2].tb_next  # type: ignore
                 if top_tb is None:

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors/utils.py

@@ -271,7 +271,7 @@ def _capture_call_site(depth: int) -> str:
         import IPython
 
         # ipykernel is required for IPython
-        import ipykernel  # type: ignore[import-not-found]
+        import ipykernel
 
         ipython = IPython.get_ipython()
         # Filtering out IPython related frames

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/errors_doc_gen.py

@@ -44,7 +44,7 @@ Error classes in PySpark
 This is a list of common, named error classes returned by PySpark which are defined at `error-conditions.json <https://github.com/apache/spark/blob/master/python/pyspark/errors/error-conditions.json>`_.
 
 When writing PySpark errors, developers must use an error class from the list. If an appropriate error class is not available, add a new one into the list. For more information, please refer to `Contributing Error and Exception <contributing.rst#contributing-error-and-exception>`_.
-"""  # noqa
+"""
     with open(output_rst_file_path, "w") as f:
         f.write(header + "\n\n")
         for error_key, error_details in ERROR_CLASSES_MAP.items():

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/install.py

@@ -17,6 +17,7 @@
 import os
 import re
 import tarfile
+import time
 import traceback
 import urllib.request
 from shutil import rmtree
@@ -143,7 +144,7 @@ def install_spark(dest, spark_version, hadoop_version, hive_version):
         tar = None
         try:
             print("Downloading %s from:\n- %s" % (pretty_pkg_name, url))
-            download_to_file(urllib.request.urlopen(url), package_local_path)
+            _download_with_retries(url, package_local_path)
 
             print("Installing to %s" % dest)
             tar = tarfile.open(package_local_path, "r:gz")
@@ -171,7 +172,7 @@ def get_preferred_mirrors():
     for _ in range(3):
         try:
             response = urllib.request.urlopen(
-                "https://www.apache.org/dyn/closer.lua?preferred=true"
+                "https://www.apache.org/dyn/closer.lua?preferred=true", timeout=10
             )
             mirror_urls.append(response.read().decode("utf-8"))
         except Exception:
@@ -186,6 +187,40 @@ def get_preferred_mirrors():
     return list(set(mirror_urls)) + [x for x in default_sites if x not in mirror_urls]
 
 
+def _download_with_retries(url, path, max_retries=3, timeout=600):
+    """
+    Download a file from a URL with retry logic and timeout handling.
+
+    Parameters
+    ----------
+    url : str
+        The URL to download from.
+    path : str
+        The local file path to save the downloaded file.
+    max_retries : int
+        Maximum number of retry attempts per URL.
+    timeout : int
+        Timeout in seconds for the HTTP request.
+    """
+    for attempt in range(max_retries):
+        try:
+            response = urllib.request.urlopen(url, timeout=timeout)
+            download_to_file(response, path)
+            return
+        except Exception as e:
+            if os.path.exists(path):
+                os.remove(path)
+            if attempt < max_retries - 1:
+                wait = 2**attempt * 5
+                print(
+                    "Download attempt %d/%d failed: %s. Retrying in %d seconds..."
+                    % (attempt + 1, max_retries, str(e), wait)
+                )
+                time.sleep(wait)
+            else:
+                raise
+
+
 def download_to_file(response, path, chunk_size=1024 * 1024):
     total_size = int(response.info().get("Content-Length").strip())
     bytes_so_far = 0

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/instrumentation_utils.py

@@ -124,7 +124,7 @@ def _attach(
     logger_module: Union[str, ModuleType],
     modules: List[ModuleType],
     classes: List[Type[Any]],
-    missings: List[Tuple[Type[Any], Type[Any]]],
+    missings: List[Tuple[Union[ModuleType, Type[Any]], Type[Any]]],
 ) -> None:
     if isinstance(logger_module, str):
         logger_module = importlib.import_module(logger_module)

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/logger/__init__.py

@@ -18,6 +18,6 @@
 """
 PySpark logging
 """
-from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA  # noqa: F401
+from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA
 
 __all__ = ["PySparkLogger", "SPARK_LOG_SCHEMA"]

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/logger/worker_io.py

@@ -223,7 +223,11 @@ def context_provider() -> dict[str, str]:
             - class_name: Name of the class that initiated the logging if available
     """
 
-    def is_pyspark_module(module_name: str) -> bool:
+    def is_pyspark_module(frame: FrameType) -> bool:
+        module_name = frame.f_globals.get("__name__", "")
+        if module_name == "__main__":
+            if (mod := sys.modules.get("__main__", None)) and mod.__spec__:
+                module_name = mod.__spec__.name
         return module_name.startswith("pyspark.") and ".tests." not in module_name
 
     bottom: Optional[FrameType] = None
@@ -236,9 +240,8 @@ def context_provider() -> dict[str, str]:
         if frame:
             while frame.f_back:
                 f_back = frame.f_back
-                module_name = f_back.f_globals.get("__name__", "")
 
-                if is_pyspark_module(module_name):
+                if is_pyspark_module(f_back):
                     if not is_in_pyspark_module:
                         bottom = frame
                         is_in_pyspark_module = True

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/ml/_typing.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Any, Dict, List, TypeVar, Tuple, Union
+from typing import Any, Dict, List, TYPE_CHECKING, TypeVar, Tuple, Union
 from typing_extensions import Literal
 
 from numpy import ndarray
@@ -24,10 +24,12 @@ from py4j.java_gateway import JavaObject
 
 import pyspark.ml.base
 import pyspark.ml.param
-import pyspark.ml.util
 from pyspark.ml.linalg import Vector
 import pyspark.ml.wrapper
 
+if TYPE_CHECKING:
+    from scipy.sparse import spmatrix, sparray
+
 ParamMap = Dict[pyspark.ml.param.Param, Any]
 PipelineStage = Union[pyspark.ml.base.Estimator, pyspark.ml.base.Transformer]
 
@@ -81,4 +83,4 @@ RankingEvaluatorMetricType = Union[
     Literal["recallAtK"],
 ]
 
-VectorLike = Union[ndarray, Vector, List[float], Tuple[float, ...]]
+VectorLike = Union[ndarray, Vector, List[float], Tuple[float, ...], "spmatrix", "sparray", range]

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/ml/classification.py

@@ -2253,7 +2253,7 @@ class RandomForestClassifier(
         return self._set(minWeightFractionPerNode=value)
 
 
-class RandomForestClassificationModel(
+class RandomForestClassificationModel(  # type: ignore[misc]
     _TreeEnsembleModel,
     _JavaProbabilisticClassificationModel[Vector],
     _RandomForestClassifierParams,

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/ml/connect/base.py

@@ -155,7 +155,7 @@ class Transformer(Params, metaclass=ABCMeta):
     ) -> Union[DataFrame, pd.DataFrame]:
         """
         Transforms the input dataset.
-        The dataset can be either pandas dataframe or spark dataframe，
+        The dataset can be either pandas dataframe or spark dataframe,
         if it is a spark DataFrame, the result of transformation is a new spark DataFrame
         that contains all existing columns and output columns with names,
         If it is a pandas DataFrame, the result of transformation is a shallow copy

{pyspark_client-4.2.0.dev1 → pyspark_client-4.2.0.dev3}/pyspark/ml/functions.py

@@ -241,7 +241,7 @@ def _validate_and_transform_single_input(
         # tensor columns
         if len(batch.columns) == 1:
             # one tensor column and one expected input, vstack rows
-            single_input = np.vstack(batch.iloc[:, 0])
+            single_input = np.vstack(batch.iloc[:, 0])  # type: ignore[call-overload]
         else:
             raise ValueError(
                 "Multiple input columns found, but model expected a single "