pyspark-client 4.2.0.dev2__tar.gz → 4.2.0.dev4__tar.gz

{pyspark_client-4.2.0.dev2/pyspark_client.egg-info → pyspark_client-4.2.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspark-client
-Version: 4.2.0.dev2
+Version: 4.2.0.dev4
 Summary: Python Spark Connect client for Apache Spark
 Home-page: https://github.com/apache/spark/tree/master/python
 Author: Spark Developers
@@ -13,7 +13,6 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{pyspark_client-4.2.0.dev2 → pyspark_client-4.2.0.dev4}/pyspark/__init__.py

@@ -55,7 +55,7 @@ from pyspark.util import is_remote_only
 if not is_remote_only():
     from pyspark.core.rdd import RDD, RDDBarrier
     from pyspark.core.files import SparkFiles
-    from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo
+    from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo, SparkExecutorInfo
     from pyspark.core.broadcast import Broadcast
     from pyspark.core import rdd, files, status, broadcast
 
@@ -144,6 +144,7 @@ __all__ = [
     "StatusTracker",
     "SparkJobInfo",
     "SparkStageInfo",
+    "SparkExecutorInfo",
     "Profiler",
     "BasicProfiler",
     "TaskContext",

{pyspark_client-4.2.0.dev2 → pyspark_client-4.2.0.dev4}/pyspark/_globals.py

@@ -54,17 +54,13 @@ class _NoValueType:
 
     __instance = None
 
-    def __new__(cls):
+    def __new__(cls) -> "_NoValueType":
         # ensure that only one instance exists
         if not cls.__instance:
-            cls.__instance = super(_NoValueType, cls).__new__(cls)
+            cls.__instance = super().__new__(cls)
         return cls.__instance
 
-    # needed for python 2 to preserve identity through a pickle
-    def __reduce__(self):
-        return (self.__class__, ())
-
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<no value>"
 
 

{pyspark_client-4.2.0.dev2 → pyspark_client-4.2.0.dev4}/pyspark/accumulators.py

@@ -27,14 +27,14 @@ from pyspark.serializers import read_int, CPickleSerializer
 from pyspark.errors import PySparkRuntimeError
 
 if TYPE_CHECKING:
-    from pyspark._typing import SupportsIAdd  # noqa: F401
-    import socketserver.BaseRequestHandler  # type: ignore[import-not-found]
+    from pyspark._typing import SupportsIAdd
+    from socketserver import BaseRequestHandler
 
 
 __all__ = ["Accumulator", "AccumulatorParam"]
 
 T = TypeVar("T")
-U = TypeVar("U", bound="SupportsIAdd")
+U = TypeVar("U", bound=Union["SupportsIAdd", int, float, complex])
 
 pickleSer = CPickleSerializer()
 
@@ -63,7 +63,6 @@ class SpecialAccumulatorIds:
 
 
 class Accumulator(Generic[T]):
-
     """
     A shared variable that can be accumulated, i.e., has a commutative and associative "add"
     operation. Worker tasks on a Spark cluster can add values to an Accumulator with the `+=`
@@ -186,7 +185,6 @@ class Accumulator(Generic[T]):
 
 
 class AccumulatorParam(Generic[T]):
-
     """
     Helper object that defines how to accumulate values of a given type.
 
@@ -229,7 +227,6 @@ class AccumulatorParam(Generic[T]):
 
 
 class AddingAccumulatorParam(AccumulatorParam[U]):
-
     """
     An AccumulatorParam that uses the + operators to add values. Designed for simple types
     such as integers, floats, and lists. Requires the zero value for the underlying type
@@ -243,27 +240,28 @@ class AddingAccumulatorParam(AccumulatorParam[U]):
         return self.zero_value
 
     def addInPlace(self, value1: U, value2: U) -> U:
-        value1 += value2  # type: ignore[operator]
+        value1 += value2  # type: ignore[operator, assignment]
         return value1
 
 
 # Singleton accumulator params for some standard types
-INT_ACCUMULATOR_PARAM = AddingAccumulatorParam(0)  # type: ignore[type-var]
-FLOAT_ACCUMULATOR_PARAM = AddingAccumulatorParam(0.0)  # type: ignore[type-var]
-COMPLEX_ACCUMULATOR_PARAM = AddingAccumulatorParam(0.0j)  # type: ignore[type-var]
+INT_ACCUMULATOR_PARAM = AddingAccumulatorParam(0)
+FLOAT_ACCUMULATOR_PARAM = AddingAccumulatorParam(0.0)
+COMPLEX_ACCUMULATOR_PARAM = AddingAccumulatorParam(0.0j)
 
 
 class UpdateRequestHandler(socketserver.StreamRequestHandler):
-
     """
     This handler will keep polling updates from the same socket until the
     server is shutdown.
     """
 
+    server: Union["AccumulatorTCPServer", "AccumulatorUnixServer"]
+
     def handle(self) -> None:
         from pyspark.accumulators import _accumulatorRegistry
 
-        auth_token = self.server.auth_token  # type: ignore[attr-defined]
+        auth_token = self.server.auth_token
 
         def poll(func: Callable[[], bool]) -> None:
             poller = None
@@ -273,7 +271,7 @@ class UpdateRequestHandler(socketserver.StreamRequestHandler):
                 poller = select.poll()
                 poller.register(self.rfile, select.POLLIN)
 
-            while not self.server.server_shutdown:  # type: ignore[attr-defined]
+            while not self.server.server_shutdown:
                 # Poll every 1 second for new data -- don't block in case of shutdown.
                 if poller is not None:
                     r = []
@@ -299,13 +297,14 @@ class UpdateRequestHandler(socketserver.StreamRequestHandler):
         def accum_updates() -> bool:
             num_updates = read_int(self.rfile)
             for _ in range(num_updates):
-                (aid, update) = pickleSer._read_with_length(self.rfile)
+                aid, update = pickleSer._read_with_length(self.rfile)
                 _accumulatorRegistry[aid] += update
             # Write a byte in acknowledgement
             self.wfile.write(struct.pack("!b", 1))
             return False
 
         def authenticate_and_accum_updates() -> bool:
+            assert auth_token is not None
             received_token: Union[bytes, str] = self.rfile.read(len(auth_token))
             if isinstance(received_token, bytes):
                 received_token = received_token.decode("utf-8")
@@ -333,7 +332,7 @@ class AccumulatorTCPServer(socketserver.TCPServer):
     def __init__(
         self,
         server_address: Tuple[str, int],
-        RequestHandlerClass: Type["socketserver.BaseRequestHandler"],
+        RequestHandlerClass: Type["BaseRequestHandler"],
         auth_token: str,
     ):
         super().__init__(server_address, RequestHandlerClass)
@@ -352,9 +351,7 @@ if hasattr(socketserver, "UnixStreamServer"):
     class AccumulatorUnixServer(socketserver.UnixStreamServer):
         server_shutdown = False
 
-        def __init__(
-            self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
-        ):
+        def __init__(self, socket_path: str, RequestHandlerClass: Type["BaseRequestHandler"]):
             super().__init__(socket_path, RequestHandlerClass)
             self.auth_token = None
 
@@ -362,15 +359,14 @@ if hasattr(socketserver, "UnixStreamServer"):
             self.server_shutdown = True
             super().shutdown()
             self.server_close()
-            if os.path.exists(self.server_address):  # type: ignore[arg-type]
-                os.remove(self.server_address)  # type: ignore[arg-type]
+            assert isinstance(self.server_address, str)
+            if os.path.exists(self.server_address):
+                os.remove(self.server_address)
 
 else:
 
     class AccumulatorUnixServer(socketserver.TCPServer):  # type: ignore[no-redef]
-        def __init__(
-            self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
-        ):
+        def __init__(self, socket_path: str, RequestHandlerClass: Type["BaseRequestHandler"]):
             raise NotImplementedError(
                 "Unix Domain Sockets are not supported on this platform. "
                 "Please disable it by setting spark.python.unix.domain.socket.enabled to false."
@@ -381,15 +377,14 @@ def _start_update_server(
     auth_token: str, is_unix_domain_sock: bool, socket_path: Optional[str] = None
 ) -> Union[AccumulatorTCPServer, AccumulatorUnixServer]:
     """Start a TCP or Unix Domain Socket server for accumulator updates."""
+    server: Union[AccumulatorTCPServer, AccumulatorUnixServer]
     if is_unix_domain_sock:
         assert socket_path is not None
         if os.path.exists(socket_path):
             os.remove(socket_path)
         server = AccumulatorUnixServer(socket_path, UpdateRequestHandler)
     else:
-        server = AccumulatorTCPServer(
-            ("localhost", 0), UpdateRequestHandler, auth_token
-        )  # type: ignore[assignment]
+        server = AccumulatorTCPServer(("localhost", 0), UpdateRequestHandler, auth_token)
 
     thread = threading.Thread(target=server.serve_forever)
     thread.daemon = True
@@ -406,7 +401,7 @@ if __name__ == "__main__":
     # The small batch size here ensures that we see multiple batches,
     # even in these small test examples:
     globs["sc"] = SparkContext("local", "test")
-    (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+    failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
     globs["sc"].stop()
     if failure_count:
         sys.exit(-1)

{pyspark_client-4.2.0.dev2 → pyspark_client-4.2.0.dev4}/pyspark/conf.py

@@ -172,12 +172,10 @@ class SparkConf:
         return self
 
     @overload
-    def setExecutorEnv(self, key: str, value: str) -> "SparkConf":
-        ...
+    def setExecutorEnv(self, key: str, value: str) -> "SparkConf": ...
 
     @overload
-    def setExecutorEnv(self, *, pairs: List[Tuple[str, str]]) -> "SparkConf":
-        ...
+    def setExecutorEnv(self, *, pairs: List[Tuple[str, str]]) -> "SparkConf": ...
 
     def setExecutorEnv(
         self,
@@ -212,16 +210,13 @@ class SparkConf:
         return self
 
     @overload
-    def get(self, key: str) -> Optional[str]:
-        ...
+    def get(self, key: str) -> Optional[str]: ...
 
     @overload
-    def get(self, key: str, defaultValue: None) -> Optional[str]:
-        ...
+    def get(self, key: str, defaultValue: None) -> Optional[str]: ...
 
     @overload
-    def get(self, key: str, defaultValue: str) -> str:
-        ...
+    def get(self, key: str, defaultValue: str) -> str: ...
 
     def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:
         """Get the configured value for some key, or return a default otherwise."""
@@ -273,7 +268,7 @@ class SparkConf:
 def _test() -> None:
     import doctest
 
-    (failure_count, test_count) = doctest.testmod(optionflags=doctest.ELLIPSIS)
+    failure_count, test_count = doctest.testmod(optionflags=doctest.ELLIPSIS)
     if failure_count:
         sys.exit(-1)
 

{pyspark_client-4.2.0.dev2 → pyspark_client-4.2.0.dev4}/pyspark/daemon.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import uuid
-import numbers
 import os
 import signal
 import select
@@ -28,20 +27,23 @@ import faulthandler
 from errno import EINTR, EAGAIN
 from socket import AF_INET, AF_INET6, SOCK_STREAM, SOMAXCONN
 from signal import SIGHUP, SIGTERM, SIGCHLD, SIG_DFL, SIG_IGN, SIGINT
+from types import FrameType
+from typing import Any, Optional
 
 from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
+from pyspark.util import enable_faulthandler
 from pyspark.errors import PySparkRuntimeError
 
 
-def compute_real_exit_code(exit_code):
-    # SystemExit's code can be integer or string, but os._exit only accepts integers
-    if isinstance(exit_code, numbers.Integral):
+def compute_real_exit_code(exit_code: Any) -> int:
+    # SystemExit's code can be anything, but os._exit only accepts integer
+    if isinstance(exit_code, int):
         return exit_code
     else:
         return 1
 
 
-def worker(sock, authenticated):
+def worker(sock: socket.socket, authenticated: bool) -> int:
     """
     Called by a worker process after the fork().
     """
@@ -108,7 +110,7 @@ def worker(sock, authenticated):
     return exit_code
 
 
-def manager():
+def manager() -> None:
     # Create a new process group to corral our children
     os.setpgid(0, 0)
 
@@ -145,7 +147,7 @@ def manager():
         write_int(listen_port, stdout_bin)
     stdout_bin.flush()
 
-    def shutdown(code):
+    def shutdown(code: int) -> None:
         if socket_path is not None and os.path.exists(socket_path):
             os.remove(socket_path)
         signal.signal(SIGTERM, SIG_DFL)
@@ -153,7 +155,7 @@ def manager():
         os.kill(0, SIGHUP)
         sys.exit(code)
 
-    def handle_sigterm(*args):
+    def handle_sigterm(signal_number: int, frame: Optional[FrameType]) -> None:
         shutdown(1)
 
     signal.signal(SIGTERM, handle_sigterm)  # Gracefully exit on SIGTERM
@@ -226,53 +228,54 @@ def manager():
 
                 if pid == 0:
                     # in child process
-                    if poller is not None:
-                        poller.unregister(0)
-                        poller.unregister(listen_sock)
-                    listen_sock.close()
+                    with enable_faulthandler():
+                        if poller is not None:
+                            poller.unregister(0)
+                            poller.unregister(listen_sock)
+                        listen_sock.close()
 
-                    # It should close the standard input in the child process so that
-                    # Python native function executions stay intact.
-                    #
-                    # Note that if we just close the standard input (file descriptor 0),
-                    # the lowest file descriptor (file descriptor 0) will be allocated,
-                    # later when other file descriptors should happen to open.
-                    #
-                    # Therefore, here we redirects it to '/dev/null' by duplicating
-                    # another file descriptor for '/dev/null' to the standard input (0).
-                    # See SPARK-26175.
-                    devnull = open(os.devnull, "r")
-                    os.dup2(devnull.fileno(), 0)
-                    devnull.close()
+                        # It should close the standard input in the child process so that
+                        # Python native function executions stay intact.
+                        #
+                        # Note that if we just close the standard input (file descriptor 0),
+                        # the lowest file descriptor (file descriptor 0) will be allocated,
+                        # later when other file descriptors should happen to open.
+                        #
+                        # Therefore, here we redirects it to '/dev/null' by duplicating
+                        # another file descriptor for '/dev/null' to the standard input (0).
+                        # See SPARK-26175.
+                        devnull = open(os.devnull, "r")
+                        os.dup2(devnull.fileno(), 0)
+                        devnull.close()
 
-                    try:
-                        # Acknowledge that the fork was successful
-                        outfile = sock.makefile(mode="wb")
-                        write_int(os.getpid(), outfile)
-                        outfile.flush()
-                        outfile.close()
-                        authenticated = (
-                            os.environ.get("PYTHON_UNIX_DOMAIN_ENABLED", "false").lower() == "true"
-                            or False
-                        )
-                        while True:
-                            code = worker(sock, authenticated)
-                            if code == 0:
-                                authenticated = True
-                            if not reuse or code:
-                                # wait for closing
-                                try:
-                                    while sock.recv(1024):
+                        try:
+                            # Acknowledge that the fork was successful
+                            outfile = sock.makefile(mode="wb")
+                            write_int(os.getpid(), outfile)
+                            outfile.flush()
+                            outfile.close()
+                            authenticated = (
+                                os.environ.get("PYTHON_UNIX_DOMAIN_ENABLED", "false").lower()
+                                == "true"
+                            )
+                            while True:
+                                code = worker(sock, authenticated)
+                                if code == 0:
+                                    authenticated = True
+                                if not reuse or code:
+                                    # wait for closing
+                                    try:
+                                        while sock.recv(1024):
+                                            pass
+                                    except Exception:
                                         pass
-                                except Exception:
-                                    pass
-                                break
-                            gc.collect()
-                    except BaseException:
-                        traceback.print_exc()
-                        os._exit(1)
-                    else:
-                        os._exit(0)
+                                    break
+                                gc.collect()
+                        except BaseException:
+                            traceback.print_exc()
+                            os._exit(1)
+                        else:
+                            os._exit(0)
                 else:
                     sock.close()
 

{pyspark_client-4.2.0.dev2 → pyspark_client-4.2.0.dev4}/pyspark/errors/__init__.py

@@ -18,7 +18,8 @@
 """
 PySpark exceptions.
 """
-from pyspark.errors.exceptions.base import (  # noqa: F401
+
+from pyspark.errors.exceptions.base import (
     PySparkException,
     AnalysisException,
     SessionNotSameException,
@@ -53,7 +54,6 @@ from pyspark.errors.exceptions.base import (  # noqa: F401
     PickleException,
 )
 
-
 __all__ = [
     "PySparkException",
     "AnalysisException",