pyspark-client 4.1.0.dev4__tar.gz → 4.2.0.dev1__tar.gz

{pyspark_client-4.1.0.dev4/pyspark_client.egg-info → pyspark_client-4.2.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspark-client
-Version: 4.1.0.dev4
+Version: 4.2.0.dev1
 Summary: Python Spark Connect client for Apache Spark
 Home-page: https://github.com/apache/spark/tree/master/python
 Author: Spark Developers

{pyspark_client-4.1.0.dev4 → pyspark_client-4.2.0.dev1}/pyspark/accumulators.py

@@ -266,12 +266,36 @@ class UpdateRequestHandler(socketserver.StreamRequestHandler):
         auth_token = self.server.auth_token  # type: ignore[attr-defined]
 
         def poll(func: Callable[[], bool]) -> None:
+            poller = None
+            if os.name == "posix":
+                # On posix systems use poll to avoid problems with file descriptor
+                # numbers above 1024.
+                poller = select.poll()
+                poller.register(self.rfile, select.POLLIN)
+
             while not self.server.server_shutdown:  # type: ignore[attr-defined]
                 # Poll every 1 second for new data -- don't block in case of shutdown.
-                r, _, _ = select.select([self.rfile], [], [], 1)
-                if self.rfile in r and func():
+                if poller is not None:
+                    r = []
+                    # Unlike select, poll timeout is in millis.
+                    for fd, event in poller.poll(1000):
+                        if event & (select.POLLIN | select.POLLHUP):
+                            # Data can be read (for POLLHUP peer hang up, so reads will return
+                            # 0 bytes, in which case we want to break out - this is consistent
+                            # with how select behaves).
+                            r.append(fd)
+                        else:
+                            # Could be POLLERR or POLLNVAL (select would raise in this case).
+                            raise PySparkRuntimeError(f"Polling error - event {event} on fd {fd}")
+                else:
+                    # If poll is not available, use select.
+                    r = select.select([self.rfile.fileno()], [], [], 1)[0]
+                if self.rfile.fileno() in r and func():
                     break
 
+            if poller is not None:
+                poller.unregister(self.rfile)
+
         def accum_updates() -> bool:
             num_updates = read_int(self.rfile)
             for _ in range(num_updates):
@@ -321,21 +345,36 @@ class AccumulatorTCPServer(socketserver.TCPServer):
         self.server_close()
 
 
-class AccumulatorUnixServer(socketserver.UnixStreamServer):
-    server_shutdown = False
+# socketserver.UnixStreamServer is not available on Windows yet
+# (https://github.com/python/cpython/issues/77589).
+if hasattr(socketserver, "UnixStreamServer"):
 
-    def __init__(
-        self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
-    ):
-        super().__init__(socket_path, RequestHandlerClass)
-        self.auth_token = None
+    class AccumulatorUnixServer(socketserver.UnixStreamServer):
+        server_shutdown = False
 
-    def shutdown(self) -> None:
-        self.server_shutdown = True
-        super().shutdown()
-        self.server_close()
-        if os.path.exists(self.server_address):  # type: ignore[arg-type]
-            os.remove(self.server_address)  # type: ignore[arg-type]
+        def __init__(
+            self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
+        ):
+            super().__init__(socket_path, RequestHandlerClass)
+            self.auth_token = None
+
+        def shutdown(self) -> None:
+            self.server_shutdown = True
+            super().shutdown()
+            self.server_close()
+            if os.path.exists(self.server_address):  # type: ignore[arg-type]
+                os.remove(self.server_address)  # type: ignore[arg-type]
+
+else:
+
+    class AccumulatorUnixServer(socketserver.TCPServer):  # type: ignore[no-redef]
+        def __init__(
+            self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
+        ):
+            raise NotImplementedError(
+                "Unix Domain Sockets are not supported on this platform. "
+                "Please disable it by setting spark.python.unix.domain.socket.enabled to false."
+            )
 
 
 def _start_update_server(

{pyspark_client-4.1.0.dev4 → pyspark_client-4.2.0.dev1}/pyspark/daemon.py

@@ -24,19 +24,13 @@ import sys
 import traceback
 import time
 import gc
+import faulthandler
 from errno import EINTR, EAGAIN
 from socket import AF_INET, AF_INET6, SOCK_STREAM, SOMAXCONN
 from signal import SIGHUP, SIGTERM, SIGCHLD, SIG_DFL, SIG_IGN, SIGINT
 
 from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
-
-if len(sys.argv) > 1 and sys.argv[1].startswith("pyspark"):
-    import importlib
-
-    worker_module = importlib.import_module(sys.argv[1])
-    worker_main = worker_module.main
-else:
-    from pyspark.worker import main as worker_main
+from pyspark.errors import PySparkRuntimeError
 
 
 def compute_real_exit_code(exit_code):
@@ -77,6 +71,19 @@ def worker(sock, authenticated):
             return 1
 
     exit_code = 0
+
+    # We don't know what could happen when we import the worker module. We have to
+    # guarantee that no thread is spawned before we fork, so we have to import the
+    # worker module after fork. For example, both pandas and pyarrow starts some
+    # threads when they are imported.
+    if len(sys.argv) > 1 and sys.argv[1].startswith("pyspark"):
+        import importlib
+
+        worker_module = importlib.import_module(sys.argv[1])
+        worker_main = worker_module.main
+    else:
+        from pyspark.worker import main as worker_main
+
     try:
         worker_main(infile, outfile)
     except SystemExit as exc:
@@ -85,7 +92,19 @@ def worker(sock, authenticated):
         try:
             outfile.flush()
         except Exception:
-            pass
+            if os.environ.get("PYTHON_DAEMON_KILL_WORKER_ON_FLUSH_FAILURE", False):
+                faulthandler_log_path = os.environ.get("PYTHON_FAULTHANDLER_DIR", None)
+                if faulthandler_log_path:
+                    faulthandler_log_path = os.path.join(faulthandler_log_path, str(os.getpid()))
+                    with open(faulthandler_log_path, "w") as faulthandler_log_file:
+                        faulthandler.dump_traceback(file=faulthandler_log_file)
+                raise
+            else:
+                print(
+                    "PySpark daemon failed to flush the output to the worker process:\n"
+                    + traceback.format_exc(),
+                    file=sys.stderr,
+                )
     return exit_code
 
 
@@ -145,14 +164,31 @@ def manager():
 
     # Initialization complete
     try:
+        poller = None
+        if os.name == "posix":
+            # select.select has a known limit on the number of file descriptors
+            # it can handle. We use select.poll instead to avoid this limit.
+            poller = select.poll()
+            fd_reverse_map = {0: 0, listen_sock.fileno(): listen_sock}
+            poller.register(0, select.POLLIN)
+            poller.register(listen_sock, select.POLLIN)
+
         while True:
-            try:
+            if poller is not None:
+                ready_fds = []
+                # Unlike select, poll timeout is in millis.
+                for fd, event in poller.poll(1000):
+                    if event & (select.POLLIN | select.POLLHUP):
+                        # Data can be read (for POLLHUP peer hang up, so reads will return
+                        # 0 bytes, in which case we want to break out - this is consistent
+                        # with how select behaves).
+                        ready_fds.append(fd_reverse_map[fd])
+                    else:
+                        # Could be POLLERR or POLLNVAL (select would raise in this case).
+                        raise PySparkRuntimeError(f"Polling error - event {event} on fd {fd}")
+            else:
+                # If poll is not available, use select.
                 ready_fds = select.select([0, listen_sock], [], [], 1)[0]
-            except select.error as ex:
-                if ex[0] == EINTR:
-                    continue
-                else:
-                    raise
 
             if 0 in ready_fds:
                 try:
@@ -190,6 +226,9 @@ def manager():
 
                 if pid == 0:
                     # in child process
+                    if poller is not None:
+                        poller.unregister(0)
+                        poller.unregister(listen_sock)
                     listen_sock.close()
 
                     # It should close the standard input in the child process so that
@@ -238,6 +277,9 @@ def manager():
                     sock.close()
 
     finally:
+        if poller is not None:
+            poller.unregister(0)
+            poller.unregister(listen_sock)
         shutdown(1)
 
 

{pyspark_client-4.1.0.dev4 → pyspark_client-4.2.0.dev1}/pyspark/errors/error-conditions.json

@@ -14,11 +14,6 @@
       "Arrow legacy IPC format is not supported in PySpark, please unset ARROW_PRE_0_15_IPC_FORMAT."
     ]
   },
-  "ATTEMPT_ANALYSIS_IN_PIPELINE_QUERY_FUNCTION": {
-    "message": [
-      "Operations that trigger DataFrame analysis or execution are not allowed in pipeline query functions. Move code outside of the pipeline query function."
-    ]
-  },
   "ATTRIBUTE_NOT_CALLABLE": {
     "message": [
       "Attribute `<attr_name>` in provided object `<obj_name>` is not callable."
@@ -27,7 +22,8 @@
   "ATTRIBUTE_NOT_SUPPORTED": {
     "message": [
       "Attribute `<attr_name>` is not supported."
-    ]
+    ],
+    "sqlState": "0A000"
   },
   "AXIS_LENGTH_MISMATCH": {
     "message": [
@@ -107,22 +103,26 @@
   "CANNOT_INFER_EMPTY_SCHEMA": {
     "message": [
       "Can not infer schema from an empty dataset."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "CANNOT_INFER_SCHEMA_FOR_TYPE": {
     "message": [
       "Can not infer schema for type: `<data_type>`."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "CANNOT_INFER_TYPE_FOR_FIELD": {
     "message": [
       "Unable to infer the type of the field `<field_name>`."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "CANNOT_MERGE_TYPE": {
     "message": [
       "Can not merge type `<data_type1>` and `<data_type2>`."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "CANNOT_OPEN_SOCKET": {
     "message": [
@@ -162,7 +162,8 @@
   "CLASSIC_OPERATION_NOT_SUPPORTED_ON_DF": {
     "message": [
       "Calling property or member '<member>' is not supported in PySpark Classic, please use Spark Connect instead."
-    ]
+    ],
+    "sqlState": "0A000"
   },
   "COLLATION_INVALID_PROVIDER": {
     "message": [
@@ -532,7 +533,8 @@
   "JVM_ATTRIBUTE_NOT_SUPPORTED": {
     "message": [
       "Attribute `<attr_name>` is not supported in Spark Connect as it depends on the JVM. If you need to use this attribute, do not use Spark Connect when creating your session. Visit https://spark.apache.org/docs/latest/sql-getting-started.html#starting-point-sparksession for creating regular Spark Session in detail."
-    ]
+    ],
+    "sqlState": "0A000"
   },
   "KEY_NOT_EXISTS": {
     "message": [
@@ -607,137 +609,164 @@
   "NOT_BOOL": {
     "message": [
       "Argument `<arg_name>` should be a bool, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_LIST_OR_STR_OR_TUPLE": {
     "message": [
       "Argument `<arg_name>` should be a bool, dict, float, int, str or tuple, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a bool, dict, float, int or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_FLOAT_OR_INT": {
     "message": [
       "Argument `<arg_name>` should be a bool, float or int, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_FLOAT_OR_INT_OR_LIST_OR_NONE_OR_STR_OR_TUPLE": {
     "message": [
       "Argument `<arg_name>` should be a bool, float, int, list, None, str or tuple, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_FLOAT_OR_INT_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a bool, float, int or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_LIST": {
     "message": [
       "Argument `<arg_name>` should be a bool or list, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_BOOL_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a bool or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_CALLABLE": {
     "message": [
       "Argument `<arg_name>` should be a callable, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN": {
     "message": [
       "Argument `<arg_name>` should be a Column, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_DATATYPE_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a Column, str or DataType, but got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_FLOAT_OR_INT_OR_LIST_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a Column, float, integer, list or string, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_INT": {
     "message": [
       "Argument `<arg_name>` should be a Column or int, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE": {
     "message": [
       "Argument `<arg_name>` should be a Column, int, list, str or tuple, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_INT_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a Column, int or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_LIST_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a Column, list or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a Column or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_COLUMN_OR_STR_OR_STRUCT": {
     "message": [
       "Argument `<arg_name>` should be a StructType, Column or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_DATAFRAME": {
     "message": [
       "Argument `<arg_name>` should be a DataFrame, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_DATATYPE_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a DataType or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_DICT": {
     "message": [
       "Argument `<arg_name>` should be a dict, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_EXPRESSION": {
     "message": [
       "Argument `<arg_name>` should be an Expression, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_FLOAT_OR_INT": {
     "message": [
       "Argument `<arg_name>` should be a float or int, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_FLOAT_OR_INT_OR_LIST_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a float, int, list or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_IMPLEMENTED": {
     "message": [
       "<feature> is not implemented."
-    ]
+    ],
+    "sqlState": "0A000"
   },
   "NOT_INT": {
     "message": [
       "Argument `<arg_name>` should be an int, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_INT_OR_SLICE_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be an int, slice or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_IN_BARRIER_STAGE": {
     "message": [
@@ -747,87 +776,104 @@
   "NOT_ITERABLE": {
     "message": [
       "<objectName> is not iterable."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST": {
     "message": [
       "Argument `<arg_name>` should be a list, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OF_COLUMN": {
     "message": [
       "Argument `<arg_name>` should be a list[Column]."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OF_COLUMN_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be a list[Column]."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OF_FLOAT_OR_INT": {
     "message": [
       "Argument `<arg_name>` should be a list[float, int], got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OF_STR": {
     "message": [
       "Argument `<arg_name>` should be a list[str], got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OR_NONE_OR_STRUCT": {
     "message": [
       "Argument `<arg_name>` should be a list, None or StructType, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OR_STR_OR_TUPLE": {
     "message": [
       "Argument `<arg_name>` should be a list, str or tuple, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_LIST_OR_TUPLE": {
     "message": [
       "Argument `<arg_name>` should be a list or tuple, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_NUMERIC_COLUMNS": {
     "message": [
       "Numeric aggregation function can only be applied on numeric columns, got <invalid_columns>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_OBSERVATION_OR_STR": {
     "message": [
       "Argument `<arg_name>` should be an Observation or str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_SAME_TYPE": {
     "message": [
       "Argument `<arg_name1>` and `<arg_name2>` should be the same type, got <arg_type1> and <arg_type2>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_STR": {
     "message": [
       "Argument `<arg_name>` should be a str, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_STRUCT": {
     "message": [
       "Argument `<arg_name>` should be a struct type, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_STR_OR_LIST_OF_RDD": {
     "message": [
       "Argument `<arg_name>` should be a str or list[RDD], got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_STR_OR_STRUCT": {
     "message": [
       "Argument `<arg_name>` should be a str or struct type, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NOT_WINDOWSPEC": {
     "message": [
       "Argument `<arg_name>` should be a WindowSpec, got <arg_type>."
-    ]
+    ],
+    "sqlState": "42K09"
   },
   "NO_ACTIVE_EXCEPTION": {
     "message": [
@@ -837,12 +883,14 @@
   "NO_ACTIVE_OR_DEFAULT_SESSION": {
     "message": [
       "No active or default Spark session found. Please create a new Spark session before running the code."
-    ]
+    ],
+    "sqlState": "08003"
   },
   "NO_ACTIVE_SESSION": {
     "message": [
       "No active Spark session found. Please create a new Spark session before running the code."
-    ]
+    ],
+    "sqlState": "08003"
   },
   "NO_OBSERVE_BEFORE_GET": {
     "message": [
@@ -867,7 +915,8 @@
   "ONLY_SUPPORTED_WITH_SPARK_CONNECT": {
     "message": [
       "<feature> is only supported with Spark Connect; however, the current Spark session does not use Spark Connect."
-    ]
+    ],
+    "sqlState": "0A000"
   },
   "PACKAGE_NOT_INSTALLED": {
     "message": [
@@ -908,7 +957,7 @@
   },
   "PIPELINE_SPEC_FILE_NOT_FOUND": {
     "message": [
-      "No pipeline.yaml or pipeline.yml file provided in arguments or found in directory `<dir_path>` or readable ancestor directories."
+      "No spark-pipeline.yaml or spark-pipeline.yml file provided in arguments or found in directory `<dir_path>` or readable ancestor directories."
     ]
   },
   "PIPELINE_SPEC_INVALID_GLOB_PATTERN": {
@@ -946,6 +995,11 @@
       "Argument <arg_name> must be a numerical column for plotting, got <arg_type>."
     ]
   },
+  "PROTOCOL_ERROR": {
+    "message": [
+      "<failure>. This usually indicates that the message does not conform to the protocol."
+    ]
+  },
   "PYTHON_HASH_SEED_NOT_SET": {
     "message": [
       "Randomness of hash of string should be disabled via PYTHONHASHSEED."
@@ -1030,7 +1084,8 @@
   "SESSION_ALREADY_EXIST": {
     "message": [
       "Cannot start a remote Spark session because there is a regular Spark session already running."
-    ]
+    ],
+    "sqlState": "08002"
   },
   "SESSION_MUTATION_IN_DECLARATIVE_PIPELINE": {
     "message": [

{pyspark_client-4.1.0.dev4 → pyspark_client-4.2.0.dev1}/pyspark/errors/exceptions/base.py

@@ -110,7 +110,7 @@ class PySparkException(Exception):
         """
         Returns an SQLSTATE as a string.
 
-        Errors generated in Python have no SQLSTATE, so it always returns None.
+        If the errorClass has no corresponding SQLSTATE, it returns None.
 
         .. versionadded:: 3.4.0
 
@@ -121,7 +121,7 @@ class PySparkException(Exception):
         :meth:`PySparkException.getMessageParameters`
         :meth:`PySparkException.getQueryContext`
         """
-        return None
+        return self._error_reader.get_sqlstate(self._errorClass)
 
     def getMessage(self) -> str:
         """

{pyspark_client-4.1.0.dev4 → pyspark_client-4.2.0.dev1}/pyspark/errors/exceptions/connect.py

@@ -99,6 +99,9 @@ def _convert_exception(
     if resp and resp.HasField("root_error_idx"):
         root_error = resp.errors[resp.root_error_idx]
         if hasattr(root_error, "spark_throwable"):
+            # Extract errorClass from FetchErrorDetailsResponse if not in metadata
+            if error_class is None and root_error.spark_throwable.HasField("error_class"):
+                error_class = root_error.spark_throwable.error_class
             message_parameters = dict(root_error.spark_throwable.message_parameters)
             contexts = [
                 SQLQueryContext(c)