panda-server 0.4.0__tar.gz → 0.4.2__tar.gz

{panda_server-0.4.0 → panda_server-0.4.2}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.3
 Name: panda-server
-Version: 0.4.0
+Version: 0.4.2
 Summary: PanDA Server Package
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>
 License: Apache-2.0
-License-File: LICENSE.txt
 Requires-Python: >=3.8
 Requires-Dist: cwl-utils>=0.13
 Requires-Dist: idds-atlas

panda_server-0.4.2/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "0.4.2"

{panda_server-0.4.0 → panda_server-0.4.2}/package/hatch_build.py

@@ -16,6 +16,7 @@ import requests
 from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 
 PACKAGE_EMOJI = ":panda_face:"
+PACKAGE_NAME = "panda-server"
 
 
 def get_user():
@@ -76,7 +77,7 @@ def mm_notification():
     # On the repository name we enter an empty space to prevent the URLs to preview on Mattermost
     # We shorten the commit hash to the first seven characters, as they are usually enough to identify a commit
     mm_message = {
-        "text": f"{PACKAGE_EMOJI}**Package upgrade on:** `{server_name}` by `{user}`.",
+        "text": f"{PACKAGE_EMOJI}**{PACKAGE_NAME}@{branch_name} upgrade on:** `{server_name}` by `{user}`.",
         "props": {
             "card": f"""
 | **Property** | **Value** |

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/brokerage/SiteMapper.py

@@ -167,7 +167,7 @@ class SiteMapper:
         # calculate the minRSS for the child queue
         if resource_spec.minrampercore is not None:
             child_site_spec.minrss = max(
-                child_site_spec.coreCount * resource_spec.minrampercore,
+                child_site_spec.coreCount * resource_spec.minrampercore - child_site_spec.coreCount + 1,
                 site_spec.minrss * child_site_spec.coreCount / core_count,
             )
         else:

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/configurator/Configurator.py

@@ -560,7 +560,7 @@ class NetworkConfigurator(threading.Thread):
         if hasattr(panda_config, "NWS_URL"):
             self.NWS_URL = panda_config.NWS_URL
         else:
-            self.NWS_URL = "http://atlas-adc-netmetrics-lb.cern.ch/metrics/latest.json"
+            self.NWS_URL = "https://atlas-rucio-network-metrics.cern.ch/metrics.json"
 
         if hasattr(panda_config, "CRIC_URL_CM"):
             self.CRIC_URL_CM = panda_config.CRIC_URL_CM
@@ -705,21 +705,6 @@ class NetworkConfigurator(threading.Thread):
             except KeyError:
                 pass
 
-            # PerfSonar latency and packetloss
-            for metric in [LATENCY, PACKETLOSS]:
-                try:
-                    struc = self.nws_dump[src_dst][metric]
-                    try:
-                        updated_at = datetime.strptime(struc[TIMESTAMP], "%Y-%m-%dT%H:%M:%S")
-                        if updated_at > latest_validity:
-                            value = struc[LATEST]
-                            data.append((source, destination, metric, value, updated_at))
-                    except KeyError:
-                        self.log_stream.debug(f"Entry {struc} ({source}->{destination}) does not follow {metric} standards")
-                        pass
-                except KeyError:
-                    continue
-
         return data
 
     def process_CRIC_cm_dump(self):
@@ -842,7 +827,7 @@ class SWTagsDumper(threading.Thread):
         else:
             self.log_stream = _logger
 
-        if hasattr(panda_config, "CRIC_URL_SCHEDCONFIG"):
+        if hasattr(panda_config, "CRIC_URL_TAGS"):
             self.CRIC_URL_TAGS = panda_config.CRIC_URL_TAGS
         else:
             self.CRIC_URL_TAGS = "https://atlas-cric.cern.ch/api/atlas/pandaqueue/query/?json&preset=tags"

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/daemons/utils.py

@@ -39,11 +39,10 @@ CMD_STOP = "__STOP"
 # epoch datetime
 EPOCH = datetime.datetime.fromtimestamp(0)
 
-
+# requester id for taskbuffer
 requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
 
 
-# kill process tree
 def kill_proc_tree(pid, sig=signal.SIGKILL, include_parent=True, timeout=None, on_terminate=None):
     """
     Kill a process tree (including grandchildren) with signal "sig" and return a (gone, still_alive) tuple.
@@ -63,8 +62,10 @@ def kill_proc_tree(pid, sig=signal.SIGKILL, include_parent=True, timeout=None, o
     return (gone, alive)
 
 
-# worker process loop of daemon
 def daemon_loop(dem_config, msg_queue, pipe_conn, worker_lifetime, tbuf=None, lock_pool=None):
+    """
+    Main loop of daemon worker process
+    """
     # pid of the worker
     my_pid = os.getpid()
     my_full_pid = f"{socket.getfqdn().split('.')[0]}-{os.getpgrp()}-{my_pid}"
@@ -79,6 +80,7 @@ def daemon_loop(dem_config, msg_queue, pipe_conn, worker_lifetime, tbuf=None, lo
 
     for sig in END_SIGNALS:
         signal.signal(sig, got_end_sig)
+
     # dict of all daemons and their script module object
     module_map = {}
     # package of daemon scripts
@@ -271,9 +273,6 @@ def daemon_loop(dem_config, msg_queue, pipe_conn, worker_lifetime, tbuf=None, lo
             # send daemon status back to master
             status_tuple = (dem_name, to_run_daemon, has_run, last_run_start_ts, last_run_end_ts)
             pipe_conn.send(status_tuple)
-            # FIXME: stop and spawn worker in every run for now since some script breaks the worker without exception
-            # tmp_log.info('as script done, stop this worker')
-            # break
         else:
             # got invalid message
             tmp_log.warning(f'got invalid message "{one_msg}", skipped it')
@@ -281,8 +280,11 @@ def daemon_loop(dem_config, msg_queue, pipe_conn, worker_lifetime, tbuf=None, lo
         time.sleep(2**-5)
 
 
-# worker class of daemon process for PanDA server
 class DaemonWorker(object):
+    """
+    Class of worker process of PanDA daemon
+    """
+
     __slots__ = (
         "pid",
         "parent_conn",
@@ -308,17 +310,23 @@ class DaemonWorker(object):
                 lock_pool=lock_pool,
             )
 
-    # make pipe connection pairs for the worker
     def _make_pipe(self):
+        """
+        make pipe connection pairs between master and this worker
+        """
         self.parent_conn, self.child_conn = multiprocessing.Pipe()
 
-    # close pipe connections
     def _close_pipe(self):
+        """
+        close pipe connection pairs between master and this worker
+        """
         self.parent_conn.close()
         self.child_conn.close()
 
-    # make associated process
     def _make_process(self, dem_config, msg_queue, worker_lifetime, tbuf, lock_pool):
+        """
+        make associate process of this worker
+        """
         args = (
             dem_config,
             msg_queue,
@@ -329,39 +337,54 @@ class DaemonWorker(object):
         )
         self.process = multiprocessing.Process(target=daemon_loop, args=args)
 
-    # start worker process
     def start(self):
+        """
+        start the worker process
+        """
         self.unset_dem()
         self.process.start()
         self.pid = self.process.pid
 
-    # whether worker process is alive
     def is_alive(self):
+        """
+        whether the worker process is alive
+        """
         return self.process.is_alive()
 
-    # kill the worker process and all its subprocesses
     def kill(self):
+        """
+        kill the worker process and all its subprocesses
+        """
         self._close_pipe()
         return kill_proc_tree(self.process.pid)
 
-    # whether the worker is running daemon
     def is_running_dem(self):
+        """
+        whether the worker is still running a daemon script
+        """
         return not (self.dem_name is None and self.dem_ts is None)
 
-    # set current running daemon in this worker
     def set_dem(self, dem_name, dem_ts):
+        """
+        set current running daemon in this worker
+        """
         if not self.is_running_dem() or dem_ts >= self.dem_ts:
             self.dem_name = dem_name
             self.dem_ts = dem_ts
 
-    # unset current running daemon in this worker
     def unset_dem(self):
+        """
+        unset current running daemon in this worker
+        """
         self.dem_ts = None
         self.dem_name = None
 
 
-# master class of main daemon process for PanDA server
 class DaemonMaster(object):
+    """
+    Class of master process of PanDA daemon
+    """
+
     # constructor
     def __init__(self, logger, n_workers=1, n_dbconn=1, worker_lifetime=28800, use_tbif=False):
         # logger
@@ -400,11 +423,16 @@ class DaemonMaster(object):
         self._spawn_workers(self.n_workers)
 
     def _reset_msg_queue(self):
+        """
+        reset the message queue for sending commands to workers
+        """
         self.msg_queue = multiprocessing.Queue()
         self.logger.info(f"reset message queue (qid={id(self.msg_queue)})")
 
-    # make common taskBuffer interface for daemon workers
     def _make_tbif(self):
+        """
+        make common taskBuffer interface for daemon workers
+        """
         try:
             # import is always required to have reserveChangedState consistent in *Spec
             from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
@@ -430,8 +458,10 @@ class DaemonMaster(object):
             self.logger.error(f"failed to initialize taskBuffer interface with {e.__class__.__name__}: {e} ; terminated")
             raise e
 
-    # spawn new workers and put into worker pool
     def _spawn_workers(self, n_workers=1, auto_start=False):
+        """
+        spawn new workers and put them into worker pool
+        """
         for j in range(n_workers):
             with self._worker_lock:
                 if self.use_tbif:
@@ -450,13 +480,17 @@ class DaemonMaster(object):
                     worker.start()
                     self.logger.debug(f"launched new worker_pid={worker.pid}")
 
-    # remove a worker from pool
     def _remove_worker(self, worker):
+        """
+        remove a worker from pool
+        """
         with self._worker_lock:
             self.worker_pool.discard(worker)
 
-    # parse daemon config
     def _parse_config(self):
+        """
+        parse configuration of PanDA daemon
+        """
         try:
             config_json = daemon_config.config
             config_dict = json.loads(config_json)
@@ -495,8 +529,10 @@ class DaemonMaster(object):
             tb = traceback.format_exc()
             self.logger.error(f"failed to parse daemon config, {e.__class__.__name__}: {e}\n{tb}\n")
 
-    # make daemon run status map
     def _make_dem_run_map(self):
+        """
+        initialize daemon run status map
+        """
         dem_run_map = {}
         for dem in self.dem_config:
             attrs = {}
@@ -507,8 +543,21 @@ class DaemonMaster(object):
             dem_run_map[dem] = attrs
         self.dem_run_map = dem_run_map
 
-    # one scheduler cycle
+    def _kill_one_worker(self, worker):
+        """
+        kill one (stuck) worker (and new worker will be re-spawned in scheduler cycle)
+        """
+        # kill worker process and remove it from pool
+        worker.kill()
+        self._remove_worker(worker)
+        # reset daemon run status map of the daemon run by the worker
+        self.dem_run_map[worker.dem_name]["msg_ongoing"] = False
+        self.dem_run_map[worker.dem_name]["dem_running"] = False
+
     def _scheduler_cycle(self):
+        """
+        main scheduler cycle
+        """
         now_ts = int(time.time())
         # check last run time from pipes
         for worker in list(self.worker_pool):
@@ -561,10 +610,7 @@ class DaemonMaster(object):
                                 timeout=run_timeout,
                             )
                         )
-                        worker.kill()
-                        self._remove_worker(worker)
-                        self.dem_run_map[worker.dem_name]["msg_ongoing"] = False
-                        self.dem_run_map[worker.dem_name]["dem_running"] = False
+                        self._kill_one_worker(worker)
         # counter for super delayed daemons
         n_super_delayed_dems = 0
         # send message to workers
@@ -607,19 +653,28 @@ class DaemonMaster(object):
         # sleep
         time.sleep(0.5)
 
-    # send stop command to all worker processes
-    def _stop_proc(self):
+    def _stop_all_workers(self):
+        """
+        stop all workers gracefully by sending stop command to them
+        """
+        # send stop command
         for worker in self.worker_pool:
             worker.parent_conn.send(CMD_STOP)
             self.logger.debug(f"sent stop command to worker_pid={worker.pid}")
+        # reset daemon run status map of all daemons
+        for dem_name in self.dem_config:
+            self.dem_run_map[dem_name]["msg_ongoing"] = False
+            self.dem_run_map[dem_name]["dem_running"] = False
 
-    # stop master
     def stop(self):
+        """
+        stop the master (and all workers)
+        """
         self.logger.info("daemon master got stop")
         # stop scheduler from sending more message
         self.to_stop_scheduler = True
-        # send stop command to workers
-        self._stop_proc()
+        # stop all workers gracefully
+        self._stop_all_workers()
         # wait a bit
         time.sleep(1)
         # close message queue
@@ -630,11 +685,13 @@ class DaemonMaster(object):
         # wait a bit
         time.sleep(2)
 
-    # revive: kill all workers, reset a new message queue, and spawn new workers with new queue
     def revive(self):
+        """
+        revive: kill all workers, reset a new message queue, and spawn new workers with new queue
+        """
         self.logger.info("daemon master reviving")
-        # send stop command to workers
-        self._stop_proc()
+        # stop all workers gracefully
+        self._stop_all_workers()
         # wait a bit
         time.sleep(3)
         # kill and remove workers
@@ -650,8 +707,10 @@ class DaemonMaster(object):
         # done
         self.logger.info("daemon master revived")
 
-    # run
     def run(self):
+        """
+        main function to run the master
+        """
         # master pid
         master_pid = os.getpid()
         self.logger.info(f"daemon master started ; master_pid={master_pid}")

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/dataservice/setupper_atlas_plugin.py

@@ -338,8 +338,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                             self.replica_map[job.dispatchDBlock][file.dataset] = self.all_replica_map[file.dataset]
 
         # register dispatch dataset
-        disp_list = self.register_dispatch_datasets(file_list, use_zip_to_pin_map, ds_task_map, tmp_logger, disp_error,
-                                                    jedi_task_id)
+        disp_list = self.register_dispatch_datasets(file_list, use_zip_to_pin_map, ds_task_map, tmp_logger, disp_error, jedi_task_id)
         # insert datasets to DB
         self.task_buffer.insertDatasets(prod_list + disp_list)
         # job status
@@ -356,13 +355,13 @@ class SetupperAtlasPlugin(SetupperPluginBase):
         del prod_error
 
     def register_dispatch_datasets(
-            self,
-            file_list: Dict[str, Dict[str, List[str]]],
-            use_zip_to_pin_map: Dict[str, bool],
-            ds_task_map: Dict[str, int],
-            tmp_logger: LogWrapper,
-            disp_error: Dict[str, str],
-            jedi_task_id: Optional[int]
+        self,
+        file_list: Dict[str, Dict[str, List[str]]],
+        use_zip_to_pin_map: Dict[str, bool],
+        ds_task_map: Dict[str, int],
+        tmp_logger: LogWrapper,
+        disp_error: Dict[str, str],
+        jedi_task_id: Optional[int],
     ) -> List[DatasetSpec]:
         """
         Register dispatch datasets in Rucio.
@@ -396,10 +395,10 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                     tmp_zip_out = {}
                 dis_files = {"lfns": [], "guids": [], "fsizes": [], "chksums": []}
                 for tmp_lfn, tmp_guid, tmp_file_size, tmp_checksum in zip(
-                        file_list[dispatch_data_block]["lfns"],
-                        file_list[dispatch_data_block]["guids"],
-                        file_list[dispatch_data_block]["fsizes"],
-                        file_list[dispatch_data_block]["chksums"],
+                    file_list[dispatch_data_block]["lfns"],
+                    file_list[dispatch_data_block]["guids"],
+                    file_list[dispatch_data_block]["fsizes"],
+                    file_list[dispatch_data_block]["chksums"],
                 ):
                     if tmp_lfn in tmp_zip_out:
                         tmp_zip_file_name = f"{tmp_zip_out[tmp_lfn]['scope']}:{tmp_zip_out[tmp_lfn]['name']}"
@@ -444,9 +443,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                     self.logger.debug(f"sleep {attempt}/{max_attempt}")
                     time.sleep(10)
             if not is_ok:
-                disp_error[
-                    dispatch_data_block] = "setupper.setup_source() could not register dispatch_data_block with {0}".format(
-                    err_str.split("\n")[-1])
+                disp_error[dispatch_data_block] = "setupper.setup_source() could not register dispatch_data_block with {0}".format(err_str.split("\n")[-1])
                 continue
             tmp_logger.debug(out)
             new_out = out
@@ -464,8 +461,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                     time.sleep(10)
             if not status:
                 tmp_logger.error(out)
-                disp_error[
-                    dispatch_data_block] = f"setupper.setup_source() could not freeze dispatch_data_block with {out}"
+                disp_error[dispatch_data_block] = f"setupper.setup_source() could not freeze dispatch_data_block with {out}"
                 continue
 
             # get VUID
@@ -479,8 +475,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                 dataset.status = "defined"
                 dataset.numberfiles = len(file_list[dispatch_data_block]["lfns"])
                 try:
-                    dataset.currentfiles = int(
-                        sum(filter(None, file_list[dispatch_data_block]["fsizes"])) / 1024 / 1024)
+                    dataset.currentfiles = int(sum(filter(None, file_list[dispatch_data_block]["fsizes"])) / 1024 / 1024)
                 except Exception:
                     dataset.currentfiles = 0
                 if jedi_task_id is not None:
@@ -774,6 +769,11 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                 # set new destDBlock
                 if dest in newname_list:
                     file.destinationDBlock = newname_list[dest]
+        # update job status if failed and increment number of files
+        for job in jobs_list:
+            # ignore failed jobs
+            if job.jobStatus in ["failed", "cancelled"] or job.isCancelled():
+                continue
             for file in job.Files:
                 dest = (
                     file.destinationDBlock,

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/jobdispatcher/JobDispatcher.py

@@ -20,10 +20,9 @@ from pandacommon.pandalogger.PandaLogger import PandaLogger
 from pandaserver.brokerage.SiteMapper import SiteMapper
 from pandaserver.config import panda_config
 from pandaserver.dataservice.adder_gen import AdderGen
-from pandaserver.jobdispatcher import DispatcherUtils, Protocol
+from pandaserver.jobdispatcher import Protocol
 from pandaserver.proxycache import panda_proxy_cache, token_cache
 from pandaserver.srvcore import CoreUtils
-from pandaserver.taskbuffer import EventServiceUtils
 
 # logger
 _logger = PandaLogger().getLogger("JobDispatcher")
@@ -164,7 +163,6 @@ class JobDispatcher:
     ):
         t_getJob_start = time.time()
         jobs = []
-        useProxyCache = False
         try:
             tmpNumJobs = int(nJobs)
         except Exception:
@@ -237,55 +235,6 @@ class JobDispatcher:
                     response.appendNode("secrets", secrets_map[tmpJob.prodUserName])
                 if panda_config.pilot_secrets in secrets_map and secrets_map[panda_config.pilot_secrets]:
                     response.appendNode("pilotSecrets", secrets_map[panda_config.pilot_secrets])
-                # check if proxy cache is used
-                if hasattr(panda_config, "useProxyCache") and panda_config.useProxyCache is True:
-                    self.specialDispatchParams.update()
-                    if "proxyCacheSites" not in self.specialDispatchParams:
-                        proxyCacheSites = {}
-                    else:
-                        proxyCacheSites = self.specialDispatchParams["proxyCacheSites"]
-                    if siteName in proxyCacheSites:
-                        useProxyCache = True
-                # set proxy
-                if useProxyCache:
-                    try:
-                        #  get compact
-                        compactDN = self.taskBuffer.cleanUserID(realDN)
-                        # check permission
-                        self.specialDispatchParams.update()
-                        if "allowProxy" not in self.specialDispatchParams:
-                            allowProxy = []
-                        else:
-                            allowProxy = self.specialDispatchParams["allowProxy"]
-                        if compactDN not in allowProxy:
-                            tmpLog.warning(f"{siteName} {node} '{compactDN}' no permission to retrieve user proxy")
-                        else:
-                            if useProxyCache:
-                                tmpStat, tmpOut = self.set_user_proxy(
-                                    response,
-                                    proxyCacheSites[siteName]["dn"],
-                                    proxyCacheSites[siteName]["role"],
-                                )
-                            else:
-                                tmpStat, tmpOut = self.set_user_proxy(response)
-                            if not tmpStat:
-                                tmpLog.warning(f"{siteName} {node} failed to get user proxy : {tmpOut}")
-                    except Exception as e:
-                        tmpLog.warning(f"{siteName} {node} failed to get user proxy with {str(e)}")
-                # panda proxy
-                if (
-                    "pandaProxySites" in self.specialDispatchParams
-                    and siteName in self.specialDispatchParams["pandaProxySites"]
-                    and (EventServiceUtils.isEventServiceJob(tmpJob) or EventServiceUtils.isEventServiceMerge(tmpJob))
-                ):
-                    # get secret key
-                    tmpSecretKey, tmpErrMsg = DispatcherUtils.getSecretKey(tmpJob.PandaID)
-                    if tmpSecretKey is None:
-                        tmpLog.warning(f"PandaID={tmpJob.PandaID} site={siteName} failed to get panda proxy secret key : {tmpErrMsg}")
-                    else:
-                        # set secret key
-                        tmpLog.debug(f"PandaID={tmpJob.PandaID} set key={tmpSecretKey}")
-                        response.setPandaProxySecretKey(tmpSecretKey)
                 # add
                 responseList.append(response.data)
             # make response for bulk
@@ -648,14 +597,6 @@ class JobDispatcher:
         # return
         return response.encode(accept_json)
 
-    # get DNs authorized for S3
-    def getDNsForS3(self):
-        # check permission
-        self.specialDispatchParams.update()
-        allowKey = self.specialDispatchParams.get("allowKeyPair", [])
-        # return
-        return json.dumps(allowKey)
-
     # get site mapper
     def getSiteMapper(self):
         return True, SiteMapper(self.taskBuffer)
@@ -1549,15 +1490,6 @@ def checkPilotPermission(req):
     return True, None
 
 
-def getDNsForS3(req):
-    """
-    # get DNs authorized for S3
-    :param req:
-    :return:
-    """
-    return jobDispatcher.getDNsForS3()
-
-
 def getCommands(req, harvester_id, n_commands, timeout=30):
     """
     This function checks the permissions and retrieves the commands for a specified harvester instance.

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/server/panda.py

@@ -35,7 +35,6 @@ from pandaserver.jobdispatcher.JobDispatcher import (
     get_max_worker_id,
     get_token_key,
     getCommands,
-    getDNsForS3,
     getEventRanges,
     getJob,
     getKeyPair,
@@ -125,7 +124,6 @@ from pandaserver.userinterface.UserIF import (
     relay_idds_command,
     release_task,
     reloadInput,
-    reportWorkerStats,
     reportWorkerStats_jobtype,
     resumeTask,
     retryTask,

{panda_server-0.4.0 → panda_server-0.4.2}/pandaserver/srvcore/allowed_methods.py

@@ -29,7 +29,6 @@ allowed_methods += [
     "updateEventRange",
     "getKeyPair",
     "updateEventRanges",
-    "getDNsForS3",
     "getProxy",
     "get_access_token",
     "get_token_key",
@@ -83,7 +82,6 @@ allowed_methods += [
     "getTaskParamsMap",
     "updateWorkers",
     "harvesterIsAlive",
-    "reportWorkerStats",
     "reportWorkerStats_jobtype",
     "getWorkerStats",
     "addHarvesterDialogs",