DIRAC 9.0.7__py3-none-any.whl → 9.0.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  DIRAC Wrapper to execute python and system commands with a wrapper, that might
3
3
  set a timeout.
4
- 3 FUNCTIONS are provided:
4
+ 3 functions are provided:
5
5
 
6
6
  - shellCall( iTimeOut, cmdSeq, callbackFunction = None, env = None ):
7
7
  it uses subprocess.Popen class with "shell = True".
@@ -26,6 +26,7 @@ set a timeout.
26
26
  should be used to wrap third party python functions
27
27
 
28
28
  """
29
+
29
30
  import os
30
31
  import selectors
31
32
  import signal
@@ -161,7 +162,6 @@ class Subprocess:
161
162
 
162
163
  self.child = None
163
164
  self.childPID = 0
164
- self.childKilled = False
165
165
  self.callback = None
166
166
  self.bufferList = []
167
167
  self.cmdSeq = []
@@ -193,7 +193,7 @@ class Subprocess:
193
193
  f"First and last data in buffer: \n{dataString[:100]} \n....\n {dataString[-100:]} ",
194
194
  )
195
195
  retDict = S_ERROR(
196
- "Reached maximum allowed length (%d bytes) " "for called function return value" % self.bufferLimit
196
+ "Reached maximum allowed length (%d bytes) for called function return value" % self.bufferLimit
197
197
  )
198
198
  retDict["Value"] = dataString
199
199
  return retDict
@@ -241,60 +241,71 @@ class Subprocess:
241
241
  events = sel.select(timeout=timeout or self.timeout or None)
242
242
  return [key.fileobj for key, event in events if event & selectors.EVENT_READ]
243
243
 
244
- def __killPid(self, pid, sig=9):
245
- """send signal :sig: to process :pid:
246
-
247
- :param int pid: process id
248
- :param int sig: signal to send, default 9 (SIGKILL)
249
- """
244
+ def __terminateProcess(self, process):
245
+ """Tries to terminate a process with SIGTERM. Returns a (gone, alive) tuple"""
246
+ self.log.verbose(f"Sending SIGTERM signal to PID {process.pid}")
250
247
  try:
251
- os.kill(pid, sig)
252
- except Exception as x:
253
- if str(x) != "[Errno 3] No such process":
254
- self.log.exception("Exception while killing timed out process")
255
- raise x
256
-
257
- def __poll(self, pid):
258
- """wait for :pid:"""
248
+ process.terminate()
249
+ except psutil.NoSuchProcess:
250
+ return ([], [])
251
+ return psutil.wait_procs([process], timeout=60)
252
+
253
+ def __poll(self, process):
254
+ """Non-blocking check of whether process `pid` is still alive.
255
+ Returns:
256
+ - (0, 0) if process is still running (like os.waitpid(pid, os.WNOHANG))
257
+ - (pid, exitcode) if process has terminated
258
+ - None if process info cannot be retrieved
259
+ """
259
260
  try:
260
- return os.waitpid(pid, os.WNOHANG)
261
- except os.error:
262
- if self.childKilled:
263
- return False
261
+ exitcode = process.wait(timeout=0)
262
+ return (process.pid, exitcode) # exited
263
+ except psutil.TimeoutExpired:
264
+ return (0, 0) # still running
265
+ except psutil.NoSuchProcess:
264
266
  return None
265
267
 
266
268
  def killChild(self, recursive=True):
267
- """kill child process
268
-
269
- :param boolean recursive: flag to kill all descendants
269
+ """Kills a process tree (including children) with signal SIGTERM. If that fails, escalate to SIGKILL
270
+ returns (gone, alive) tuple.
270
271
  """
271
- pgid = os.getpgid(self.childPID)
272
- if pgid != os.getpgrp():
273
- try:
274
- # Child is in its own group: kill the group
275
- os.killpg(pgid, signal.SIGTERM)
276
- except OSError:
277
- # Process is already dead
278
- pass
279
- else:
280
- # No separate group: walk the tree
281
- parent = psutil.Process(self.childPID)
282
- procs = parent.children(recursive=recursive)
283
- procs.append(parent)
284
- for p in procs:
272
+
273
+ self.log.info(f"Killing childPID {self.childPID}")
274
+
275
+ gone, alive = [], []
276
+ try:
277
+ child_process = psutil.Process(self.childPID)
278
+ except psutil.NoSuchProcess:
279
+ self.log.warn(f"Child PID {self.childPID} no longer exists")
280
+ return (gone, alive)
281
+
282
+ if recursive:
283
+ # grandchildren
284
+ children = child_process.children(recursive=True)
285
+ self.log.info(f"Sending kill signal to {len(children)} children PIDs")
286
+ for p in children:
285
287
  try:
286
288
  p.terminate()
287
289
  except psutil.NoSuchProcess:
288
- pass
289
- _gone, alive = psutil.wait_procs(procs, timeout=10)
290
- # Escalate any survivors
290
+ continue
291
+ g, a = psutil.wait_procs(children, timeout=60)
292
+ gone.extend(g)
293
+ alive.extend(a)
294
+
295
+ # now killing the child_process
296
+ g, a = self.__terminateProcess(child_process)
297
+ gone.extend(g)
298
+ alive.extend(a)
299
+
300
+ # if there's something still alive, use SIGKILL
301
+ if alive:
291
302
  for p in alive:
292
303
  try:
293
304
  p.kill()
294
305
  except psutil.NoSuchProcess:
295
306
  pass
296
307
 
297
- self.childKilled = True
308
+ return psutil.wait_procs(alive, timeout=60)
298
309
 
299
310
  def pythonCall(self, function, *stArgs, **stKeyArgs):
300
311
  """call python function :function: with :stArgs: and :stKeyArgs:"""
@@ -309,8 +320,6 @@ class Subprocess:
309
320
  if pid == 0:
310
321
  os.close(readFD)
311
322
  self.__executePythonFunction(function, writeFD, *stArgs, **stKeyArgs)
312
- # FIXME: the close it is done at __executePythonFunction, do we need it here?
313
- os.close(writeFD)
314
323
  else:
315
324
  os.close(writeFD)
316
325
  readSeq = self.__selectFD([readFD])
@@ -319,14 +328,13 @@ class Subprocess:
319
328
  try:
320
329
  if len(readSeq) == 0:
321
330
  self.log.debug("Timeout limit reached for pythonCall", function.__name__)
322
- self.__killPid(pid)
323
-
324
- # HACK to avoid python bug
325
- # self.wait()
326
- retries = 10000
327
- while os.waitpid(pid, 0) == -1 and retries > 0:
328
- time.sleep(0.001)
329
- retries -= 1
331
+ gone, alive = self.__terminateProcess(psutil.Process(pid))
332
+ if alive:
333
+ for p in alive:
334
+ try:
335
+ p.kill()
336
+ except psutil.NoSuchProcess:
337
+ continue
330
338
 
331
339
  return S_ERROR('%d seconds timeout for "%s" call' % (self.timeout, function.__name__))
332
340
  elif readSeq[0] == readFD:
@@ -400,7 +408,7 @@ class Subprocess:
400
408
  if len(dataString) + baseLength > self.bufferLimit:
401
409
  self.log.error("Maximum output buffer length reached")
402
410
  retDict = S_ERROR(
403
- "Reached maximum allowed length (%d bytes) for called " "function return value" % self.bufferLimit
411
+ "Reached maximum allowed length (%d bytes) for called function return value" % self.bufferLimit
404
412
  )
405
413
  retDict["Value"] = dataString
406
414
  return retDict
@@ -446,6 +454,7 @@ class Subprocess:
446
454
  start_new_session=start_new_session,
447
455
  )
448
456
  self.childPID = self.child.pid
457
+ child_process = psutil.Process(self.childPID)
449
458
  except OSError as v:
450
459
  retDict = S_ERROR(repr(v))
451
460
  retDict["Value"] = (-1, "", str(v))
@@ -464,9 +473,9 @@ class Subprocess:
464
473
  self.bufferList = [["", 0], ["", 0]]
465
474
  initialTime = time.time()
466
475
 
467
- exitStatus = self.__poll(self.child.pid)
476
+ exitStatus = self.__poll(child_process)
468
477
 
469
- while (0, 0) == exitStatus or exitStatus is None:
478
+ while (0, 0) == exitStatus: # This means that the process is still alive
470
479
  retDict = self.__readFromCommand()
471
480
  if not retDict["OK"]:
472
481
  return retDict
@@ -478,14 +487,14 @@ class Subprocess:
478
487
  1, "Timeout (%d seconds) for '%s' call" % (self.timeout, cmdSeq)
479
488
  )
480
489
  time.sleep(0.01)
481
- exitStatus = self.__poll(self.child.pid)
490
+ exitStatus = self.__poll(child_process)
482
491
 
483
492
  self.__readFromCommand()
484
493
 
485
494
  if exitStatus:
486
495
  exitStatus = exitStatus[1]
487
496
 
488
- if exitStatus >= 256:
497
+ if exitStatus and exitStatus >= 256:
489
498
  exitStatus = int(exitStatus / 256)
490
499
  return S_OK((exitStatus, self.bufferList[0][0], self.bufferList[1][0]))
491
500
  finally:
@@ -3,23 +3,25 @@
3
3
  # Date: 2012/12/11 18:04:25
4
4
  ########################################################################
5
5
 
6
- """ :mod: SubprocessTests
7
- =======================
6
+ """:mod: SubprocessTests
7
+ =======================
8
8
 
9
- .. module: SubprocessTests
10
- :synopsis: unittest for Subprocess module
11
- .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
9
+ .. module: SubprocessTests
10
+ :synopsis: unittest for Subprocess module
11
+ .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
12
12
 
13
- unittest for Subprocess module
13
+ unittest for Subprocess module
14
14
  """
15
- import time
15
+
16
16
  import platform
17
+ import time
17
18
  from os.path import dirname, join
18
19
  from subprocess import Popen
19
20
 
21
+ import psutil
20
22
  import pytest
21
23
 
22
- from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall, pythonCall, getChildrenPIDs, Subprocess
24
+ from DIRAC.Core.Utilities.Subprocess import Subprocess, getChildrenPIDs, pythonCall, shellCall, systemCall
23
25
 
24
26
  # Mark this entire module as slow
25
27
  pytestmark = pytest.mark.slow
@@ -72,3 +74,51 @@ def test_decodingCommandOutput():
72
74
  retVal = sp.systemCall(r"""python -c 'import os; os.fdopen(2, "wb").write(b"\xdf")'""", shell=True)
73
75
  assert retVal["OK"]
74
76
  assert retVal["Value"] == (0, "", "\ufffd")
77
+
78
+
79
+ @pytest.fixture
80
+ def subprocess_instance():
81
+ """Provides a Subprocess instance for testing."""
82
+ subp = Subprocess()
83
+ return subp
84
+
85
+
86
+ @pytest.fixture
87
+ def dummy_child():
88
+ """Spawn a dummy process tree: parent -> child."""
89
+ # Start a shell that sleeps, with a subprocess child
90
+ parent = Popen(["bash", "-c", "sleep 10 & wait"])
91
+ time.sleep(0.2) # give it a moment to start
92
+ yield parent
93
+ # Ensure cleanup
94
+ try:
95
+ parent.terminate()
96
+ parent.wait(timeout=1)
97
+ except Exception:
98
+ pass
99
+
100
+
101
+ def test_kill_child_process_tree(subprocess_instance, dummy_child):
102
+ """Test that killChild kills both parent and its children."""
103
+ subprocess_instance.childPID = dummy_child.pid
104
+ parent_proc = psutil.Process(subprocess_instance.childPID)
105
+
106
+ # Sanity check: parent should exist
107
+ assert parent_proc.is_running()
108
+
109
+ # It should have at least one sleeping child
110
+ children = parent_proc.children(recursive=True)
111
+ assert children, "Expected dummy process to have at least one child"
112
+
113
+ # Kill the tree
114
+ gone, alive = subprocess_instance.killChild(recursive=True)
115
+
116
+ # Verify the parent and children are terminated
117
+ for p in gone:
118
+ assert not p.is_running(), f"Process {p.pid} still alive"
119
+ for p in alive:
120
+ assert not p.is_running(), f"Process {p.pid} still alive"
121
+
122
+ # Verify parent is gone
123
+ with pytest.raises(psutil.NoSuchProcess):
124
+ psutil.Process(subprocess_instance.childPID)
@@ -3,9 +3,13 @@
3
3
  import datetime
4
4
  import errno
5
5
  import os
6
+ import requests
6
7
  from packaging.version import Version
7
8
 
8
- from cachetools import cachedmethod, LRUCache
9
+ from cachetools import cachedmethod, LRUCache, TTLCache, cached
10
+ from threading import Lock
11
+ from typing import Optional
12
+
9
13
 
10
14
  # Requires at least version 3.3.3
11
15
  from fts3 import __version__ as fts3_version
@@ -44,6 +48,50 @@ BRING_ONLINE_TIMEOUT = 259200
44
48
  IDP_CACHE_SIZE = 8
45
49
 
46
50
 
51
+ _scitag_cache = TTLCache(maxsize=10, ttl=3600)
52
+ _scitag_lock = Lock()
53
+ _scitag_json_cache = TTLCache(maxsize=1, ttl=86400)
54
+ _scitag_json_lock = Lock()
55
+
56
+
57
+ @cached(_scitag_cache, lock=_scitag_lock)
58
+ def get_scitag(vo: str, activity: Optional[str] = None) -> int:
59
+ """
60
+ Get the scitag based on the VO and activity.
61
+ If the VO is not found in the scitag.json, it defaults to 1.
62
+ If no specific activity is provided, it defaults to the "default" activityName.
63
+
64
+ :param vo: The VO for which to get the scitag
65
+ :param activity: The activity for which to get the scitag
66
+ :return: The scitag value
67
+ """
68
+
69
+ @cached(_scitag_json_cache, lock=_scitag_json_lock)
70
+ def get_remote_json():
71
+ gLogger.verbose("Fetching https://scitags.org/api.json from the network")
72
+ response = requests.get("https://scitags.org/api.json")
73
+ response.raise_for_status()
74
+ return response.json()
75
+
76
+ vo_id = 1 # Default VO ID
77
+ activity_id = 1 # Default activity ID
78
+
79
+ try:
80
+ # Load the JSON data from the cache or network
81
+ sj = get_remote_json()
82
+
83
+ for experiment in sj.get("experiments", []):
84
+ if experiment.get("expName") == vo.lower():
85
+ vo_id = experiment.get("expId")
86
+ for act in experiment.get("activities", []):
87
+ if act.get("activityName") == activity:
88
+ activity_id = act.get("activityId")
89
+ except Exception as e:
90
+ gLogger.error(f"Error fetching or parsing scitag.json. Using default scitag values.", repr(e))
91
+ # Logic to determine the scitag based on vo and activity (this is what FTS wants)
92
+ return vo_id << 6 | activity_id # Example logic, replace with actual implementation
93
+
94
+
47
95
  class FTS3Job(JSerializable):
48
96
  """Abstract class to represent a job to be executed by FTS. It belongs
49
97
  to an FTS3Operation
@@ -470,6 +518,9 @@ class FTS3Job(JSerializable):
470
518
 
471
519
  ftsFileID = getattr(ftsFile, "fileID")
472
520
 
521
+ # scitag 65 is 1 << 6 | 1 (default experiment, default activity)
522
+ scitag = get_scitag(vo=self.vo, activity=self.activity)
523
+
473
524
  # Under normal circumstances, we simply submit an fts transfer as such:
474
525
  # * srcProto://myFile -> destProto://myFile
475
526
  #
@@ -499,6 +550,7 @@ class FTS3Job(JSerializable):
499
550
  filesize=ftsFile.size,
500
551
  metadata=stageTrans_metadata,
501
552
  activity=self.activity,
553
+ scitag=scitag,
502
554
  )
503
555
  transfers.append(stageTrans)
504
556
 
@@ -572,6 +624,7 @@ class FTS3Job(JSerializable):
572
624
  activity=self.activity,
573
625
  source_token=srcToken,
574
626
  destination_token=dstToken,
627
+ scitag=scitag,
575
628
  )
576
629
 
577
630
  transfers.append(trans)
@@ -204,6 +204,7 @@ def generateFTS3Job(sourceSE, targetSE, lfns, multiHopSE=None):
204
204
  newJob.sourceSE = sourceSE
205
205
  newJob.targetSE = targetSE
206
206
  newJob.multiHopSE = multiHopSE
207
+ newJob.vo = "lhcb"
207
208
  filesToSubmit = []
208
209
 
209
210
  for i, lfn in enumerate(lfns, start=1):
@@ -0,0 +1,69 @@
1
+ from unittest.mock import Mock, patch
2
+
3
+ import pytest
4
+
5
+ from DIRAC.DataManagementSystem.Client.FTS3Job import get_scitag
6
+
7
+
8
+ class TestGetScitag:
9
+ def test_valid_vo_and_activity(self):
10
+ """Test get_scitag with valid VO and activity."""
11
+ result = get_scitag("atlas", "Analysis Input")
12
+ expected = 2 << 6 | 17 # atlas expId=2, analysis activityId=17
13
+ assert result == expected
14
+
15
+ def test_valid_vo_no_activity(self):
16
+ """Test get_scitag with valid VO but no specific activity (should use default)."""
17
+ result = get_scitag("cms")
18
+ expected = 3 << 6 | 1 # cms expId=200, default activityId=1
19
+ assert result == expected
20
+
21
+ def test_invalid_vo(self):
22
+ """Test get_scitag with invalid VO (should use default vo_id=1)."""
23
+ result = get_scitag("nonexistent")
24
+ expected = 1 << 6 | 1 # default vo_id=1, default activity_id=1
25
+ assert result == expected
26
+
27
+ def test_valid_vo_invalid_activity(self):
28
+ """Test get_scitag with valid VO but invalid activity."""
29
+ result = get_scitag("atlas", "nonexistent_activity")
30
+ expected = 2 << 6 | 1 # atlas expId=2, default activity_id=1
31
+ assert result == expected
32
+
33
+ def test_case_insensitive_vo(self):
34
+ """Test that VO matching is case insensitive."""
35
+ result = get_scitag("ATLAS", "Data Brokering")
36
+ expected = 2 << 6 | 3 # atlas expId=2, production activityId=3
37
+ assert result == expected
38
+
39
+
40
+ @pytest.mark.parametrize(
41
+ "vo,activity,expected_vo_id,expected_activity_id",
42
+ [
43
+ ("atlas", "Analysis Output", 2, 18),
44
+ ("atlas", "Debug", 2, 9),
45
+ ("cms", "Cache", 3, 3),
46
+ ("cms", "default", 3, 1),
47
+ ("nonexistent", "any", 1, 1), # defaults
48
+ ("atlas", "nonexistent", 2, 1), # valid vo, invalid activity
49
+ ],
50
+ )
51
+ def test_parametrized_scenarios(vo, activity, expected_vo_id, expected_activity_id):
52
+ """Parametrized test for various VO and activity combinations."""
53
+ result = get_scitag(vo, activity)
54
+ expected = expected_vo_id << 6 | expected_activity_id
55
+ assert result == expected
56
+
57
+
58
+ @pytest.mark.parametrize(
59
+ "vo,expected_result",
60
+ [
61
+ ("atlas", 2 << 6 | 1), # Should use default activity
62
+ ("cms", 3 << 6 | 1), # Should use default activity
63
+ ("unknown", 1 << 6 | 1), # Should use all defaults
64
+ ],
65
+ )
66
+ def test_no_activity_parameter(vo, expected_result):
67
+ """Test behavior when no activity parameter is provided."""
68
+ result = get_scitag(vo)
69
+ assert result == expected_result
@@ -148,10 +148,10 @@ class SiteDirector(AgentModule):
148
148
  self.sendSubmissionAccounting = True
149
149
 
150
150
  # Get the site description dictionary
151
- siteNames = self.am_getOption("Site")
152
- ceTypes = self.am_getOption("CETypes")
153
- ces = self.am_getOption("CEs")
154
- tags = self.am_getOption("Tags")
151
+ siteNames = self.am_getOption("Site", []) or None
152
+ ceTypes = self.am_getOption("CETypes", []) or None
153
+ ces = self.am_getOption("CEs", []) or None
154
+ tags = self.am_getOption("Tags", []) or None
155
155
 
156
156
  # Display options used
157
157
  self.log.always("VO:", self.vo)
@@ -263,10 +263,14 @@ Agents
263
263
  # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
264
264
  PilotDN =
265
265
 
266
- # Site = # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
267
- # CEs = # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
268
- # CETypes = # List of Tags that are required to be present in the CE/Queue definition
269
- # Tags =
266
+ # List of Sites that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
267
+ Site =
268
+ # List of CEs that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
269
+ CEs =
270
+ # List of CETypes that are required to be present in the CE/Queue definition
271
+ CETypes =
272
+ # List of Tags that are required to be present in the CE/Queue definition
273
+ Tags =
270
274
 
271
275
  # How many cycles to skip if queue is not working
272
276
  FailedQueueCycleFactor = 10
@@ -1,22 +1,22 @@
1
- """ The Watchdog class is used by the Job Wrapper to resolve and monitor
2
- the system resource consumption. The Watchdog can determine if
3
- a running job is stalled and indicate this to the Job Wrapper.
4
- Furthermore, the Watchdog will identify when the Job CPU limit has been
5
- exceeded and fail jobs meaningfully.
6
-
7
- Information is returned to the WMS via the heart-beat mechanism. This
8
- also interprets control signals from the WMS e.g. to kill a running
9
- job.
10
-
11
- - Still to implement:
12
- - CPU normalization for correct comparison with job limit
1
+ """The Watchdog class is used by the Job Wrapper to resolve and monitor
2
+ the system resource consumption. The Watchdog can determine if
3
+ a running job is stalled and indicate this to the Job Wrapper.
4
+ Furthermore, the Watchdog will identify when the Job CPU limit has been
5
+ exceeded and fail jobs meaningfully.
6
+
7
+ Information is returned to the WMS via the heart-beat mechanism. This
8
+ also interprets control signals from the WMS e.g. to kill a running
9
+ job.
10
+
11
+ - Still to implement:
12
+ - CPU normalization for correct comparison with job limit
13
13
  """
14
+
14
15
  import datetime
15
16
  import errno
16
17
  import getpass
17
18
  import math
18
19
  import os
19
- import signal
20
20
  import socket
21
21
  import time
22
22
  from pathlib import Path
@@ -32,28 +32,6 @@ from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus
32
32
  from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
33
33
 
34
34
 
35
- def kill_proc_tree(pid, sig=signal.SIGTERM, includeParent=True):
36
- """Kill a process tree (including grandchildren) with signal
37
- "sig" and return a (gone, still_alive) tuple.
38
- called as soon as a child terminates.
39
-
40
- Taken from https://psutil.readthedocs.io/en/latest/index.html#kill-process-tree
41
- """
42
- assert pid != os.getpid(), "won't kill myself"
43
- parent = psutil.Process(pid)
44
- children = parent.children(recursive=True)
45
- if includeParent:
46
- children.append(parent)
47
- for p in children:
48
- try:
49
- p.send_signal(sig)
50
- except psutil.NoSuchProcess:
51
- pass
52
- _gone, alive = psutil.wait_procs(children, timeout=10)
53
- for p in alive:
54
- p.kill()
55
-
56
-
57
35
  class Watchdog:
58
36
  #############################################################################
59
37
  def __init__(self, pid, exeThread, spObject, jobCPUTime, memoryLimit=0, processors=1, jobArgs={}):
@@ -212,7 +190,7 @@ class Watchdog:
212
190
  if self.littleTimeLeftCount == 0 and self.__timeLeft() == -1:
213
191
  self.checkError = JobMinorStatus.JOB_EXCEEDED_CPU
214
192
  self.log.error(self.checkError, self.timeLeft)
215
- self.__killRunningThread()
193
+ self.spObject.killChild()
216
194
  return S_OK()
217
195
 
218
196
  self.littleTimeLeftCount -= 1
@@ -321,7 +299,7 @@ class Watchdog:
321
299
 
322
300
  self.log.info("=================END=================")
323
301
 
324
- self.__killRunningThread()
302
+ self.spObject.killChild()
325
303
  return S_OK()
326
304
 
327
305
  recentStdOut = "None"
@@ -408,7 +386,7 @@ class Watchdog:
408
386
  if "Kill" in signalDict:
409
387
  self.log.info("Received Kill signal, stopping job via control signal")
410
388
  self.checkError = JobMinorStatus.RECEIVED_KILL_SIGNAL
411
- self.__killRunningThread()
389
+ self.spObject.killChild()
412
390
  else:
413
391
  self.log.info("The following control signal was sent but not understood by the watchdog:")
414
392
  self.log.info(signalDict)
@@ -862,13 +840,6 @@ class Watchdog:
862
840
 
863
841
  return result
864
842
 
865
- #############################################################################
866
- def __killRunningThread(self):
867
- """Will kill the running thread process and any child processes."""
868
- self.log.info("Sending kill signal to application PID", self.spObject.getChildPID())
869
- self.spObject.killChild()
870
- return S_OK("Thread killed")
871
-
872
843
  #############################################################################
873
844
  def __sendSignOfLife(self, jobID, heartBeatDict, staticParamDict):
874
845
  """Sends sign of life 'heartbeat' signal and triggers control signal
@@ -1,5 +1,5 @@
1
- """ Test class for JobWrapper
2
- """
1
+ """Test class for JobWrapper"""
2
+
3
3
  import os
4
4
  import shutil
5
5
  import tempfile
@@ -314,8 +314,7 @@ def test_processKilledSubprocess(mocker):
314
314
  result = jw.process("sleep 20", {})
315
315
 
316
316
  assert result["OK"]
317
- assert result["Value"]["payloadStatus"] == 15 # SIGTERM
318
- assert not result["Value"]["payloadOutput"]
317
+ assert result["Value"]["payloadStatus"] is None
319
318
  assert not result["Value"]["payloadExecutorError"]
320
319
  assert result["Value"]["watchdogError"] == "Job is stalled!" # Error message from the watchdog
321
320
 
@@ -218,7 +218,7 @@ class PilotCStoJSONSynchronizer:
218
218
 
219
219
  preferredURLPatterns = gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns")
220
220
  if preferredURLPatterns:
221
- pilotDict["PreferredURLPatterns"] = preferredURLPatterns
221
+ pilotDict["PreferredURLPatterns"] = preferredURLPatterns.replace(" ", "").split(",")
222
222
 
223
223
  self.log.debug("Got pilotDict", str(pilotDict))
224
224
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: DIRAC
3
- Version: 9.0.7
3
+ Version: 9.0.9
4
4
  Summary: DIRAC is an interware, meaning a software framework for distributed computing.
5
5
  Home-page: https://github.com/DIRACGrid/DIRAC/
6
6
  License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
19
19
  Requires-Dist: certifi
20
20
  Requires-Dist: cwltool
21
21
  Requires-Dist: diraccfg
22
- Requires-Dist: DIRACCommon==v9.0.7
22
+ Requires-Dist: DIRACCommon==v9.0.9
23
23
  Requires-Dist: diracx-client>=v0.0.1
24
24
  Requires-Dist: diracx-core>=v0.0.1
25
25
  Requires-Dist: diracx-cli>=v0.0.1
@@ -259,7 +259,7 @@ DIRAC/Core/Utilities/ReturnValues.py,sha256=EHtnn1I3Lvq6RO9tTVZMlf19a8QGX1B6Qa7l
259
259
  DIRAC/Core/Utilities/Shifter.py,sha256=EkJGPR_zbL0SS3E2_cq1-MjkTH91ZaFJlh-tQ2PxDGk,2571
260
260
  DIRAC/Core/Utilities/SiteSEMapping.py,sha256=Uynx_evXQPmfbMKH-TfqzLq4e_7APVrP6gC20cJEBYs,5683
261
261
  DIRAC/Core/Utilities/StateMachine.py,sha256=e1rc0X6J8Cf_W9HlLnLgroLkdrksUQUxdkH_qSoZVoA,769
262
- DIRAC/Core/Utilities/Subprocess.py,sha256=FcdEKKEySh_62S8VLjWp_HnuFYkKAjqsS8FLpPMS1b8,22756
262
+ DIRAC/Core/Utilities/Subprocess.py,sha256=_lNp7ncg6BYC2TLvkcqACelZSsb0_5oyM2Jq-T5YbFc,23315
263
263
  DIRAC/Core/Utilities/ThreadPool.py,sha256=tETtgUdW1dlq_7xp_nQUTxNC87jAkrNnwvuDpGS3ffE,11484
264
264
  DIRAC/Core/Utilities/ThreadSafe.py,sha256=EAxEMqEuYlzDvAaupW3fi2MY-3kGZJ0zbK9i7JhEmt4,1117
265
265
  DIRAC/Core/Utilities/ThreadScheduler.py,sha256=IzyS59NF0ZhQwGYLpZ-yMX8_6kXFSGL9FQapylnoPLE,6212
@@ -304,7 +304,7 @@ DIRAC/Core/Utilities/test/Test_Pfn.py,sha256=XWTXejQf_TnicaPbQ4ZyDsl1KXDpq3IMNFl
304
304
  DIRAC/Core/Utilities/test/Test_ProcessPool.py,sha256=NnfHNkhTOgADk-P9ds_ADuvcSNlK_XdoibRkk13r_NE,10890
305
305
  DIRAC/Core/Utilities/test/Test_Profiler.py,sha256=8QRRXm-PwJpjLGwAOJxnrpnnMbmVzXhJ1ZcQ9gDBX5c,4215
306
306
  DIRAC/Core/Utilities/test/Test_ReturnValues.py,sha256=w6Jz-Vblgu8RDXPzVi6BZjuFXcSmW-Zb0mcBgW1RWOw,1926
307
- DIRAC/Core/Utilities/test/Test_Subprocess.py,sha256=nAiVF5oMnle-4E8ijlWF-ilBfgN2VGkkFQQSKhIKN2o,2141
307
+ DIRAC/Core/Utilities/test/Test_Subprocess.py,sha256=f9THTKnVdGcNwpZ0PtfCQz0UAtNmF06126-F0mUAxWQ,3605
308
308
  DIRAC/Core/Utilities/test/Test_entrypoints.py,sha256=z_3f7m59v3W6ZsqgeCFbJoBnMY-cKR_blKbPqcV7528,413
309
309
  DIRAC/Core/Utilities/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
310
310
  DIRAC/Core/Workflow/Module.py,sha256=XF_iDyJ1wIwUesHLfDGz9BywHgEuOwqzC2f7fY3E7r4,12631
@@ -365,7 +365,7 @@ DIRAC/DataManagementSystem/Client/DataManager.py,sha256=edLmRUHJtZEYAMyen3y9qwdG
365
365
  DIRAC/DataManagementSystem/Client/DirectoryListing.py,sha256=CvQZ5Da5WhgwdDL8iLk0xVBjKZrBHTdwMKWwSJwzwEI,7582
366
366
  DIRAC/DataManagementSystem/Client/FTS3Client.py,sha256=Kk1wy4YnkYuOI6IltyDYZqkzxV7Zsf7ZuoKT12_VbTw,3531
367
367
  DIRAC/DataManagementSystem/Client/FTS3File.py,sha256=gZFN_Uxc2tblUCETb4qWPJ2M1tSmKUvTMiZUL-yJ6M0,3122
368
- DIRAC/DataManagementSystem/Client/FTS3Job.py,sha256=ZrQEMRFEfCTTkbcaio6DLV1dijp4e8gJgO8kvphW9p4,40045
368
+ DIRAC/DataManagementSystem/Client/FTS3Job.py,sha256=5MAtSZoWb_rwglNHdYJAuoXzoW5_TAX96E5jUWEYds8,42090
369
369
  DIRAC/DataManagementSystem/Client/FTS3Operation.py,sha256=L3yqc0X0ziluMPegoQU6o3aYDnTYwLtujShgbGFJjsw,24455
370
370
  DIRAC/DataManagementSystem/Client/FailoverTransfer.py,sha256=2V-dKtgAndMuw9iZHYV019V0hLwlezUNP33wvClpsaA,13373
371
371
  DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py,sha256=sc_6mT0BCbXgj5wOEFtCxNc0AvyQ186A_yLi2lnduvw,80094
@@ -376,7 +376,8 @@ DIRAC/DataManagementSystem/Client/CmdDirCompletion/AbstractFileSystem.py,sha256=
376
376
  DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py,sha256=TdOCEfVQ3QppTl0iHrf472rT3YkDw2EX2sUviKIYL2s,2394
377
377
  DIRAC/DataManagementSystem/Client/CmdDirCompletion/__init__.py,sha256=IU_DNOfkPtg8iBqQatUKTNBMj2LldGrhf6KseBgx14M,236
378
378
  DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py,sha256=qb0mERTlHOysiSfNPDsbLbadWgbuQLulT9Py8RqTIYc,3956
379
- DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py,sha256=PwdHIHEJpP4I7plHRhrIBHrftIW0aRHCasYx7AUQfd4,15432
379
+ DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py,sha256=DZHRZPfiTJrqinXolneRE06Wf51sVtRw0Fx0bq85AAA,15455
380
+ DIRAC/DataManagementSystem/Client/test/Test_scitag.py,sha256=llvY98ODaBZhyhDquP9IMzCB-uXJz8xL-TO2-sbNf68,2545
380
381
  DIRAC/DataManagementSystem/Client/test/__init__.py,sha256=IL2xDyzb5nr2dkH2hpT4Zx6vvc9m4jDryuUmfyIY_RY,58
381
382
  DIRAC/DataManagementSystem/Client/test/mock_DM.py,sha256=S7qWHjN3ZyjqKIjHkXwuUWJqp75waEchc1IsNB9y-28,1167
382
383
  DIRAC/DataManagementSystem/Client/test/new_dir_completion.py,sha256=_Km7jE4IRe-pY8BJ62hK6IbwRROoj0wxvL7WMClEJ0E,1212
@@ -1118,7 +1119,7 @@ DIRAC/Workflow/Modules/test/Test_Modules.py,sha256=d92bA2SpF7Wwg9gy3roEvPt_7QlnB
1118
1119
  DIRAC/Workflow/Utilities/Utils.py,sha256=9f395ylWaLRaaC6EExPAyvOhPK3FXD7I_XGevqueRkM,2545
1119
1120
  DIRAC/Workflow/Utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
1120
1121
  DIRAC/Workflow/Utilities/test/Test_Utilities.py,sha256=DNjFPpBmpojwCWhZSUSoG3AIhMQvqyiZdSuVkrHkOtk,2595
1121
- DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg,sha256=lCelD92HtTXp46CCwCb8S58BjITG1fBgtFs116vehww,8949
1122
+ DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg,sha256=uHa3ZiSC7cyhbJUNvWwqiL0E-m9UmBinrLIQdWmre3k,9030
1122
1123
  DIRAC/WorkloadManagementSystem/__init__.py,sha256=9-_-HOT_8S3i-TMmTR_gFVVlNyktBRk-S2qSuOBKoIc,50
1123
1124
  DIRAC/WorkloadManagementSystem/Agent/JobAgent.py,sha256=LzLypd3m4a6M-J5_nmUMsMqgflb_1Xm77eONDE6G1Vg,40843
1124
1125
  DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py,sha256=TR8nDO43DuJxOQVIdOUrqpWjRjJp52dVzOKUUewL9U4,14708
@@ -1126,7 +1127,7 @@ DIRAC/WorkloadManagementSystem/Agent/PilotLoggingAgent.py,sha256=ZIgvFpasGTh76GW
1126
1127
  DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py,sha256=qY6TbYCPOFFXhHffmRJLNEbWvZPyg5Lc5B_8BbyQ7zc,9711
1127
1128
  DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py,sha256=qzDFCGZ8EtjxDUaPgyFDlSmJfyF2KLuyXTC7au3-p2Q,4860
1128
1129
  DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py,sha256=IvHshnw2xN0AZrruEu86C47GDez8enBD6jjNIZd6QcA,38594
1129
- DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py,sha256=7b0CVkSgJ9b_lX-6CT6exzaWgEHH6pLjj5T1RJTnYJU,45114
1130
+ DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py,sha256=ZSGWVKO64ztnv_R19I6TT7660jf7LMuYcff_aPq2xCM,45162
1130
1131
  DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py,sha256=foEbmRotEmfeQG6nyIsJv1kSJkm4flkQsPYbSylS3SM,24572
1131
1132
  DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py,sha256=iNIlWQEDBk6R1S8oHOIusZUwxOwLtgwuzR_4s32-o5w,8707
1132
1133
  DIRAC/WorkloadManagementSystem/Agent/TaskQueuesAgent.py,sha256=ypsmo233TFXq9IC5uz6pum7_joOh2gFPUYNmmCpukAY,943
@@ -1208,9 +1209,9 @@ DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py,sha256=s_YHg_PsTGVHBLrS1
1208
1209
  DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperOfflineTemplate.py,sha256=wem5VDN9XiC7szAzdsbgHUxpIOQB2Hj36DIVMoV9px8,2490
1209
1210
  DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py,sha256=4QgcFPMLRaTagP9e_Vvsla8pFH8HdewklHfS-gyS4-g,3313
1210
1211
  DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperUtilities.py,sha256=5w_4PMnaHhuexChADDvt1L9Ih1PstdUuYWObnlv9Dto,10072
1211
- DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py,sha256=wGpIdnyVzI5T9agxNp94gmPZPceXWREaJiEtZg1lAzk,39997
1212
+ DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py,sha256=UWVDC_tWzlE9i5PsLycrRda4j3hWjaK1RnQVOvydg6U,38857
1212
1213
  DIRAC/WorkloadManagementSystem/JobWrapper/__init__.py,sha256=e9Oa_ddNLweR3Lp_HOMK6WqqCWWj2SLPxF5UH4F19ic,61
1213
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py,sha256=R9onrrnfyc1v4USt2nnvrYHFo5tKbpLjhMkXl4n0y2Y,39177
1214
+ DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py,sha256=AhevZTkZfFQV9XN5UTWFmoMe9GvwN_Caorto1o_YoU8,39119
1214
1215
  DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py,sha256=dC_SvC5Rlchlj2NvBfN7FH1ioXgC8bf9U8BQnEL5GYg,21982
1215
1216
  DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_Watchdog.py,sha256=a-QJ1E1ZcWObhOVgxZYD_nYjseCWsbjT0KxjZDNWyAQ,882
1216
1217
  DIRAC/WorkloadManagementSystem/JobWrapper/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1241,7 +1242,7 @@ DIRAC/WorkloadManagementSystem/Utilities/JobModel.py,sha256=jN9sFbzMZo9tab6Kp7Oe
1241
1242
  DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py,sha256=JW3AAEtBJn1gIO_rm2Ft5qqjfLteIo3HpQtGNZBfhxE,8365
1242
1243
  DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py,sha256=WtGJzC7fHvydANh8JH6e1Kk_jebrCMPr2c5cw3ufjm8,7826
1243
1244
  DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py,sha256=FNUsGhvsFVrtmA7r8G-sd4QTMeBkqG1sdtwiBUKQyd0,605
1244
- DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py,sha256=Ezpfd90dV_j6fOn25v5gr-wNWh1nMpCOQZfPmGDHJD4,12448
1245
+ DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py,sha256=ZQk2tD40986awO9pae1zmdPEWlnMJt4m61Z_RU3LWl8,12476
1245
1246
  DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py,sha256=VcvQTpeyTbVYqSsPQDyAt37N2CaEAnIuvbR6yk4kYk8,15465
1246
1247
  DIRAC/WorkloadManagementSystem/Utilities/QueueUtilities.py,sha256=J5-n_lvWbW_TRjrlqp8hx1SHEaXDW2Dxp3R1hBBrWnE,12082
1247
1248
  DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py,sha256=7FcEtlYSJMzdbLIFBUKD-j_wqRHya-ISqk8w-JRy3kw,12159
@@ -1295,9 +1296,9 @@ DIRAC/tests/Workflow/Integration/exe-script.py,sha256=B_slYdTocEzqfQLRhwuPiLyYUn
1295
1296
  DIRAC/tests/Workflow/Integration/helloWorld.py,sha256=tBgEHH3ZF7ZiTS57gtmm3DW-Qxgm_57HWHpM-Y8XSws,205
1296
1297
  DIRAC/tests/Workflow/Regression/helloWorld.py,sha256=69eCgFuVSYo-mK3Dj2dw1c6g86sF5FksKCf8V2aGVoM,509
1297
1298
  DIRAC/tests/Workflow/Regression/helloWorld.xml,sha256=xwydIcFTAHIX-YPfQfyxuQ7hzvIO3IhR3UAF7ORgkGg,5310
1298
- dirac-9.0.7.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
1299
- dirac-9.0.7.dist-info/METADATA,sha256=0Y2bxOpUIA4SmXbeMd2Lp1_MLU8_2m83KFlge8LwsLs,10016
1300
- dirac-9.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1301
- dirac-9.0.7.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
1302
- dirac-9.0.7.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
1303
- dirac-9.0.7.dist-info/RECORD,,
1299
+ dirac-9.0.9.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
1300
+ dirac-9.0.9.dist-info/METADATA,sha256=UjEl56_6P3co7mVc4nFt9FsB51VuvNFA-MkK1kHsQGk,10016
1301
+ dirac-9.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1302
+ dirac-9.0.9.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
1303
+ dirac-9.0.9.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
1304
+ dirac-9.0.9.dist-info/RECORD,,
File without changes