DIRAC 9.0.0a64__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +11 -8
  2. DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +1 -1
  3. DIRAC/Core/Security/IAMService.py +4 -3
  4. DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
  5. DIRAC/Core/Utilities/DErrno.py +5 -309
  6. DIRAC/Core/Utilities/JDL.py +1 -195
  7. DIRAC/Core/Utilities/List.py +1 -127
  8. DIRAC/Core/Utilities/ReturnValues.py +7 -252
  9. DIRAC/Core/Utilities/StateMachine.py +12 -178
  10. DIRAC/Core/Utilities/TimeUtilities.py +10 -253
  11. DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
  12. DIRAC/Core/scripts/dirac_agent.py +1 -1
  13. DIRAC/DataManagementSystem/DB/FTS3DB.py +3 -0
  14. DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
  15. DIRAC/Resources/Catalog/RucioFileCatalogClient.py +1 -1
  16. DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
  17. DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +1 -1
  18. DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
  19. DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +1 -1
  20. DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +1 -1
  21. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +2 -0
  22. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
  23. DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
  24. DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -93
  25. DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
  26. DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
  27. DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
  28. DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +4 -2
  29. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +21 -5
  30. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
  31. DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +1 -1
  32. DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -199
  33. DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +1 -63
  34. DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
  35. DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +0 -123
  36. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -5
  37. DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
  38. DIRAC/__init__.py +55 -54
  39. {dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/METADATA +2 -1
  40. {dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/RECORD +44 -45
  41. DIRAC/Core/Utilities/test/Test_List.py +0 -150
  42. DIRAC/Core/Utilities/test/Test_Time.py +0 -88
  43. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobAdministration.py +0 -28
  44. {dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/WHEEL +0 -0
  45. {dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/entry_points.txt +0 -0
  46. {dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/licenses/LICENSE +0 -0
  47. {dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/top_level.txt +0 -0
@@ -1,255 +1,10 @@
1
- """
2
- DIRAC return dictionary
1
+ """Backward compatibility wrapper - moved to DIRACCommon
3
2
 
4
- Message values are converted to string
3
+ This module has been moved to DIRACCommon.Core.Utilities.ReturnValues to avoid
4
+ circular dependencies and allow DiracX to use these utilities without
5
+ triggering DIRAC's global state initialization.
5
6
 
6
- keys are converted to string
7
+ All exports are maintained for backward compatibility.
7
8
  """
8
- from __future__ import annotations
9
-
10
- import functools
11
- import sys
12
- import traceback
13
- from types import TracebackType
14
- from typing import Any, Callable, cast, Generic, Literal, overload, Type, TypeVar, Union
15
- from typing_extensions import TypedDict, ParamSpec, NotRequired
16
-
17
- from DIRAC.Core.Utilities.DErrno import strerror
18
-
19
-
20
- T = TypeVar("T")
21
- P = ParamSpec("P")
22
-
23
-
24
- class DOKReturnType(TypedDict, Generic[T]):
25
- """used for typing the DIRAC return structure"""
26
-
27
- OK: Literal[True]
28
- Value: T
29
-
30
-
31
- class DErrorReturnType(TypedDict):
32
- """used for typing the DIRAC return structure"""
33
-
34
- OK: Literal[False]
35
- Message: str
36
- Errno: int
37
- ExecInfo: NotRequired[tuple[type[BaseException], BaseException, TracebackType]]
38
- CallStack: NotRequired[list[str]]
39
-
40
-
41
- DReturnType = Union[DOKReturnType[T], DErrorReturnType]
42
-
43
-
44
- def S_ERROR(*args: Any, **kwargs: Any) -> DErrorReturnType:
45
- """return value on error condition
46
-
47
- Arguments are either Errno and ErrorMessage or just ErrorMessage fro backward compatibility
48
-
49
- :param int errno: Error number
50
- :param string message: Error message
51
- :param list callStack: Manually override the CallStack attribute better performance
52
- """
53
- callStack = kwargs.pop("callStack", None)
54
-
55
- result: DErrorReturnType = {"OK": False, "Errno": 0, "Message": ""}
56
-
57
- message = ""
58
- if args:
59
- if isinstance(args[0], int):
60
- result["Errno"] = args[0]
61
- if len(args) > 1:
62
- message = args[1]
63
- else:
64
- message = args[0]
65
-
66
- if result["Errno"]:
67
- message = f"{strerror(result['Errno'])} ( {result['Errno']} : {message})"
68
- result["Message"] = message
69
-
70
- if callStack is None:
71
- try:
72
- callStack = traceback.format_stack()
73
- callStack.pop()
74
- except Exception:
75
- callStack = []
76
-
77
- result["CallStack"] = callStack
78
-
79
- return result
80
-
81
-
82
- # mypy doesn't understand default parameter values with generics so use overloads (python/mypy#3737)
83
- @overload
84
- def S_OK() -> DOKReturnType[None]:
85
- ...
86
-
87
-
88
- @overload
89
- def S_OK(value: T) -> DOKReturnType[T]:
90
- ...
91
-
92
-
93
- def S_OK(value=None): # type: ignore
94
- """return value on success
95
-
96
- :param value: value of the 'Value'
97
- :return: dictionary { 'OK' : True, 'Value' : value }
98
- """
99
- return {"OK": True, "Value": value}
100
-
101
-
102
- def isReturnStructure(unk: Any) -> bool:
103
- """Check if value is an `S_OK`/`S_ERROR` object"""
104
- if not isinstance(unk, dict):
105
- return False
106
- if "OK" not in unk:
107
- return False
108
- if unk["OK"]:
109
- return "Value" in unk
110
- else:
111
- return "Message" in unk
112
-
113
-
114
- def isSError(value: Any) -> bool:
115
- """Check if value is an `S_ERROR` object"""
116
- if not isinstance(value, dict):
117
- return False
118
- if "OK" not in value:
119
- return False
120
- return "Message" in value
121
-
122
-
123
- def reprReturnErrorStructure(struct: DErrorReturnType, full: bool = False) -> str:
124
- errorNumber = struct.get("Errno", 0)
125
- message = struct.get("Message", "")
126
- if errorNumber:
127
- reprStr = f"{strerror(errorNumber)} ( {errorNumber} : {message})"
128
- else:
129
- reprStr = message
130
-
131
- if full:
132
- callStack = struct.get("CallStack")
133
- if callStack:
134
- reprStr += "\n" + "".join(callStack)
135
-
136
- return reprStr
137
-
138
-
139
- def returnSingleResult(dictRes: DReturnType[Any]) -> DReturnType[Any]:
140
- """Transform the S_OK{Successful/Failed} dictionary convention into
141
- an S_OK/S_ERROR return. To be used when a single returned entity
142
- is expected from a generally bulk call.
143
-
144
- :param dictRes: S_ERROR or S_OK( "Failed" : {}, "Successful" : {})
145
- :returns: S_ERROR or S_OK(value)
146
-
147
- The following rules are applied:
148
-
149
- - if dictRes is an S_ERROR: returns it as is
150
- - we start by looking at the Failed directory
151
- - if there are several items in a dictionary, we return the first one
152
- - if both dictionaries are empty, we return S_ERROR
153
- - For an item in Failed, we return S_ERROR
154
- - Far an item in Successful we return S_OK
155
-
156
- Behavior examples (would be perfect unit test :-) )::
157
-
158
- {'Message': 'Kaput', 'OK': False} -> {'Message': 'Kaput', 'OK': False}
159
- {'OK': True, 'Value': {'Successful': {}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False}
160
- {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {}}} -> {'OK': True, 'Value': 2}
161
- {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False}
162
- {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1, 'c': 3}}} -> {'Message': '1', 'OK': False}
163
- {'OK': True, 'Value': {'Successful': {'b': 2, 'd': 4}, 'Failed': {}}} -> {'OK': True, 'Value': 2}
164
- {'OK': True, 'Value': {'Successful': {}, 'Failed': {}}} ->
165
- {'Message': 'returnSingleResult: Failed and Successful dictionaries are empty', 'OK': False}
166
- """
167
- # if S_ERROR was returned, we return it as well
168
- if not dictRes["OK"]:
169
- return dictRes
170
- # if there is a Failed, we return the first one in an S_ERROR
171
- if "Failed" in dictRes["Value"] and len(dictRes["Value"]["Failed"]):
172
- errorMessage = list(dictRes["Value"]["Failed"].values())[0]
173
- if isinstance(errorMessage, dict):
174
- if isReturnStructure(errorMessage):
175
- return cast(DErrorReturnType, errorMessage)
176
- else:
177
- return S_ERROR(str(errorMessage))
178
- return S_ERROR(errorMessage)
179
- # if there is a Successful, we return the first one in an S_OK
180
- elif "Successful" in dictRes["Value"] and len(dictRes["Value"]["Successful"]):
181
- return S_OK(list(dictRes["Value"]["Successful"].values())[0])
182
- else:
183
- return S_ERROR("returnSingleResult: Failed and Successful dictionaries are empty")
184
-
185
-
186
- class SErrorException(Exception):
187
- """Exception class for use with `convertToReturnValue`"""
188
-
189
- def __init__(self, result: DErrorReturnType | str, errCode: int = 0):
190
- """Create a new exception return value
191
-
192
- If `result` is a `S_ERROR` return it directly else convert it to an
193
- appropriate value using `S_ERROR(errCode, result)`.
194
-
195
- :param result: The error to propagate
196
- :param errCode: the error code to propagate
197
- """
198
- if not isSError(result):
199
- result = S_ERROR(errCode, result)
200
- self.result = cast(DErrorReturnType, result)
201
-
202
-
203
- def returnValueOrRaise(result: DReturnType[T], *, errorCode: int = 0) -> T:
204
- """Unwrap an S_OK/S_ERROR response into a value or Exception
205
-
206
- This method assists with using exceptions in DIRAC code by raising
207
- :exc:`SErrorException` if `result` is an error. This can then by propagated
208
- automatically as an `S_ERROR` by wrapping public facing functions with
209
- `@convertToReturnValue`.
210
-
211
- :param result: Result of a DIRAC function which returns `S_OK`/`S_ERROR`
212
- :returns: The value associated with the `S_OK` object
213
- :raises: If `result["OK"]` is falsey the original exception is re-raised.
214
- If no exception is known an :exc:`SErrorException` is raised.
215
- """
216
- if not result["OK"]:
217
- if "ExecInfo" in result:
218
- raise result["ExecInfo"][0]
219
- else:
220
- raise SErrorException(result, errorCode)
221
- return result["Value"]
222
-
223
-
224
- def convertToReturnValue(func: Callable[P, T]) -> Callable[P, DReturnType[T]]:
225
- """Decorate a function to convert return values to `S_OK`/`S_ERROR`
226
-
227
- If `func` returns, wrap the return value in `S_OK`.
228
- If `func` raises :exc:`SErrorException`, return the associated `S_ERROR`
229
- If `func` raises any other exception type, convert it to an `S_ERROR` object
230
-
231
- :param result: The bare result of a function call
232
- :returns: `S_OK`/`S_ERROR`
233
- """
234
-
235
- @functools.wraps(func)
236
- def wrapped(*args: P.args, **kwargs: P.kwargs) -> DReturnType[T]:
237
- try:
238
- value = func(*args, **kwargs)
239
- except SErrorException as e:
240
- return e.result
241
- except Exception as e:
242
- retval = S_ERROR(f"{repr(e)}: {e}")
243
- # Replace CallStack with the one from the exception
244
- # Use cast as mypy doesn't understand that sys.exc_info can't return None in an exception block
245
- retval["ExecInfo"] = cast(tuple[type[BaseException], BaseException, TracebackType], sys.exc_info())
246
- exc_type, exc_value, exc_tb = retval["ExecInfo"]
247
- retval["CallStack"] = traceback.format_tb(exc_tb)
248
- return retval
249
- else:
250
- return S_OK(value)
251
-
252
- # functools will copy the annotations. Since we change the return type
253
- # we have to update it
254
- wrapped.__annotations__["return"] = DReturnType
255
- return wrapped
9
+ # Re-export everything from DIRACCommon for backward compatibility
10
+ from DIRACCommon.Core.Utilities.ReturnValues import * # noqa: F401, F403
@@ -1,185 +1,19 @@
1
- """ StateMachine
1
+ """Backward compatibility wrapper - moved to DIRACCommon
2
2
 
3
- This module contains the basic blocks to build a state machine (State and StateMachine)
4
- """
5
- from DIRAC import S_OK, S_ERROR, gLogger
6
-
7
-
8
- class State:
9
- """
10
- State class that represents a single step on a StateMachine, with all the
11
- possible transitions, the default transition and an ordering level.
12
-
13
-
14
- examples:
15
- >>> s0 = State(100)
16
- >>> s1 = State(0, ['StateName1', 'StateName2'], defState='StateName1')
17
- >>> s2 = State(0, ['StateName1', 'StateName2'])
18
- # this example is tricky. The transition rule says that will go to
19
- # nextState, e.g. 'StateNext'. But, it is not on the stateMap, and there
20
- # is no default defined, so it will end up going to StateNext anyway. You
21
- # must be careful while defining states and their stateMaps and defaults.
22
- """
23
-
24
- def __init__(self, level, stateMap=None, defState=None):
25
- """
26
- :param int level: each state is mapped to an integer, which is used to sort the states according to that integer.
27
- :param list stateMap: it is a list (of strings) with the reachable states from this particular status.
28
- If not defined, we assume there are no restrictions.
29
- :param str defState: default state used in case the next state is not in stateMap (not defined or simply not there).
30
- """
31
-
32
- self.level = level
33
- self.stateMap = stateMap if stateMap else []
34
- self.default = defState
35
-
36
- def transitionRule(self, nextState):
37
- """
38
- Method that selects next state, knowing the default and the transitions
39
- map, and the proposed next state. If <nextState> is in stateMap, goes there.
40
- If not, then goes to <self.default> if any. Otherwise, goes to <nextState>
41
- anyway.
42
-
43
- examples:
44
- >>> s0.transitionRule('nextState')
45
- 'nextState'
46
- >>> s1.transitionRule('StateName2')
47
- 'StateName2'
48
- >>> s1.transitionRule('StateNameNotInMap')
49
- 'StateName1'
50
- >>> s2.transitionRule('StateNameNotInMap')
51
- 'StateNameNotInMap'
52
-
53
- :param str nextState: name of the state in the stateMap
54
- :return: state name
55
- :rtype: str
56
- """
57
-
58
- # If next state is on the list of next states, go ahead.
59
- if nextState in self.stateMap:
60
- return nextState
61
-
62
- # If not, calculate defaultState:
63
- # if there is a default, that one
64
- # otherwise is nextState (states with empty list have no movement restrictions)
65
- defaultNext = self.default if self.default else nextState
66
- return defaultNext
67
-
68
-
69
- class StateMachine:
70
- """
71
- StateMachine class that represents the whole state machine with all transitions.
72
-
73
- examples:
74
- >>> sm0 = StateMachine()
75
- >>> sm1 = StateMachine(state = 'Active')
76
-
77
- :param state: current state of the StateMachine, could be None if we do not use the
78
- StateMachine to calculate transitions. Beware, it is not checked if the
79
- state is on the states map !
80
- :type state: None or str
81
-
82
- """
3
+ This module has been moved to DIRACCommon.Core.Utilities.StateMachine to avoid
4
+ circular dependencies and allow DiracX to use these utilities without
5
+ triggering DIRAC's global state initialization.
83
6
 
84
- def __init__(self, state=None):
85
- """
86
- Constructor.
87
- """
88
-
89
- self.state = state
90
- # To be overwritten by child classes, unless you like Nirvana state that much.
91
- self.states = {"Nirvana": State(100)}
92
-
93
- def getLevelOfState(self, state):
94
- """
95
- Given a state name, it returns its level (integer), which defines the hierarchy.
7
+ All exports are maintained for backward compatibility.
8
+ """
9
+ # Re-export everything from DIRACCommon for backward compatibility
10
+ from DIRACCommon.Core.Utilities.StateMachine import * # noqa: F401, F403
96
11
 
97
- >>> sm0.getLevelOfState('Nirvana')
98
- 100
99
- >>> sm0.getLevelOfState('AnotherState')
100
- -1
12
+ from DIRAC import gLogger
101
13
 
102
- :param str state: name of the state, it should be on <self.states> key set
103
- :return: `int` || -1 (if not in <self.states>)
104
- """
105
14
 
106
- if state not in self.states:
107
- return -1
108
- return self.states[state].level
15
+ class StateMachine(StateMachine): # noqa: F405 pylint: disable=function-redefined
16
+ """Backward compatibility wrapper - moved to DIRACCommon"""
109
17
 
110
18
  def setState(self, candidateState, noWarn=False):
111
- """Makes sure the state is either None or known to the machine, and that it is a valid state to move into.
112
- Final states are also checked.
113
-
114
- examples:
115
- >>> sm0.setState(None)['OK']
116
- True
117
- >>> sm0.setState('Nirvana')['OK']
118
- True
119
- >>> sm0.setState('AnotherState')['OK']
120
- False
121
-
122
- :param state: state which will be set as current state of the StateMachine
123
- :type state: None or str
124
- :return: S_OK || S_ERROR
125
- """
126
- if candidateState == self.state:
127
- return S_OK(candidateState)
128
-
129
- if not candidateState:
130
- self.state = candidateState
131
- elif candidateState in self.states:
132
- if not self.states[self.state].stateMap:
133
- if not noWarn:
134
- gLogger.warn("Final state, won't move", f"({self.state}, asked to move to {candidateState})")
135
- return S_OK(self.state)
136
- if candidateState not in self.states[self.state].stateMap:
137
- gLogger.warn(f"Can't move from {self.state} to {candidateState}, choosing a good one")
138
- result = self.getNextState(candidateState)
139
- if not result["OK"]:
140
- return result
141
- self.state = result["Value"]
142
- # If the StateMachine does not accept the candidate, return error message
143
- else:
144
- return S_ERROR(f"setState: {candidateState!r} is not a valid state")
145
-
146
- return S_OK(self.state)
147
-
148
- def getStates(self):
149
- """
150
- Returns all possible states in the state map
151
-
152
- examples:
153
- >>> sm0.getStates()
154
- [ 'Nirvana' ]
155
-
156
- :return: list(stateNames)
157
- """
158
-
159
- return list(self.states)
160
-
161
- def getNextState(self, candidateState):
162
- """
163
- Method that gets the next state, given the proposed transition to candidateState.
164
- If candidateState is not on the state map <self.states>, it is rejected. If it is
165
- not the case, we have two options: if <self.state> is None, then the next state
166
- will be <candidateState>. Otherwise, the current state is using its own
167
- transition rule to decide.
168
-
169
- examples:
170
- >>> sm0.getNextState(None)
171
- S_OK(None)
172
- >>> sm0.getNextState('NextState')
173
- S_OK('NextState')
174
-
175
- :param str candidateState: name of the next state
176
- :return: S_OK(nextState) || S_ERROR
177
- """
178
- if candidateState not in self.states:
179
- return S_ERROR(f"getNextState: {candidateState!r} is not a valid state")
180
-
181
- # FIXME: do we need this anymore ?
182
- if self.state is None:
183
- return S_OK(candidateState)
184
-
185
- return S_OK(self.states[self.state].transitionRule(candidateState))
19
+ return super().setState(candidateState, noWarn, logger_warn=gLogger.warn)