ob-metaflow 2.11.4.9__py2.py3-none-any.whl → 2.11.9.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow might be problematic. Click here for more details.
- metaflow/cli.py +15 -10
- metaflow/clone_util.py +71 -0
- metaflow/cmd/develop/stub_generator.py +2 -0
- metaflow/cmd/develop/stubs.py +17 -8
- metaflow/metaflow_config.py +3 -0
- metaflow/package.py +4 -3
- metaflow/parameters.py +2 -2
- metaflow/plugins/aws/batch/batch.py +12 -0
- metaflow/plugins/aws/batch/batch_cli.py +25 -0
- metaflow/plugins/aws/batch/batch_client.py +40 -0
- metaflow/plugins/aws/batch/batch_decorator.py +32 -1
- metaflow/plugins/aws/step_functions/step_functions.py +3 -0
- metaflow/plugins/datatools/s3/s3op.py +4 -3
- metaflow/plugins/env_escape/client.py +154 -27
- metaflow/plugins/env_escape/client_modules.py +15 -47
- metaflow/plugins/env_escape/configurations/emulate_test_lib/overrides.py +31 -42
- metaflow/plugins/env_escape/configurations/emulate_test_lib/server_mappings.py +8 -3
- metaflow/plugins/env_escape/configurations/test_lib_impl/test_lib.py +74 -22
- metaflow/plugins/env_escape/consts.py +1 -0
- metaflow/plugins/env_escape/exception_transferer.py +46 -112
- metaflow/plugins/env_escape/override_decorators.py +8 -8
- metaflow/plugins/env_escape/server.py +42 -5
- metaflow/plugins/env_escape/stub.py +168 -23
- metaflow/plugins/env_escape/utils.py +3 -3
- metaflow/plugins/gcp/gcp_secret_manager_secrets_provider.py +3 -2
- metaflow/plugins/pypi/conda_environment.py +9 -0
- metaflow/plugins/pypi/pip.py +17 -2
- metaflow/runtime.py +252 -61
- metaflow/sidecar/sidecar.py +11 -1
- metaflow/sidecar/sidecar_subprocess.py +34 -18
- metaflow/task.py +28 -54
- metaflow/version.py +1 -1
- {ob_metaflow-2.11.4.9.dist-info → ob_metaflow-2.11.9.1.dist-info}/METADATA +2 -2
- {ob_metaflow-2.11.4.9.dist-info → ob_metaflow-2.11.9.1.dist-info}/RECORD +38 -37
- {ob_metaflow-2.11.4.9.dist-info → ob_metaflow-2.11.9.1.dist-info}/WHEEL +1 -1
- {ob_metaflow-2.11.4.9.dist-info → ob_metaflow-2.11.9.1.dist-info}/LICENSE +0 -0
- {ob_metaflow-2.11.4.9.dist-info → ob_metaflow-2.11.9.1.dist-info}/entry_points.txt +0 -0
- {ob_metaflow-2.11.4.9.dist-info → ob_metaflow-2.11.9.1.dist-info}/top_level.txt +0 -0
|
@@ -4,11 +4,9 @@ import traceback
|
|
|
4
4
|
try:
|
|
5
5
|
# Import from client
|
|
6
6
|
from .data_transferer import DataTransferer
|
|
7
|
-
from .stub import Stub
|
|
8
7
|
except ImportError:
|
|
9
8
|
# Import from server
|
|
10
9
|
from data_transferer import DataTransferer
|
|
11
|
-
from stub import Stub
|
|
12
10
|
|
|
13
11
|
|
|
14
12
|
# This file is heavily inspired from the RPYC project
|
|
@@ -39,7 +37,6 @@ except ImportError:
|
|
|
39
37
|
FIELD_EXC_MODULE = "m"
|
|
40
38
|
FIELD_EXC_NAME = "n"
|
|
41
39
|
FIELD_EXC_ARGS = "arg"
|
|
42
|
-
FIELD_EXC_ATTR = "atr"
|
|
43
40
|
FIELD_EXC_TB = "tb"
|
|
44
41
|
FIELD_EXC_USER = "u"
|
|
45
42
|
FIELD_EXC_SI = "si"
|
|
@@ -54,7 +51,6 @@ def dump_exception(data_transferer, exception_type, exception_val, tb, user_data
|
|
|
54
51
|
traceback.format_exception(exception_type, exception_val, tb)
|
|
55
52
|
)
|
|
56
53
|
exception_args = []
|
|
57
|
-
exception_attrs = []
|
|
58
54
|
str_repr = None
|
|
59
55
|
repr_repr = None
|
|
60
56
|
for name in dir(exception_val):
|
|
@@ -72,20 +68,10 @@ def dump_exception(data_transferer, exception_type, exception_val, tb, user_data
|
|
|
72
68
|
repr_repr = repr(exception_val)
|
|
73
69
|
elif name.startswith("_") or name == "with_traceback":
|
|
74
70
|
continue
|
|
75
|
-
else:
|
|
76
|
-
try:
|
|
77
|
-
attr = getattr(exception_val, name)
|
|
78
|
-
except AttributeError:
|
|
79
|
-
continue
|
|
80
|
-
if DataTransferer.can_simple_dump(attr):
|
|
81
|
-
exception_attrs.append((name, attr))
|
|
82
|
-
else:
|
|
83
|
-
exception_attrs.append((name, repr(attr)))
|
|
84
71
|
to_return = {
|
|
85
72
|
FIELD_EXC_MODULE: exception_type.__module__,
|
|
86
73
|
FIELD_EXC_NAME: exception_type.__name__,
|
|
87
74
|
FIELD_EXC_ARGS: exception_args,
|
|
88
|
-
FIELD_EXC_ATTR: exception_attrs,
|
|
89
75
|
FIELD_EXC_TB: local_formatted_exception,
|
|
90
76
|
FIELD_EXC_STR: str_repr,
|
|
91
77
|
FIELD_EXC_REPR: repr_repr,
|
|
@@ -98,121 +84,69 @@ def dump_exception(data_transferer, exception_type, exception_val, tb, user_data
|
|
|
98
84
|
return data_transferer.dump(to_return)
|
|
99
85
|
|
|
100
86
|
|
|
101
|
-
def load_exception(
|
|
102
|
-
|
|
87
|
+
def load_exception(client, json_obj):
|
|
88
|
+
from .stub import Stub
|
|
89
|
+
|
|
90
|
+
json_obj = client.decode(json_obj)
|
|
91
|
+
|
|
103
92
|
if json_obj.get(FIELD_EXC_SI) is not None:
|
|
104
93
|
return StopIteration
|
|
105
94
|
|
|
106
95
|
exception_module = json_obj.get(FIELD_EXC_MODULE)
|
|
107
96
|
exception_name = json_obj.get(FIELD_EXC_NAME)
|
|
108
97
|
exception_class = None
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
# otherwise it will just ping pong.
|
|
124
|
-
name = "%s.%s" % (exception_module, exception_name)
|
|
125
|
-
exception_class = _remote_exceptions_class.setdefault(
|
|
126
|
-
name,
|
|
127
|
-
type(
|
|
128
|
-
name,
|
|
129
|
-
(RemoteInterpreterException,),
|
|
130
|
-
{"__module__": "%s/%s" % (__name__, exception_module)},
|
|
131
|
-
),
|
|
132
|
-
)
|
|
133
|
-
exception_class = _wrap_exception(exception_class)
|
|
134
|
-
raised_exception = exception_class.__new__(exception_class)
|
|
135
|
-
raised_exception.args = json_obj.get(FIELD_EXC_ARGS)
|
|
136
|
-
for name, attr in json_obj.get(FIELD_EXC_ATTR):
|
|
137
|
-
try:
|
|
138
|
-
if name in raised_exception.__user_defined__:
|
|
139
|
-
setattr(raised_exception, "_original_%s" % name, attr)
|
|
140
|
-
else:
|
|
141
|
-
setattr(raised_exception, name, attr)
|
|
142
|
-
except AttributeError:
|
|
143
|
-
# In case some things are read only
|
|
144
|
-
pass
|
|
145
|
-
s = json_obj.get(FIELD_EXC_STR)
|
|
146
|
-
if s:
|
|
147
|
-
try:
|
|
148
|
-
if "__str__" in raised_exception.__user_defined__:
|
|
149
|
-
setattr(raised_exception, "_original___str__", s)
|
|
150
|
-
else:
|
|
151
|
-
setattr(raised_exception, "__str__", lambda x, s=s: s)
|
|
152
|
-
except AttributeError:
|
|
153
|
-
raised_exception._missing_str = True
|
|
154
|
-
s = json_obj.get(FIELD_EXC_REPR)
|
|
155
|
-
if s:
|
|
156
|
-
try:
|
|
157
|
-
if "__repr__" in raised_exception.__user_defined__:
|
|
158
|
-
setattr(raised_exception, "_original___repr__", s)
|
|
159
|
-
else:
|
|
160
|
-
setattr(raised_exception, "__repr__", lambda x, s=s: s)
|
|
161
|
-
except AttributeError:
|
|
162
|
-
raised_exception._missing_repr = True
|
|
98
|
+
# This name is already cannonical since we cannonicalize it on the server side
|
|
99
|
+
full_name = "%s.%s" % (exception_module, exception_name)
|
|
100
|
+
|
|
101
|
+
exception_class = client.get_local_class(full_name, is_returned_exception=True)
|
|
102
|
+
|
|
103
|
+
if issubclass(exception_class, Stub):
|
|
104
|
+
raised_exception = exception_class(_is_returned_exception=True)
|
|
105
|
+
raised_exception.args = tuple(json_obj.get(FIELD_EXC_ARGS))
|
|
106
|
+
else:
|
|
107
|
+
raised_exception = exception_class(*json_obj.get(FIELD_EXC_ARGS))
|
|
108
|
+
raised_exception._exception_str = json_obj.get(FIELD_EXC_STR, None)
|
|
109
|
+
raised_exception._exception_repr = json_obj.get(FIELD_EXC_REPR, None)
|
|
110
|
+
raised_exception._exception_tb = json_obj.get(FIELD_EXC_TB, None)
|
|
111
|
+
|
|
163
112
|
user_args = json_obj.get(FIELD_EXC_USER)
|
|
164
113
|
if user_args is not None:
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
raised_exception._missing_deserializer = True
|
|
169
|
-
else:
|
|
170
|
-
deserializer(user_args)
|
|
171
|
-
raised_exception._remote_tb = json_obj[FIELD_EXC_TB]
|
|
114
|
+
deserializer = client.get_exception_deserializer(full_name)
|
|
115
|
+
if deserializer is not None:
|
|
116
|
+
deserializer(raised_exception, user_args)
|
|
172
117
|
return raised_exception
|
|
173
118
|
|
|
174
119
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
# text += "\n\n===== WARNING: Could not set class specific __str__ "
|
|
193
|
-
# "-- possible missing information =====\n"
|
|
194
|
-
# if getattr(self, "_missing_repr", False):
|
|
195
|
-
# text += "\n\n===== WARNING: Could not set class specific __repr__ "
|
|
196
|
-
# "-- possible missing information =====\n"
|
|
197
|
-
remote_tb = getattr(self, "_remote_tb", "No remote traceback available")
|
|
120
|
+
class ExceptionMetaClass(type):
|
|
121
|
+
def __init__(cls, class_name, base_classes, class_dict):
|
|
122
|
+
super(ExceptionMetaClass, cls).__init__(class_name, base_classes, class_dict)
|
|
123
|
+
cls.__orig_str__ = cls.__str__
|
|
124
|
+
cls.__orig_repr__ = cls.__repr__
|
|
125
|
+
for n in ("_exception_str", "_exception_repr", "_exception_tb"):
|
|
126
|
+
setattr(
|
|
127
|
+
cls,
|
|
128
|
+
n,
|
|
129
|
+
property(
|
|
130
|
+
lambda self, n=n: getattr(self, "%s_val" % n, "<missing>"),
|
|
131
|
+
lambda self, v, n=n: setattr(self, "%s_val" % n, v),
|
|
132
|
+
),
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def _do_str(self):
|
|
136
|
+
text = self._exception_str
|
|
198
137
|
text += "\n\n===== Remote (on server) traceback =====\n"
|
|
199
|
-
text +=
|
|
138
|
+
text += self._exception_tb
|
|
200
139
|
text += "========================================\n"
|
|
201
140
|
return text
|
|
202
141
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
WithPrettyPrinting.__realclass__ = exception_class
|
|
206
|
-
_derived_exceptions[exception_class] = WithPrettyPrinting
|
|
207
|
-
return WithPrettyPrinting
|
|
142
|
+
cls.__str__ = _do_str
|
|
143
|
+
cls.__repr__ = lambda self: self._exception_repr
|
|
208
144
|
|
|
209
145
|
|
|
210
146
|
class RemoteInterpreterException(Exception):
|
|
211
|
-
"""
|
|
212
|
-
the
|
|
147
|
+
"""
|
|
148
|
+
A 'generic' exception that was raised on the server side for which we have no
|
|
149
|
+
equivalent exception on this side
|
|
150
|
+
"""
|
|
213
151
|
|
|
214
152
|
pass
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
_remote_exceptions_class = {} # Exception name -> type of that exception
|
|
218
|
-
_derived_exceptions = {}
|
|
@@ -110,18 +110,18 @@ def remote_setattr_override(obj_mapping):
|
|
|
110
110
|
return _wrapped
|
|
111
111
|
|
|
112
112
|
|
|
113
|
-
class
|
|
114
|
-
def __init__(self, class_path,
|
|
113
|
+
class LocalExceptionDeserializer(object):
|
|
114
|
+
def __init__(self, class_path, deserializer):
|
|
115
115
|
self._class_path = class_path
|
|
116
|
-
self.
|
|
116
|
+
self._deserializer = deserializer
|
|
117
117
|
|
|
118
118
|
@property
|
|
119
119
|
def class_path(self):
|
|
120
120
|
return self._class_path
|
|
121
121
|
|
|
122
122
|
@property
|
|
123
|
-
def
|
|
124
|
-
return self.
|
|
123
|
+
def deserializer(self):
|
|
124
|
+
return self._deserializer
|
|
125
125
|
|
|
126
126
|
|
|
127
127
|
class RemoteExceptionSerializer(object):
|
|
@@ -138,9 +138,9 @@ class RemoteExceptionSerializer(object):
|
|
|
138
138
|
return self._serializer
|
|
139
139
|
|
|
140
140
|
|
|
141
|
-
def
|
|
142
|
-
def _wrapped(
|
|
143
|
-
return
|
|
141
|
+
def local_exception_deserialize(class_path):
|
|
142
|
+
def _wrapped(func):
|
|
143
|
+
return LocalExceptionDeserializer(class_path, func)
|
|
144
144
|
|
|
145
145
|
return _wrapped
|
|
146
146
|
|
|
@@ -36,6 +36,7 @@ from .consts import (
|
|
|
36
36
|
OP_GETVAL,
|
|
37
37
|
OP_SETVAL,
|
|
38
38
|
OP_INIT,
|
|
39
|
+
OP_SUBCLASSCHECK,
|
|
39
40
|
VALUE_LOCAL,
|
|
40
41
|
VALUE_REMOTE,
|
|
41
42
|
CONTROL_GETEXPORTS,
|
|
@@ -113,9 +114,21 @@ class Server(object):
|
|
|
113
114
|
# this by listing aliases in the same order so we don't support
|
|
114
115
|
# it for now.
|
|
115
116
|
raise ValueError(
|
|
116
|
-
"%s is an alias to both %s and %s
|
|
117
|
+
"%s is an alias to both %s and %s -- make sure all aliases "
|
|
118
|
+
"are listed in the same order" % (alias, base_name, a)
|
|
117
119
|
)
|
|
118
120
|
|
|
121
|
+
# Detect circular aliases. If a user lists ("a", "b") and then ("b", "a"), we
|
|
122
|
+
# will have an entry in aliases saying b is an alias for a and a is an alias
|
|
123
|
+
# for b which is a recipe for disaster since we no longer have a cannonical name
|
|
124
|
+
# for things.
|
|
125
|
+
for alias, base_name in self._aliases.items():
|
|
126
|
+
if base_name in self._aliases:
|
|
127
|
+
raise ValueError(
|
|
128
|
+
"%s and %s are circular aliases -- make sure all aliases "
|
|
129
|
+
"are listed in the same order" % (alias, base_name)
|
|
130
|
+
)
|
|
131
|
+
|
|
119
132
|
# Determine if we have any overrides
|
|
120
133
|
self._overrides = {}
|
|
121
134
|
self._getattr_overrides = {}
|
|
@@ -124,8 +137,9 @@ class Server(object):
|
|
|
124
137
|
for override in override_values:
|
|
125
138
|
if isinstance(override, (RemoteAttrOverride, RemoteOverride)):
|
|
126
139
|
for obj_name, obj_funcs in override.obj_mapping.items():
|
|
140
|
+
canonical_name = get_canonical_name(obj_name, self._aliases)
|
|
127
141
|
obj_type = self._known_classes.get(
|
|
128
|
-
|
|
142
|
+
canonical_name, self._proxied_types.get(obj_name)
|
|
129
143
|
)
|
|
130
144
|
if obj_type is None:
|
|
131
145
|
raise ValueError(
|
|
@@ -146,11 +160,17 @@ class Server(object):
|
|
|
146
160
|
)
|
|
147
161
|
override_dict[name] = override.func
|
|
148
162
|
elif isinstance(override, RemoteExceptionSerializer):
|
|
163
|
+
canonical_name = get_canonical_name(override.class_path, self._aliases)
|
|
164
|
+
if canonical_name not in self._known_exceptions:
|
|
165
|
+
raise ValueError(
|
|
166
|
+
"%s does not refer to an exported exception"
|
|
167
|
+
% override.class_path
|
|
168
|
+
)
|
|
149
169
|
if override.class_path in self._exception_serializers:
|
|
150
170
|
raise ValueError(
|
|
151
171
|
"%s exception serializer already defined" % override.class_path
|
|
152
172
|
)
|
|
153
|
-
self._exception_serializers[
|
|
173
|
+
self._exception_serializers[canonical_name] = override.serializer
|
|
154
174
|
|
|
155
175
|
# Process the exceptions making sure we have all the ones we need and building a
|
|
156
176
|
# topologically sorted list for the client to instantiate
|
|
@@ -181,8 +201,8 @@ class Server(object):
|
|
|
181
201
|
else:
|
|
182
202
|
raise ValueError(
|
|
183
203
|
"Exported exception %s has non exported and non builtin parent "
|
|
184
|
-
"exception: %s. Known exceptions: %s"
|
|
185
|
-
% (ex_name, fqn, str(self._known_exceptions))
|
|
204
|
+
"exception: %s (%s). Known exceptions: %s."
|
|
205
|
+
% (ex_name, fqn, canonical_fqn, str(self._known_exceptions))
|
|
186
206
|
)
|
|
187
207
|
name_to_parent_count[ex_name_canonical] = len(parents) - 1
|
|
188
208
|
name_to_parents[ex_name_canonical] = parents
|
|
@@ -236,6 +256,7 @@ class Server(object):
|
|
|
236
256
|
OP_GETVAL: self._handle_getval,
|
|
237
257
|
OP_SETVAL: self._handle_setval,
|
|
238
258
|
OP_INIT: self._handle_init,
|
|
259
|
+
OP_SUBCLASSCHECK: self._handle_subclasscheck,
|
|
239
260
|
}
|
|
240
261
|
|
|
241
262
|
self._local_objects = {}
|
|
@@ -273,6 +294,7 @@ class Server(object):
|
|
|
273
294
|
def encode_exception(self, ex_type, ex, trace_back):
|
|
274
295
|
try:
|
|
275
296
|
full_name = "%s.%s" % (ex_type.__module__, ex_type.__name__)
|
|
297
|
+
get_canonical_name(full_name, self._aliases)
|
|
276
298
|
serializer = self._exception_serializers.get(full_name)
|
|
277
299
|
except AttributeError:
|
|
278
300
|
# Ignore if no __module__ for example -- definitely not something we built
|
|
@@ -483,6 +505,21 @@ class Server(object):
|
|
|
483
505
|
raise ValueError("Unknown class %s" % class_name)
|
|
484
506
|
return class_type(*args, **kwargs)
|
|
485
507
|
|
|
508
|
+
def _handle_subclasscheck(self, target, class_name, otherclass_name, reverse=False):
|
|
509
|
+
class_type = self._known_classes.get(class_name)
|
|
510
|
+
if class_type is None:
|
|
511
|
+
raise ValueError("Unknown class %s" % class_name)
|
|
512
|
+
try:
|
|
513
|
+
sub_module, sub_name = otherclass_name.rsplit(".", 1)
|
|
514
|
+
__import__(sub_module, None, None, "*")
|
|
515
|
+
except Exception:
|
|
516
|
+
sub_module = None
|
|
517
|
+
if sub_module is None:
|
|
518
|
+
return False
|
|
519
|
+
if reverse:
|
|
520
|
+
return issubclass(class_type, getattr(sys.modules[sub_module], sub_name))
|
|
521
|
+
return issubclass(getattr(sys.modules[sub_module], sub_name), class_type)
|
|
522
|
+
|
|
486
523
|
|
|
487
524
|
if __name__ == "__main__":
|
|
488
525
|
max_pickle_version = int(sys.argv[1])
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import functools
|
|
2
2
|
import pickle
|
|
3
|
+
from typing import Any
|
|
3
4
|
|
|
4
5
|
from .consts import (
|
|
5
6
|
OP_GETATTR,
|
|
@@ -15,8 +16,11 @@ from .consts import (
|
|
|
15
16
|
OP_PICKLE,
|
|
16
17
|
OP_DIR,
|
|
17
18
|
OP_INIT,
|
|
19
|
+
OP_SUBCLASSCHECK,
|
|
18
20
|
)
|
|
19
21
|
|
|
22
|
+
from .exception_transferer import ExceptionMetaClass
|
|
23
|
+
|
|
20
24
|
DELETED_ATTRS = frozenset(["__array_struct__", "__array_interface__"])
|
|
21
25
|
|
|
22
26
|
# These attributes are accessed directly on the stub (not directly forwarded)
|
|
@@ -26,7 +30,10 @@ LOCAL_ATTRS = (
|
|
|
26
30
|
"___remote_class_name___",
|
|
27
31
|
"___identifier___",
|
|
28
32
|
"___connection___",
|
|
29
|
-
"___local_overrides___"
|
|
33
|
+
"___local_overrides___",
|
|
34
|
+
"___is_returned_exception___",
|
|
35
|
+
"___exception_attributes___",
|
|
36
|
+
"__class__",
|
|
30
37
|
"__init__",
|
|
31
38
|
"__del__",
|
|
32
39
|
"__delattr__",
|
|
@@ -36,9 +43,11 @@ LOCAL_ATTRS = (
|
|
|
36
43
|
"__getattribute__",
|
|
37
44
|
"__hash__",
|
|
38
45
|
"__instancecheck__",
|
|
46
|
+
"__subclasscheck__",
|
|
39
47
|
"__init__",
|
|
40
48
|
"__metaclass__",
|
|
41
49
|
"__module__",
|
|
50
|
+
"__name__",
|
|
42
51
|
"__new__",
|
|
43
52
|
"__reduce__",
|
|
44
53
|
"__reduce_ex__",
|
|
@@ -62,17 +71,19 @@ CLASS_METHOD = 2
|
|
|
62
71
|
|
|
63
72
|
def fwd_request(stub, request_type, *args, **kwargs):
|
|
64
73
|
connection = object.__getattribute__(stub, "___connection___")
|
|
65
|
-
|
|
74
|
+
if connection:
|
|
75
|
+
return connection.stub_request(stub, request_type, *args, **kwargs)
|
|
76
|
+
raise RuntimeError(
|
|
77
|
+
"Returned exception stub cannot be used to make further remote requests"
|
|
78
|
+
)
|
|
66
79
|
|
|
67
80
|
|
|
68
81
|
class StubMetaClass(type):
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
if self.__module__:
|
|
73
|
-
return "<stub class '%s.%s'>" % (self.__module__, self.__name__)
|
|
82
|
+
def __repr__(cls):
|
|
83
|
+
if cls.__module__:
|
|
84
|
+
return "<stub class '%s.%s'>" % (cls.__module__, cls.__name__)
|
|
74
85
|
else:
|
|
75
|
-
return "<stub class '%s'>" % (
|
|
86
|
+
return "<stub class '%s'>" % (cls.__name__,)
|
|
76
87
|
|
|
77
88
|
|
|
78
89
|
def with_metaclass(meta, *bases):
|
|
@@ -94,24 +105,25 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
94
105
|
happen on the remote side (server).
|
|
95
106
|
"""
|
|
96
107
|
|
|
97
|
-
__slots__ =
|
|
98
|
-
"___remote_class_name___",
|
|
99
|
-
"___identifier___",
|
|
100
|
-
"___connection___",
|
|
101
|
-
"__weakref__",
|
|
102
|
-
]
|
|
103
|
-
|
|
108
|
+
__slots__ = ()
|
|
104
109
|
# def __iter__(self): # FIXME: Keep debugger QUIET!!
|
|
105
110
|
# raise AttributeError
|
|
106
111
|
|
|
107
|
-
def __init__(
|
|
112
|
+
def __init__(
|
|
113
|
+
self, connection, remote_class_name, identifier, _is_returned_exception=False
|
|
114
|
+
):
|
|
108
115
|
self.___remote_class_name___ = remote_class_name
|
|
109
116
|
self.___identifier___ = identifier
|
|
110
117
|
self.___connection___ = connection
|
|
118
|
+
# If it is a returned exception (ie: it was raised by the server), it behaves
|
|
119
|
+
# a bit differently for methods like __str__ and __repr__ (we try not to get
|
|
120
|
+
# stuff from the server)
|
|
121
|
+
self.___is_returned_exception___ = _is_returned_exception
|
|
111
122
|
|
|
112
123
|
def __del__(self):
|
|
113
124
|
try:
|
|
114
|
-
|
|
125
|
+
if not self.___is_returned_exception___:
|
|
126
|
+
fwd_request(self, OP_DEL)
|
|
115
127
|
except Exception:
|
|
116
128
|
# raised in a destructor, most likely on program termination,
|
|
117
129
|
# when the connection might have already been closed.
|
|
@@ -120,9 +132,7 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
120
132
|
|
|
121
133
|
def __getattribute__(self, name):
|
|
122
134
|
if name in LOCAL_ATTRS:
|
|
123
|
-
if name == "
|
|
124
|
-
return None
|
|
125
|
-
elif name == "__doc__":
|
|
135
|
+
if name == "__doc__":
|
|
126
136
|
return self.__getattr__("__doc__")
|
|
127
137
|
elif name in DELETED_ATTRS:
|
|
128
138
|
raise AttributeError()
|
|
@@ -144,10 +154,16 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
144
154
|
if name in LOCAL_ATTRS:
|
|
145
155
|
object.__delattr__(self, name)
|
|
146
156
|
else:
|
|
157
|
+
if self.___is_returned_exception___:
|
|
158
|
+
raise AttributeError()
|
|
147
159
|
return fwd_request(self, OP_DELATTR, name)
|
|
148
160
|
|
|
149
161
|
def __setattr__(self, name, value):
|
|
150
|
-
if
|
|
162
|
+
if (
|
|
163
|
+
name in LOCAL_ATTRS
|
|
164
|
+
or name in self.___local_overrides___
|
|
165
|
+
or self.___is_returned_exception___
|
|
166
|
+
):
|
|
151
167
|
object.__setattr__(self, name, value)
|
|
152
168
|
else:
|
|
153
169
|
fwd_request(self, OP_SETATTR, name, value)
|
|
@@ -159,9 +175,13 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
159
175
|
return fwd_request(self, OP_HASH)
|
|
160
176
|
|
|
161
177
|
def __repr__(self):
|
|
178
|
+
if self.___is_returned_exception___:
|
|
179
|
+
return self.__exception_repr__()
|
|
162
180
|
return fwd_request(self, OP_REPR)
|
|
163
181
|
|
|
164
182
|
def __str__(self):
|
|
183
|
+
if self.___is_returned_exception___:
|
|
184
|
+
return self.__exception_str__()
|
|
165
185
|
return fwd_request(self, OP_STR)
|
|
166
186
|
|
|
167
187
|
def __exit__(self, exc, typ, tb):
|
|
@@ -173,6 +193,16 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
173
193
|
# support for pickling
|
|
174
194
|
return pickle.loads, (fwd_request(self, OP_PICKLE, proto),)
|
|
175
195
|
|
|
196
|
+
@classmethod
|
|
197
|
+
def __subclasshook__(cls, parent):
|
|
198
|
+
if parent.__bases__[0] == Stub:
|
|
199
|
+
raise NotImplementedError # Follow the usual mechanism
|
|
200
|
+
# If this is not a stub, we go over to the other side
|
|
201
|
+
parent_name = "%s.%s" % (parent.__module__, parent.__name__)
|
|
202
|
+
return cls.___class_connection___.stub_request(
|
|
203
|
+
None, OP_SUBCLASSCHECK, cls.___class_remote_class_name___, parent_name, True
|
|
204
|
+
)
|
|
205
|
+
|
|
176
206
|
|
|
177
207
|
def _make_method(method_type, connection, class_name, name, doc):
|
|
178
208
|
if name == "__call__":
|
|
@@ -248,6 +278,80 @@ class MetaWithConnection(StubMetaClass):
|
|
|
248
278
|
None, OP_INIT, cls.___class_remote_class_name___, *args, **kwargs
|
|
249
279
|
)
|
|
250
280
|
|
|
281
|
+
def __subclasscheck__(cls, subclass):
|
|
282
|
+
subclass_name = "%s.%s" % (subclass.__module__, subclass.__name__)
|
|
283
|
+
if subclass.__bases__[0] == Stub:
|
|
284
|
+
subclass_name = subclass.___class_remote_class_name___
|
|
285
|
+
return cls.___class_connection___.stub_request(
|
|
286
|
+
None,
|
|
287
|
+
OP_SUBCLASSCHECK,
|
|
288
|
+
cls.___class_remote_class_name___,
|
|
289
|
+
subclass_name,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
def __instancecheck__(cls, instance):
|
|
293
|
+
if type(instance) == cls:
|
|
294
|
+
# Fast path if it's just an object of this class
|
|
295
|
+
return True
|
|
296
|
+
# Goes to __subclasscheck__ above
|
|
297
|
+
return cls.__subclasscheck__(type(instance))
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
class MetaExceptionWithConnection(StubMetaClass, ExceptionMetaClass):
|
|
301
|
+
def __new__(cls, class_name, base_classes, class_dict, connection):
|
|
302
|
+
return type.__new__(cls, class_name, base_classes, class_dict)
|
|
303
|
+
|
|
304
|
+
def __init__(cls, class_name, base_classes, class_dict, connection):
|
|
305
|
+
cls.___class_remote_class_name___ = class_name
|
|
306
|
+
cls.___class_connection___ = connection
|
|
307
|
+
|
|
308
|
+
# We call the one on ExceptionMetaClass which does everything needed (StubMetaClass
|
|
309
|
+
# does not do anything special for init)
|
|
310
|
+
ExceptionMetaClass.__init__(cls, class_name, base_classes, class_dict)
|
|
311
|
+
|
|
312
|
+
# Restore __str__ and __repr__ to the original ones because we need to determine
|
|
313
|
+
# if we call them depending on whether or not the object is a returned exception
|
|
314
|
+
# or not
|
|
315
|
+
cls.__exception_str__ = cls.__str__
|
|
316
|
+
cls.__exception_repr__ = cls.__repr__
|
|
317
|
+
cls.__str__ = cls.__orig_str__
|
|
318
|
+
cls.__repr__ = cls.__orig_repr__
|
|
319
|
+
|
|
320
|
+
def __call__(cls, *args, **kwargs):
|
|
321
|
+
# Very similar to the other case but we also need to be able to detect
|
|
322
|
+
# local instantiation of an exception so that we can set the __is_returned_exception__
|
|
323
|
+
if len(args) > 0 and id(args[0]) == id(cls.___class_connection___):
|
|
324
|
+
return super(MetaExceptionWithConnection, cls).__call__(*args, **kwargs)
|
|
325
|
+
elif kwargs and kwargs.get("_is_returned_exception", False):
|
|
326
|
+
return super(MetaExceptionWithConnection, cls).__call__(
|
|
327
|
+
None, None, None, _is_returned_exception=True
|
|
328
|
+
)
|
|
329
|
+
else:
|
|
330
|
+
return cls.___class_connection___.stub_request(
|
|
331
|
+
None, OP_INIT, cls.___class_remote_class_name___, *args, **kwargs
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
# The issue is that for a proxied object that is also an exception, we now have
|
|
335
|
+
# two classes representing it, one that includes the Stub class and one that doesn't
|
|
336
|
+
# Concretely:
|
|
337
|
+
# - test.MyException would return a class that derives from Stub
|
|
338
|
+
# - test.MySubException would return a class that derives from Stub and test.MyException
|
|
339
|
+
# but WITHOUT the Stub portion (see get_local_class).
|
|
340
|
+
# - we want issubclass(test.MySubException, test.MyException) to return True and
|
|
341
|
+
# the same with instance checks.
|
|
342
|
+
def __instancecheck__(cls, instance):
|
|
343
|
+
return cls.__subclasscheck__(type(instance))
|
|
344
|
+
|
|
345
|
+
def __subclasscheck__(cls, subclass):
|
|
346
|
+
# __mro__[0] is this class itself
|
|
347
|
+
# __mro__[1] is the stub so we start checking at 2
|
|
348
|
+
return any(
|
|
349
|
+
[
|
|
350
|
+
subclass.__mro__[i] in cls.__mro__[2:]
|
|
351
|
+
for i in range(2, len(subclass.__mro__))
|
|
352
|
+
]
|
|
353
|
+
)
|
|
354
|
+
|
|
251
355
|
|
|
252
356
|
def create_class(
|
|
253
357
|
connection,
|
|
@@ -256,8 +360,16 @@ def create_class(
|
|
|
256
360
|
getattr_overrides,
|
|
257
361
|
setattr_overrides,
|
|
258
362
|
class_methods,
|
|
363
|
+
parents,
|
|
259
364
|
):
|
|
260
|
-
class_dict = {
|
|
365
|
+
class_dict = {
|
|
366
|
+
"__slots__": [
|
|
367
|
+
"___remote_class_name___",
|
|
368
|
+
"___identifier___",
|
|
369
|
+
"___connection___",
|
|
370
|
+
"___is_returned_exception___",
|
|
371
|
+
]
|
|
372
|
+
}
|
|
261
373
|
for name, doc in class_methods.items():
|
|
262
374
|
method_type = NORMAL_METHOD
|
|
263
375
|
if name.startswith("___s___"):
|
|
@@ -318,5 +430,38 @@ def create_class(
|
|
|
318
430
|
)
|
|
319
431
|
overriden_attrs.add(attr)
|
|
320
432
|
class_dict[attr] = property(getter, setter)
|
|
433
|
+
if parents:
|
|
434
|
+
# This means this is also an exception so we add a few more things to it
|
|
435
|
+
# so that it
|
|
436
|
+
# This is copied from ExceptionMetaClass in exception_transferer.py
|
|
437
|
+
for n in ("_exception_str", "_exception_repr", "_exception_tb"):
|
|
438
|
+
class_dict[n] = property(
|
|
439
|
+
lambda self, n=n: getattr(self, "%s_val" % n, "<missing>"),
|
|
440
|
+
lambda self, v, n=n: setattr(self, "%s_val" % n, v),
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
def _do_str(self):
|
|
444
|
+
text = self._exception_str
|
|
445
|
+
text += "\n\n===== Remote (on server) traceback =====\n"
|
|
446
|
+
text += self._exception_tb
|
|
447
|
+
text += "========================================\n"
|
|
448
|
+
return text
|
|
449
|
+
|
|
450
|
+
class_dict["__exception_str__"] = _do_str
|
|
451
|
+
class_dict["__exception_repr__"] = lambda self: self._exception_repr
|
|
452
|
+
else:
|
|
453
|
+
# If we are based on an exception, we already have __weakref__ so we don't add
|
|
454
|
+
# it but not the case if we are not.
|
|
455
|
+
class_dict["__slots__"].append("__weakref__")
|
|
456
|
+
|
|
457
|
+
class_module, class_name_only = class_name.rsplit(".", 1)
|
|
321
458
|
class_dict["___local_overrides___"] = overriden_attrs
|
|
322
|
-
|
|
459
|
+
class_dict["__module__"] = class_module
|
|
460
|
+
if parents:
|
|
461
|
+
to_return = MetaExceptionWithConnection(
|
|
462
|
+
class_name, (Stub, *parents), class_dict, connection
|
|
463
|
+
)
|
|
464
|
+
else:
|
|
465
|
+
to_return = MetaWithConnection(class_name, (Stub,), class_dict, connection)
|
|
466
|
+
to_return.__name__ = class_name_only
|
|
467
|
+
return to_return
|