pyflyby 1.10.1__cp311-cp311-manylinux_2_24_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyflyby might be problematic. Click here for more details.
- pyflyby/__init__.py +61 -0
- pyflyby/__main__.py +9 -0
- pyflyby/_autoimp.py +2229 -0
- pyflyby/_cmdline.py +548 -0
- pyflyby/_comms.py +221 -0
- pyflyby/_dbg.py +1367 -0
- pyflyby/_docxref.py +379 -0
- pyflyby/_dynimp.py +154 -0
- pyflyby/_fast_iter_modules.cpython-311-x86_64-linux-gnu.so +0 -0
- pyflyby/_file.py +771 -0
- pyflyby/_flags.py +230 -0
- pyflyby/_format.py +186 -0
- pyflyby/_idents.py +227 -0
- pyflyby/_import_sorting.py +165 -0
- pyflyby/_importclns.py +658 -0
- pyflyby/_importdb.py +680 -0
- pyflyby/_imports2s.py +643 -0
- pyflyby/_importstmt.py +723 -0
- pyflyby/_interactive.py +2113 -0
- pyflyby/_livepatch.py +793 -0
- pyflyby/_log.py +104 -0
- pyflyby/_modules.py +641 -0
- pyflyby/_parse.py +1381 -0
- pyflyby/_py.py +2166 -0
- pyflyby/_saveframe.py +1145 -0
- pyflyby/_saveframe_reader.py +471 -0
- pyflyby/_util.py +458 -0
- pyflyby/_version.py +7 -0
- pyflyby/autoimport.py +20 -0
- pyflyby/etc/pyflyby/canonical.py +10 -0
- pyflyby/etc/pyflyby/common.py +27 -0
- pyflyby/etc/pyflyby/forget.py +10 -0
- pyflyby/etc/pyflyby/mandatory.py +10 -0
- pyflyby/etc/pyflyby/numpy.py +156 -0
- pyflyby/etc/pyflyby/std.py +335 -0
- pyflyby/importdb.py +19 -0
- pyflyby/libexec/pyflyby/colordiff +34 -0
- pyflyby/libexec/pyflyby/diff-colorize +148 -0
- pyflyby/share/emacs/site-lisp/pyflyby.el +108 -0
- pyflyby-1.10.1.data/scripts/collect-exports +76 -0
- pyflyby-1.10.1.data/scripts/collect-imports +58 -0
- pyflyby-1.10.1.data/scripts/find-import +38 -0
- pyflyby-1.10.1.data/scripts/list-bad-xrefs +34 -0
- pyflyby-1.10.1.data/scripts/prune-broken-imports +34 -0
- pyflyby-1.10.1.data/scripts/pyflyby-diff +34 -0
- pyflyby-1.10.1.data/scripts/reformat-imports +27 -0
- pyflyby-1.10.1.data/scripts/replace-star-imports +37 -0
- pyflyby-1.10.1.data/scripts/saveframe +299 -0
- pyflyby-1.10.1.data/scripts/tidy-imports +163 -0
- pyflyby-1.10.1.data/scripts/transform-imports +47 -0
- pyflyby-1.10.1.dist-info/METADATA +591 -0
- pyflyby-1.10.1.dist-info/RECORD +55 -0
- pyflyby-1.10.1.dist-info/WHEEL +5 -0
- pyflyby-1.10.1.dist-info/entry_points.txt +4 -0
- pyflyby-1.10.1.dist-info/licenses/LICENSE.txt +23 -0
pyflyby/_livepatch.py
ADDED
|
@@ -0,0 +1,793 @@
|
|
|
1
|
+
# pyflyby/_livepatch.py
|
|
2
|
+
# Copyright (C) 2011, 2012, 2013, 2014, 2015 Karl Chen.
|
|
3
|
+
|
|
4
|
+
r"""
|
|
5
|
+
livepatch/xreload: Alternative to reload().
|
|
6
|
+
|
|
7
|
+
xreload performs a "live patch" of the modules/classes/functions/etc that have
|
|
8
|
+
already been loaded in memory. It does so by executing the module in a
|
|
9
|
+
scratch namespace, and then patching classes, methods and functions in-place.
|
|
10
|
+
New objects are copied into the target namespace.
|
|
11
|
+
|
|
12
|
+
This addresses cases where one module imported functions from another
|
|
13
|
+
module.
|
|
14
|
+
|
|
15
|
+
For example, suppose m1.py contains::
|
|
16
|
+
|
|
17
|
+
from m2 import foo
|
|
18
|
+
def print_foo():
|
|
19
|
+
return foo()
|
|
20
|
+
|
|
21
|
+
and m2.py contains::
|
|
22
|
+
|
|
23
|
+
def foo():
|
|
24
|
+
return 42
|
|
25
|
+
|
|
26
|
+
If you edit m2.py and modify ``foo``, then reload(m2) on its own would not do
|
|
27
|
+
what you want. You would also need to reload(m1) after reload(m2). This is
|
|
28
|
+
because the built-in reload affects the module being reloaded, but references
|
|
29
|
+
to the old module remain. On the other hand, xreload() patches the existing
|
|
30
|
+
m2.foo, so that live references to it are updated.
|
|
31
|
+
|
|
32
|
+
In table form::
|
|
33
|
+
|
|
34
|
+
Undesired effect: reload(m2)
|
|
35
|
+
Undesired effect: reload(m1); reload(m2)
|
|
36
|
+
Desired effect: reload(m2); reload(m1)
|
|
37
|
+
|
|
38
|
+
Desired effect: xreload(m2)
|
|
39
|
+
Desired effect: xreload(m1); xreload(m2)
|
|
40
|
+
Desired effect: xreload(m2); xreload(m1)
|
|
41
|
+
|
|
42
|
+
Even with just two modules, we can see that xreload() is an improvement. When
|
|
43
|
+
working with a large set of interdependent modules, it becomes infeasible to
|
|
44
|
+
know the precise sequence of reload() calls that would be necessary.
|
|
45
|
+
xreload() really shines in that case.
|
|
46
|
+
|
|
47
|
+
This implementation of xreload() was originally based the following
|
|
48
|
+
mailing-list post by Guido van Rossum:
|
|
49
|
+
|
|
50
|
+
https://mail.python.org/pipermail/edu-sig/2007-February/007787.html
|
|
51
|
+
|
|
52
|
+
Customizing behavior
|
|
53
|
+
====================
|
|
54
|
+
|
|
55
|
+
If a class/function/module/etc has an attribute __livepatch__, then this
|
|
56
|
+
function is called *instead* of performing the regular livepatch mechanism.
|
|
57
|
+
|
|
58
|
+
The __livepatch__() function is called with the following arguments:
|
|
59
|
+
|
|
60
|
+
- ``old`` : The object to be updated with contents of ``new``
|
|
61
|
+
- ``new`` : The object whose contents to put into ``old``
|
|
62
|
+
- ``do_livepatch``: A function that can be called to do the standard
|
|
63
|
+
livepatch, replacing the contents of ``old`` with ``new``.
|
|
64
|
+
If it's not possible to livepatch ``old``, it returns
|
|
65
|
+
``new``. The ``do_livepatch`` function takes no arguments.
|
|
66
|
+
Calling the ``do_livepatch`` function is roughly
|
|
67
|
+
equivalent to calling ``pyflyby.livepatch(old, new,
|
|
68
|
+
modname=modname, heed_hook=False)``.
|
|
69
|
+
- ``modname`` : The module currently being updated. Recursively called
|
|
70
|
+
updates should keep track of the module being updated to
|
|
71
|
+
avoid touching other modules.
|
|
72
|
+
|
|
73
|
+
These arguments are matched by *name* and are passed only if the
|
|
74
|
+
``__livepatch__`` function is declared to take such named arguments or it takes
|
|
75
|
+
\**kwargs. If the ``__livepatch__`` function takes \**kwargs, it should ignore
|
|
76
|
+
unknown arguments, in case new parameters are added in the future.
|
|
77
|
+
|
|
78
|
+
If the object being updated is an object instance, and ``__livepatch__`` is a
|
|
79
|
+
method, then the function is bound to the new object, i.e. the ``self``
|
|
80
|
+
parameter is the same as ``new``.
|
|
81
|
+
|
|
82
|
+
If the ``__livepatch__`` function successfully patched the ``old`` object, then
|
|
83
|
+
it should return ``old``. If it is unable to patch, it should return ``new``.
|
|
84
|
+
|
|
85
|
+
Examples
|
|
86
|
+
--------
|
|
87
|
+
|
|
88
|
+
By default, any attributes on an existing function are updated with ones from
|
|
89
|
+
the new function. If you want a memoized function to keep its cache across
|
|
90
|
+
xreload, you could implement that like this::
|
|
91
|
+
|
|
92
|
+
def memoize(function):
|
|
93
|
+
cache = {}
|
|
94
|
+
def wrapped_fn(*args):
|
|
95
|
+
try:
|
|
96
|
+
return cache[args]
|
|
97
|
+
except KeyError:
|
|
98
|
+
result = function(*args)
|
|
99
|
+
cache[args] = result
|
|
100
|
+
return result
|
|
101
|
+
wrapped_fn.cache = cache
|
|
102
|
+
def my_livepatch(old, new, do_livepatch):
|
|
103
|
+
keep_cache = dict(old.cache)
|
|
104
|
+
result = do_livepatch()
|
|
105
|
+
result.cache.update(keep_cache)
|
|
106
|
+
return result
|
|
107
|
+
wrapped_fn.__livepatch__ = my_livepatch
|
|
108
|
+
return wrapped_fn
|
|
109
|
+
|
|
110
|
+
XXX change example b/c cache is already cleared by default
|
|
111
|
+
XXX maybe global cache
|
|
112
|
+
|
|
113
|
+
class MyObj(...):
|
|
114
|
+
def __livepatch__(self, old):
|
|
115
|
+
self.__dict__.update(old.__dict__)
|
|
116
|
+
return self
|
|
117
|
+
|
|
118
|
+
class MyObj(...):
|
|
119
|
+
def __init__(self):
|
|
120
|
+
self._my_cache = {}
|
|
121
|
+
|
|
122
|
+
def __livepatch__(self, old, do_livepatch):
|
|
123
|
+
keep_cache = dict(old._my_cache)
|
|
124
|
+
result = do_livepatch()
|
|
125
|
+
result._my_cache.update(keep_cache)
|
|
126
|
+
return result
|
|
127
|
+
|
|
128
|
+
XXX test
|
|
129
|
+
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
import ast
|
|
135
|
+
import os
|
|
136
|
+
import re
|
|
137
|
+
import sys
|
|
138
|
+
import time
|
|
139
|
+
import types
|
|
140
|
+
|
|
141
|
+
from importlib import reload as reload_module
|
|
142
|
+
|
|
143
|
+
import inspect
|
|
144
|
+
from pyflyby._log import logger
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# Keep track of when the process was started.
|
|
148
|
+
if os.uname()[0] == 'Linux':
|
|
149
|
+
_PROCESS_START_TIME = os.stat("/proc/%d"%os.getpid()).st_ctime
|
|
150
|
+
else:
|
|
151
|
+
try:
|
|
152
|
+
import psutil
|
|
153
|
+
except ImportError:
|
|
154
|
+
# Todo: better fallback
|
|
155
|
+
_PROCESS_START_TIME = time.time()
|
|
156
|
+
else:
|
|
157
|
+
_PROCESS_START_TIME = psutil.Process(os.getpid()).create_time()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class UnknownModuleError(ImportError):
|
|
161
|
+
pass
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def livepatch(old, new, modname=None,
|
|
165
|
+
visit_stack=(), cache=None, assume_type=None,
|
|
166
|
+
heed_hook=True):
|
|
167
|
+
"""
|
|
168
|
+
Livepatch ``old`` with contents of ``new``.
|
|
169
|
+
|
|
170
|
+
If ``old`` can't be livepatched, then return ``new``.
|
|
171
|
+
|
|
172
|
+
:param old:
|
|
173
|
+
The object to be updated
|
|
174
|
+
:param new:
|
|
175
|
+
The object used as the source for the update.
|
|
176
|
+
:type modname:
|
|
177
|
+
``str``
|
|
178
|
+
:param modname:
|
|
179
|
+
Only livepatch ``old`` if it was defined in the given fully-qualified
|
|
180
|
+
module name. If ``None``, then update regardless of module.
|
|
181
|
+
:param assume_type:
|
|
182
|
+
Update as if both ``old`` and ``new`` were of type ``assume_type``. If
|
|
183
|
+
``None``, then ``old`` and ``new`` must have the same type.
|
|
184
|
+
For internal use.
|
|
185
|
+
:param cache:
|
|
186
|
+
Cache of already-updated objects. Map from (id(old), id(new)) to result.
|
|
187
|
+
:param visit_stack:
|
|
188
|
+
Ids of objects that are currently being updated.
|
|
189
|
+
Used to deal with reference cycles.
|
|
190
|
+
For internal use.
|
|
191
|
+
:param heed_hook:
|
|
192
|
+
If ``True``, heed the ``__livepatch__`` hook on ``new``, if any.
|
|
193
|
+
If ``False``, ignore any ``__livepatch__`` hook on ``new``.
|
|
194
|
+
:return:
|
|
195
|
+
Either live-patched ``old``, or ``new``.
|
|
196
|
+
"""
|
|
197
|
+
if old is new:
|
|
198
|
+
return new
|
|
199
|
+
# If we're already visiting this object (due to a reference cycle), then
|
|
200
|
+
# don't recurse again.
|
|
201
|
+
if id(old) in visit_stack:
|
|
202
|
+
return old
|
|
203
|
+
if cache is None:
|
|
204
|
+
cache = {}
|
|
205
|
+
cachekey = (id(old), id(new))
|
|
206
|
+
try:
|
|
207
|
+
return cache[cachekey]
|
|
208
|
+
except KeyError:
|
|
209
|
+
pass
|
|
210
|
+
visit_stack += (id(old),)
|
|
211
|
+
def do_livepatch():
|
|
212
|
+
new_modname = _get_definition_module(new)
|
|
213
|
+
if modname and new_modname and new_modname != modname:
|
|
214
|
+
# Ignore objects that have been imported from another module.
|
|
215
|
+
# Just update their references.
|
|
216
|
+
return new
|
|
217
|
+
if assume_type is not None:
|
|
218
|
+
use_type = assume_type
|
|
219
|
+
else:
|
|
220
|
+
oldtype = type(old)
|
|
221
|
+
newtype = type(new)
|
|
222
|
+
if oldtype is newtype:
|
|
223
|
+
# Easy, common case: Type didn't change.
|
|
224
|
+
use_type = oldtype
|
|
225
|
+
elif (oldtype.__name__ == newtype.__name__ and
|
|
226
|
+
oldtype.__module__ == newtype.__module__ == modname and
|
|
227
|
+
getattr(sys.modules[modname],
|
|
228
|
+
newtype.__name__, None) is newtype and
|
|
229
|
+
oldtype is livepatch(
|
|
230
|
+
oldtype, newtype, modname=modname,
|
|
231
|
+
visit_stack=visit_stack, cache=cache)):
|
|
232
|
+
# Type of this object was defined in this module. This
|
|
233
|
+
# includes metaclasses defined in the same module.
|
|
234
|
+
use_type = oldtype
|
|
235
|
+
else:
|
|
236
|
+
# If the type changed, then give up.
|
|
237
|
+
return new
|
|
238
|
+
try:
|
|
239
|
+
mro = type.mro(use_type)
|
|
240
|
+
except TypeError:
|
|
241
|
+
mro = [use_type, object] # old-style class
|
|
242
|
+
# Dispatch on type. Include parent classes (in C3 linearized
|
|
243
|
+
# method resolution order), in particular so that this works on
|
|
244
|
+
# classes with custom metaclasses that subclass ``type``.
|
|
245
|
+
for t in mro:
|
|
246
|
+
try:
|
|
247
|
+
update = _LIVEPATCH_DISPATCH_TABLE[t]
|
|
248
|
+
break
|
|
249
|
+
except KeyError:
|
|
250
|
+
pass
|
|
251
|
+
else:
|
|
252
|
+
# We should have found at least ``object``
|
|
253
|
+
raise AssertionError("unreachable")
|
|
254
|
+
# Dispatch.
|
|
255
|
+
return update(old, new, modname=modname,
|
|
256
|
+
cache=cache, visit_stack=visit_stack)
|
|
257
|
+
if heed_hook:
|
|
258
|
+
hook = (getattr(new, "__livepatch__", None) or
|
|
259
|
+
getattr(new, "__reload_update__", None))
|
|
260
|
+
# XXX if unbound method or a descriptor, then we should ignore it.
|
|
261
|
+
# XXX test for that.
|
|
262
|
+
else:
|
|
263
|
+
hook = None
|
|
264
|
+
if hook is None:
|
|
265
|
+
# No hook is defined or the caller instructed us to ignore it.
|
|
266
|
+
# Do the standard livepatch.
|
|
267
|
+
result = do_livepatch()
|
|
268
|
+
else:
|
|
269
|
+
# Call a hook for updating.
|
|
270
|
+
# Build dict of optional kwargs.
|
|
271
|
+
avail_kwargs = dict(
|
|
272
|
+
old=old,
|
|
273
|
+
new=new,
|
|
274
|
+
do_livepatch=do_livepatch,
|
|
275
|
+
modname=modname,
|
|
276
|
+
cache=cache,
|
|
277
|
+
visit_stack=visit_stack)
|
|
278
|
+
# Find out which optional kwargs the hook wants.
|
|
279
|
+
kwargs = {}
|
|
280
|
+
argspec = inspect.getfullargspec(hook)
|
|
281
|
+
argnames = argspec.args
|
|
282
|
+
if hasattr(hook, "__func__"):
|
|
283
|
+
# Skip 'self' arg.
|
|
284
|
+
argnames = argnames[1:]
|
|
285
|
+
# Pick kwargs that are wanted and available.
|
|
286
|
+
args = []
|
|
287
|
+
kwargs = {}
|
|
288
|
+
for n in argnames:
|
|
289
|
+
try:
|
|
290
|
+
kwargs[n] = avail_kwargs[n]
|
|
291
|
+
if argspec.varkw:
|
|
292
|
+
break
|
|
293
|
+
except KeyError:
|
|
294
|
+
# For compatibility, allow first argument to be 'old' with any
|
|
295
|
+
# name, as long as there's no other arg 'old'.
|
|
296
|
+
# We intentionally allow this even if the user specified
|
|
297
|
+
# **kwargs.
|
|
298
|
+
if not args and not kwargs and 'old' not in argnames:
|
|
299
|
+
args.append(old)
|
|
300
|
+
else:
|
|
301
|
+
# Rely on default being set. If a default isn't set, the
|
|
302
|
+
# user will get a TypeError.
|
|
303
|
+
pass
|
|
304
|
+
if argspec.varkw:
|
|
305
|
+
# Use all available kwargs.
|
|
306
|
+
kwargs = avail_kwargs
|
|
307
|
+
# Call hook.
|
|
308
|
+
result = hook(*args, **kwargs)
|
|
309
|
+
cache[cachekey] = result
|
|
310
|
+
return result
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _livepatch__module(old_mod, new_mod, modname, cache, visit_stack):
|
|
314
|
+
"""
|
|
315
|
+
Livepatch a module.
|
|
316
|
+
"""
|
|
317
|
+
result = livepatch(old_mod.__dict__, new_mod.__dict__,
|
|
318
|
+
modname=modname,
|
|
319
|
+
cache=cache, visit_stack=visit_stack)
|
|
320
|
+
assert result is old_mod.__dict__
|
|
321
|
+
return old_mod
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _livepatch__dict(old_dict, new_dict, modname, cache, visit_stack):
|
|
325
|
+
"""
|
|
326
|
+
Livepatch a dict.
|
|
327
|
+
"""
|
|
328
|
+
oldnames = set(old_dict)
|
|
329
|
+
newnames = set(new_dict)
|
|
330
|
+
# Add newly introduced names.
|
|
331
|
+
for name in newnames - oldnames:
|
|
332
|
+
old_dict[name] = new_dict[name]
|
|
333
|
+
# Delete names that are no longer current.
|
|
334
|
+
for name in oldnames - newnames:
|
|
335
|
+
del old_dict[name]
|
|
336
|
+
# Livepatch existing entries.
|
|
337
|
+
updated_names = sorted(oldnames & newnames, key=str)
|
|
338
|
+
for name in updated_names:
|
|
339
|
+
old = old_dict[name]
|
|
340
|
+
updated = livepatch(old, new_dict[name],
|
|
341
|
+
modname=modname,
|
|
342
|
+
cache=cache, visit_stack=visit_stack)
|
|
343
|
+
if updated is not old:
|
|
344
|
+
old_dict[name] = updated
|
|
345
|
+
return old_dict
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _livepatch__function(old_func, new_func, modname, cache, visit_stack):
|
|
349
|
+
"""
|
|
350
|
+
Livepatch a function.
|
|
351
|
+
"""
|
|
352
|
+
# If the name differs, then don't update the existing function - this
|
|
353
|
+
# is probably a reassigned function.
|
|
354
|
+
if old_func.__name__ != new_func.__name__:
|
|
355
|
+
return new_func
|
|
356
|
+
# Check if the function's closure is compatible. If not, then return the
|
|
357
|
+
# new function without livepatching. Note that cell closures can't be
|
|
358
|
+
# modified; we can only livepatch cell values.
|
|
359
|
+
old_closure = old_func.__closure__ or ()
|
|
360
|
+
new_closure = new_func.__closure__ or ()
|
|
361
|
+
if len(old_closure) != len(new_closure):
|
|
362
|
+
return new_func
|
|
363
|
+
if old_func.__code__.co_freevars != new_func.__code__.co_freevars:
|
|
364
|
+
return new_func
|
|
365
|
+
for oldcell, newcell in zip(old_closure, new_closure):
|
|
366
|
+
oldcellv = oldcell.cell_contents
|
|
367
|
+
newcellv = newcell.cell_contents
|
|
368
|
+
if type(oldcellv) != type(newcellv):
|
|
369
|
+
return new_func
|
|
370
|
+
if isinstance(oldcellv, (
|
|
371
|
+
types.FunctionType, types.MethodType, type, dict)):
|
|
372
|
+
# Updateable type. (Todo: make this configured globally.)
|
|
373
|
+
continue
|
|
374
|
+
try:
|
|
375
|
+
if oldcellv is newcellv or oldcellv == newcellv:
|
|
376
|
+
continue
|
|
377
|
+
except Exception:
|
|
378
|
+
pass
|
|
379
|
+
# Non-updateable and not the same as before.
|
|
380
|
+
return new_func
|
|
381
|
+
# Update function code, defaults, doc.
|
|
382
|
+
old_func.__code__ = new_func.__code__
|
|
383
|
+
old_func.__defaults__ = new_func.__defaults__
|
|
384
|
+
old_func.__doc__ = new_func.__doc__
|
|
385
|
+
# Update dict.
|
|
386
|
+
livepatch(old_func.__dict__, new_func.__dict__,
|
|
387
|
+
modname=modname, cache=cache, visit_stack=visit_stack)
|
|
388
|
+
# Update the __closure__. We can't set __closure__ because it's a
|
|
389
|
+
# read-only attribute; we can only livepatch its cells' values.
|
|
390
|
+
for oldcell, newcell in zip(old_closure, new_closure):
|
|
391
|
+
oldcellv = oldcell.cell_contents
|
|
392
|
+
newcellv = newcell.cell_contents
|
|
393
|
+
livepatch(oldcellv, newcellv,
|
|
394
|
+
modname=modname, cache=cache, visit_stack=visit_stack)
|
|
395
|
+
return old_func
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _livepatch__method(old_method, new_method, modname, cache, visit_stack):
|
|
399
|
+
"""
|
|
400
|
+
Livepatch a method.
|
|
401
|
+
"""
|
|
402
|
+
_livepatch__function(old_method.__func__, new_method.__func__,
|
|
403
|
+
modname=modname,
|
|
404
|
+
cache=cache, visit_stack=visit_stack)
|
|
405
|
+
return old_method
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _livepatch__setattr(oldobj, newobj, name, modname, cache, visit_stack):
|
|
409
|
+
"""
|
|
410
|
+
Livepatch something via setattr, i.e.::
|
|
411
|
+
|
|
412
|
+
oldobj.{name} = livepatch(oldobj.{name}, newobj.{name}, ...)
|
|
413
|
+
"""
|
|
414
|
+
newval = getattr(newobj, name)
|
|
415
|
+
assert type(newval) is not types.MemberDescriptorType
|
|
416
|
+
try:
|
|
417
|
+
oldval = getattr(oldobj, name)
|
|
418
|
+
except AttributeError:
|
|
419
|
+
# This shouldn't happen, but just ignore it.
|
|
420
|
+
setattr(oldobj, name, newval)
|
|
421
|
+
return
|
|
422
|
+
# If it's the same object, then skip. Note that if even if 'newval ==
|
|
423
|
+
# oldval', as long as they're not the same object instance, we still
|
|
424
|
+
# livepatch. We want mutable data structures get livepatched instead of
|
|
425
|
+
# replaced. Avoiding calling '==' also avoids the risk of user code
|
|
426
|
+
# having defined '==' to do something unexpected.
|
|
427
|
+
if newval is oldval:
|
|
428
|
+
return
|
|
429
|
+
# Livepatch the member object.
|
|
430
|
+
newval = livepatch(
|
|
431
|
+
oldval, newval, modname=modname, cache=cache, visit_stack=visit_stack)
|
|
432
|
+
# If the livepatch succeeded then we don't need to setattr. It should be
|
|
433
|
+
# a no-op but we avoid it just to minimize any chance of setattr causing
|
|
434
|
+
# problems in corner cases.
|
|
435
|
+
if newval is oldval:
|
|
436
|
+
return
|
|
437
|
+
# Livepatch failed, so we have to update the container with the new member
|
|
438
|
+
# value.
|
|
439
|
+
setattr(oldobj, name, newval)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _livepatch__class(oldclass, newclass, modname, cache, visit_stack):
|
|
443
|
+
"""
|
|
444
|
+
Livepatch a class.
|
|
445
|
+
|
|
446
|
+
This is similar to _livepatch__dict(oldclass.__dict__, newclass.__dict__).
|
|
447
|
+
However, we can't just operate on the dict, because class dictionaries are
|
|
448
|
+
special objects that don't allow setitem, even though we can setattr on
|
|
449
|
+
the class.
|
|
450
|
+
"""
|
|
451
|
+
# Collect the names to update.
|
|
452
|
+
olddict = oldclass.__dict__
|
|
453
|
+
newdict = newclass.__dict__
|
|
454
|
+
# Make sure slottiness hasn't changed -- i.e. if class was changed to have
|
|
455
|
+
# slots, or changed to not have slots, or if the slot names changed in any
|
|
456
|
+
# way, then we can't livepatch the class.
|
|
457
|
+
# Note that this is about whether instances of this class are affected by
|
|
458
|
+
# __slots__ or not. The class type itself will always use a __dict__.
|
|
459
|
+
if olddict.get("__slots__") != newdict.get("__slots__"):
|
|
460
|
+
return newclass
|
|
461
|
+
oldnames = set(olddict)
|
|
462
|
+
newnames = set(newdict)
|
|
463
|
+
for name in oldnames - newnames:
|
|
464
|
+
delattr(oldclass, name)
|
|
465
|
+
for name in newnames - oldnames:
|
|
466
|
+
setattr(oldclass, name, newdict[name])
|
|
467
|
+
oldclass.__bases__ = newclass.__bases__
|
|
468
|
+
names = oldnames & newnames
|
|
469
|
+
names.difference_update(olddict.get("__slots__", []))
|
|
470
|
+
names.discard("__slots__")
|
|
471
|
+
names.discard("__dict__")
|
|
472
|
+
# Python < 3.3 doesn't support modifying __doc__ on classes with
|
|
473
|
+
# non-custom metaclasses. Attempt to do it and ignore failures.
|
|
474
|
+
# http://bugs.python.org/issue12773
|
|
475
|
+
names.discard("__doc__")
|
|
476
|
+
try:
|
|
477
|
+
oldclass.__doc__ = newclass.__doc__
|
|
478
|
+
except AttributeError:
|
|
479
|
+
pass
|
|
480
|
+
# Loop over attributes to be updated.
|
|
481
|
+
for name in sorted(names):
|
|
482
|
+
_livepatch__setattr(
|
|
483
|
+
oldclass, newclass, name, modname, cache, visit_stack)
|
|
484
|
+
return oldclass
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _livepatch__object(oldobj, newobj, modname, cache, visit_stack):
|
|
488
|
+
"""
|
|
489
|
+
Livepatch a general object.
|
|
490
|
+
"""
|
|
491
|
+
# It's not obvious whether ``oldobj`` and ``newobj`` are actually supposed
|
|
492
|
+
# to represent the same object. For now, we take a middle ground of
|
|
493
|
+
# livepatching iff the class was also defined in the same module. In that
|
|
494
|
+
# case at least we know that the object was defined in this module and
|
|
495
|
+
# therefore more likely that we should livepatch.
|
|
496
|
+
if modname and _get_definition_module(type(oldobj)) != modname:
|
|
497
|
+
return newobj
|
|
498
|
+
if hasattr(type(oldobj), "__slots__"):
|
|
499
|
+
assert oldobj.__slots__ == newobj.__slots__
|
|
500
|
+
for name in newobj.__slots__:
|
|
501
|
+
hasold = hasattr(oldobj, name)
|
|
502
|
+
hasnew = hasattr(newobj, name)
|
|
503
|
+
if hasold and hasnew:
|
|
504
|
+
_livepatch__setattr(oldobj, newobj, name,
|
|
505
|
+
modname, cache, visit_stack)
|
|
506
|
+
elif hasold and not hasnew:
|
|
507
|
+
delattr(oldobj, name)
|
|
508
|
+
elif not hasold and hasnew:
|
|
509
|
+
setattr(oldobj, getattr(newobj, name))
|
|
510
|
+
elif not hasold and not hasnew:
|
|
511
|
+
pass
|
|
512
|
+
else:
|
|
513
|
+
raise AssertionError
|
|
514
|
+
return oldobj
|
|
515
|
+
elif type(getattr(oldobj, "__dict__", None)) is dict:
|
|
516
|
+
livepatch(
|
|
517
|
+
oldobj.__dict__, newobj.__dict__,
|
|
518
|
+
modname=modname, cache=cache, visit_stack=visit_stack)
|
|
519
|
+
return oldobj
|
|
520
|
+
else:
|
|
521
|
+
return newobj
|
|
522
|
+
|
|
523
|
+
_LIVEPATCH_DISPATCH_TABLE = {
|
|
524
|
+
object : _livepatch__object,
|
|
525
|
+
dict : _livepatch__dict,
|
|
526
|
+
type : _livepatch__class,
|
|
527
|
+
types.FunctionType: _livepatch__function,
|
|
528
|
+
types.MethodType : _livepatch__method,
|
|
529
|
+
types.ModuleType : _livepatch__module,
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _get_definition_module(obj):
|
|
534
|
+
"""
|
|
535
|
+
Get the name of the module that an object is defined in, or ``None`` if
|
|
536
|
+
unknown.
|
|
537
|
+
|
|
538
|
+
For classes and functions, this returns the ``__module__`` attribute.
|
|
539
|
+
|
|
540
|
+
For object instances, this returns ``None``, ignoring the ``__module__``
|
|
541
|
+
attribute. The reason is that the ``__module__`` attribute on an instance
|
|
542
|
+
just gives the module that the class was defined in, which is not
|
|
543
|
+
necessarily the module where the instance was constructed.
|
|
544
|
+
|
|
545
|
+
:rtype:
|
|
546
|
+
``str``
|
|
547
|
+
"""
|
|
548
|
+
if isinstance(obj, (type, types.FunctionType,
|
|
549
|
+
types.MethodType)):
|
|
550
|
+
return getattr(obj, "__module__", None)
|
|
551
|
+
else:
|
|
552
|
+
return None
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _format_age(t):
|
|
556
|
+
secs = time.time() - t
|
|
557
|
+
if secs > 120:
|
|
558
|
+
return "%dm%ds" %(secs//60, secs%60)
|
|
559
|
+
else:
|
|
560
|
+
return "%ds" %(secs,)
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _interpret_module(arg):
|
|
564
|
+
def mod_fn(module):
|
|
565
|
+
return getattr(module, "__file__", None)
|
|
566
|
+
|
|
567
|
+
if isinstance(arg, str):
|
|
568
|
+
try:
|
|
569
|
+
return sys.modules[arg]
|
|
570
|
+
except KeyError:
|
|
571
|
+
pass
|
|
572
|
+
if arg.startswith("/"):
|
|
573
|
+
fn = os.path.realpath(arg)
|
|
574
|
+
if fn.endswith(".pyc") or fn.endswith(".pyo"):
|
|
575
|
+
fn = fn[:-1]
|
|
576
|
+
if fn.endswith(".py"):
|
|
577
|
+
relevant_fns = set([fn, fn+"c", fn+"o"])
|
|
578
|
+
else:
|
|
579
|
+
relevant_fns = set([fn])
|
|
580
|
+
found_modules = [
|
|
581
|
+
m for _,m in sorted(sys.modules.items())
|
|
582
|
+
if os.path.realpath(mod_fn(m) or "/") in relevant_fns ]
|
|
583
|
+
if not found_modules:
|
|
584
|
+
raise UnknownModuleError(
|
|
585
|
+
"No loaded module uses path %s" % (fn,))
|
|
586
|
+
if len(found_modules) > 1:
|
|
587
|
+
raise UnknownModuleError(
|
|
588
|
+
"Multiple loaded modules use path %s: %r"
|
|
589
|
+
% (fn, found_modules))
|
|
590
|
+
return found_modules[0]
|
|
591
|
+
if arg.endswith(".py") and "/" not in arg:
|
|
592
|
+
name = arg[:-3]
|
|
593
|
+
relevant_bns = set([arg, arg+"c", arg+"o"])
|
|
594
|
+
found_modules = [
|
|
595
|
+
m for n,m in sorted(sys.modules.items())
|
|
596
|
+
if (n==name or
|
|
597
|
+
os.path.basename(mod_fn(m) or "/") in relevant_bns)]
|
|
598
|
+
if not found_modules:
|
|
599
|
+
raise UnknownModuleError(
|
|
600
|
+
"No loaded module named %s" % (name,))
|
|
601
|
+
if len(found_modules) > 1:
|
|
602
|
+
raise UnknownModuleError(
|
|
603
|
+
"Multiple loaded modules named %s: %r"
|
|
604
|
+
% (name, found_modules))
|
|
605
|
+
return found_modules[0]
|
|
606
|
+
raise UnknownModuleError(arg)
|
|
607
|
+
if isinstance(arg, types.ModuleType):
|
|
608
|
+
return arg
|
|
609
|
+
try:
|
|
610
|
+
# Allow fake modules.
|
|
611
|
+
if sys.modules[arg.__name__] is arg:
|
|
612
|
+
return arg
|
|
613
|
+
except Exception:
|
|
614
|
+
pass
|
|
615
|
+
raise TypeError("Expected module, module name, or filename; got %s"
|
|
616
|
+
% (type(arg).__name__))
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _xreload_module(module, filename, force=False):
|
|
620
|
+
"""
|
|
621
|
+
Reload a module in place, using livepatch.
|
|
622
|
+
|
|
623
|
+
:type module:
|
|
624
|
+
``ModuleType``
|
|
625
|
+
:param module:
|
|
626
|
+
Module to reload.
|
|
627
|
+
:param force:
|
|
628
|
+
Whether to reload even if the module has not been modified since the
|
|
629
|
+
previous load. If ``False``, then do nothing. If ``True``, then reload.
|
|
630
|
+
"""
|
|
631
|
+
import linecache
|
|
632
|
+
if not filename or not filename.endswith(".py"):
|
|
633
|
+
# If there's no *.py source file for this module, then fallback to
|
|
634
|
+
# built-in reload().
|
|
635
|
+
return reload_module(module)
|
|
636
|
+
# Compare mtime of the file with the load time of the module. If the file
|
|
637
|
+
# wasn't touched, we don't need to do anything.
|
|
638
|
+
try:
|
|
639
|
+
mtime = os.stat(filename).st_mtime
|
|
640
|
+
except OSError:
|
|
641
|
+
logger.info("Can't find %s", filename)
|
|
642
|
+
return None
|
|
643
|
+
if not force:
|
|
644
|
+
try:
|
|
645
|
+
old_loadtime = module.__loadtime__
|
|
646
|
+
except AttributeError:
|
|
647
|
+
# We only have a __loadtime__ attribute if we were the ones that
|
|
648
|
+
# loaded it. Otherwise, fall back to the process start time as a
|
|
649
|
+
# conservative bound.
|
|
650
|
+
old_loadtime = _PROCESS_START_TIME
|
|
651
|
+
if old_loadtime > mtime:
|
|
652
|
+
logger.debug(
|
|
653
|
+
"NOT reloading %s (file %s modified %s ago but loaded %s ago)",
|
|
654
|
+
module.__name__, filename, _format_age(mtime),
|
|
655
|
+
_format_age(old_loadtime))
|
|
656
|
+
return None
|
|
657
|
+
# Keep track of previously imported source. If the file's timestamp
|
|
658
|
+
# was touched, but the content unchanged, we can avoid reloading.
|
|
659
|
+
cached_lines = linecache.cache.get(filename, (None,None,None,None))[2]
|
|
660
|
+
else:
|
|
661
|
+
cached_lines = None
|
|
662
|
+
# Re-read source for module from disk, and update the linecache.
|
|
663
|
+
source = ''.join(linecache.updatecache(filename))
|
|
664
|
+
# Skip reload if the content didn't change.
|
|
665
|
+
if cached_lines is not None and source == ''.join(cached_lines):
|
|
666
|
+
logger.debug(
|
|
667
|
+
"NOT reloading %s (file %s touched %s ago but content unchanged)",
|
|
668
|
+
module.__name__, filename, _format_age(mtime))
|
|
669
|
+
return module
|
|
670
|
+
logger.info("Reloading %s (modified %s ago) from %s",
|
|
671
|
+
module.__name__, _format_age(mtime), filename)
|
|
672
|
+
# Compile into AST. We do this as a separate step from compiling to byte
|
|
673
|
+
# code so that we can get the module docstring.
|
|
674
|
+
astnode = compile(source, filename, "exec", ast.PyCF_ONLY_AST, 1)
|
|
675
|
+
# Get the new docstring.
|
|
676
|
+
try:
|
|
677
|
+
if sys.versin_info > (3,10):
|
|
678
|
+
doc = astnode.body[0].value.value
|
|
679
|
+
else:
|
|
680
|
+
doc = astnode.body[0].value.s
|
|
681
|
+
except (AttributeError, IndexError):
|
|
682
|
+
doc = None
|
|
683
|
+
# Compile into code.
|
|
684
|
+
code = compile(astnode, filename, "exec", 0, 1)
|
|
685
|
+
# Execute the code. We do so in a temporary namespace so that if this
|
|
686
|
+
# fails, nothing changes. It's important to set __name__ so that relative
|
|
687
|
+
# imports work correctly.
|
|
688
|
+
new_mod = types.ModuleType(module.__name__)
|
|
689
|
+
new_mod.__file__ = filename
|
|
690
|
+
new_mod.__doc__ = doc
|
|
691
|
+
if hasattr(module, "__path__"):
|
|
692
|
+
new_mod.__path__ = module.__path__
|
|
693
|
+
MISSING = object()
|
|
694
|
+
saved_mod = sys.modules.get(module.__name__, MISSING)
|
|
695
|
+
try:
|
|
696
|
+
# Temporarily put the temporary module in sys.modules, in case the
|
|
697
|
+
# code references sys.modules[__name__] for some reason. Normally on
|
|
698
|
+
# success, we will revert this what that was there before (which
|
|
699
|
+
# normally should be ``module``). If an error occurs, we'll also
|
|
700
|
+
# revert. If the user has defined a __livepatch__ hook at the module
|
|
701
|
+
# level, it's possible for result to not be the old module.
|
|
702
|
+
sys.modules[module.__name__] = new_mod
|
|
703
|
+
# *** Execute new code ***
|
|
704
|
+
exec(code, new_mod.__dict__)
|
|
705
|
+
# Normally ``module`` is of type ``ModuleType``. However, in some
|
|
706
|
+
# cases, the module might have done a "proxy module" trick where the
|
|
707
|
+
# module is replaced by a proxy object of some other type. Regardless
|
|
708
|
+
# of the actual type, we do the update as ``module`` were of type
|
|
709
|
+
# ``ModuleType``.
|
|
710
|
+
assume_type = types.ModuleType
|
|
711
|
+
# Livepatch the module.
|
|
712
|
+
result = livepatch(module, new_mod, module.__name__,
|
|
713
|
+
assume_type=assume_type)
|
|
714
|
+
sys.modules[module.__name__] = result
|
|
715
|
+
except:
|
|
716
|
+
# Either the module failed executing or the livepatch failed.
|
|
717
|
+
# Revert to previous state.
|
|
718
|
+
# Note that this isn't perfect because it's possible that the module
|
|
719
|
+
# modified some global state in other modules.
|
|
720
|
+
if saved_mod is MISSING:
|
|
721
|
+
del sys.modules[module.__name__]
|
|
722
|
+
else:
|
|
723
|
+
sys.modules[module.__name__] = saved_mod
|
|
724
|
+
raise
|
|
725
|
+
# Update the time we last loaded the module. We intentionally use mtime
|
|
726
|
+
# here instead of time.time(). If we are on NFS, it's possible for the
|
|
727
|
+
# filer's mtime and time.time() to not be synchronized. We will be
|
|
728
|
+
# comparing to mtime next time, so if we use only mtime, we'll be fine.
|
|
729
|
+
module.__loadtime__ = mtime
|
|
730
|
+
return module
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _get_module_py_file(module):
|
|
734
|
+
filename = getattr(module, "__file__", None)
|
|
735
|
+
if not filename:
|
|
736
|
+
return None
|
|
737
|
+
filename = re.sub("[.]py[co]$", ".py", filename)
|
|
738
|
+
return filename
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def xreload(*args):
|
|
742
|
+
"""
|
|
743
|
+
Reload module(s).
|
|
744
|
+
|
|
745
|
+
This function is more useful than the built-in reload(). xreload() uses a
|
|
746
|
+
"live patch" approach that modifies existing functions, classes, and
|
|
747
|
+
objects in-place.
|
|
748
|
+
|
|
749
|
+
This addresses cases where one module imported functions from another
|
|
750
|
+
module.
|
|
751
|
+
|
|
752
|
+
For example, suppose m1.py contains::
|
|
753
|
+
|
|
754
|
+
from m2 import foo
|
|
755
|
+
def print_foo():
|
|
756
|
+
return foo()
|
|
757
|
+
|
|
758
|
+
and m2.py contains::
|
|
759
|
+
|
|
760
|
+
def foo():
|
|
761
|
+
return 42
|
|
762
|
+
|
|
763
|
+
If you edit m2.py and modify ``foo``, then reload(m2) on its own would not
|
|
764
|
+
do what you want. The built-in reload affects the module being reloaded,
|
|
765
|
+
but references to the old module remain. On the other hand, xreload()
|
|
766
|
+
patches the existing m2.foo, so that live references to it are updated.
|
|
767
|
+
|
|
768
|
+
:type args:
|
|
769
|
+
``str`` s and/or ``ModuleType`` s
|
|
770
|
+
:param args:
|
|
771
|
+
Module(s) to reload. If no argument is specified, then reload all
|
|
772
|
+
recently modified modules.
|
|
773
|
+
"""
|
|
774
|
+
if not args:
|
|
775
|
+
for name, module in sorted(sys.modules.items()):
|
|
776
|
+
if name == "__main__":
|
|
777
|
+
continue
|
|
778
|
+
filename = _get_module_py_file(module)
|
|
779
|
+
if not filename:
|
|
780
|
+
continue
|
|
781
|
+
_xreload_module(module, filename)
|
|
782
|
+
return
|
|
783
|
+
# Treat xreload(list_of_module) like xreload(*list_of_modules). We
|
|
784
|
+
# intentionally do this after the above check so that xreload([]) does
|
|
785
|
+
# nothing.
|
|
786
|
+
if len(args) == 1 and isinstance(args[0], (tuple, list)):
|
|
787
|
+
args = args[0]
|
|
788
|
+
for arg in args:
|
|
789
|
+
module = _interpret_module(arg)
|
|
790
|
+
# Get the *.py filename for this module.
|
|
791
|
+
filename = _get_module_py_file(module)
|
|
792
|
+
# Reload the module.
|
|
793
|
+
_xreload_module(module, filename)
|