nlpertools 1.0.5__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nlpertools/__init__.py +24 -20
- nlpertools/algo/ac.py +18 -0
- nlpertools/algo/bit_ops.py +28 -0
- nlpertools/algo/kmp.py +94 -55
- nlpertools/algo/num_ops.py +12 -0
- nlpertools/algo/template.py +116 -0
- nlpertools/algo/union.py +13 -0
- nlpertools/data_client.py +387 -257
- nlpertools/data_structure/base_structure.py +109 -13
- nlpertools/dataprocess.py +611 -3
- nlpertools/default_db_config.yml +41 -0
- nlpertools/io/__init__.py +3 -3
- nlpertools/io/dir.py +54 -36
- nlpertools/io/file.py +277 -222
- nlpertools/ml.py +483 -460
- nlpertools/monitor/__init__.py +0 -0
- nlpertools/monitor/gpu.py +18 -0
- nlpertools/monitor/memory.py +24 -0
- nlpertools/movie.py +36 -0
- nlpertools/nlpertools_config.yml +1 -0
- nlpertools/{openApi.py → open_api.py} +65 -65
- nlpertools/other.py +364 -249
- nlpertools/pic.py +288 -0
- nlpertools/plugin.py +43 -43
- nlpertools/reminder.py +98 -87
- nlpertools/utils/__init__.py +3 -3
- nlpertools/utils/lazy.py +727 -0
- nlpertools/utils/log_util.py +20 -0
- nlpertools/utils/package.py +89 -76
- nlpertools/utils/package_v1.py +94 -0
- nlpertools/utils/package_v2.py +117 -0
- nlpertools/utils_for_nlpertools.py +93 -93
- nlpertools/vector_index_demo.py +108 -0
- nlpertools/wrapper.py +161 -96
- {nlpertools-1.0.5.dist-info → nlpertools-1.0.6.dev0.dist-info}/LICENSE +200 -200
- nlpertools-1.0.6.dev0.dist-info/METADATA +111 -0
- nlpertools-1.0.6.dev0.dist-info/RECORD +43 -0
- {nlpertools-1.0.5.dist-info → nlpertools-1.0.6.dev0.dist-info}/WHEEL +1 -1
- nlpertools-1.0.6.dev0.dist-info/top_level.txt +2 -0
- nlpertools_helper/__init__.py +10 -0
- nlpertools-1.0.5.dist-info/METADATA +0 -85
- nlpertools-1.0.5.dist-info/RECORD +0 -25
- nlpertools-1.0.5.dist-info/top_level.txt +0 -1
nlpertools/utils/lazy.py
ADDED
@@ -0,0 +1,727 @@
|
|
1
|
+
"""
|
2
|
+
Write at first
|
3
|
+
|
4
|
+
This function comes from https://github.com/mnmelo/lazy_import
|
5
|
+
I can't install it property, so i add it to my project
|
6
|
+
|
7
|
+
# TODO failed in torch
|
8
|
+
"""
|
9
|
+
# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*-
|
10
|
+
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
|
11
|
+
#
|
12
|
+
# lazy_import --- https://github.com/mnmelo/lazy_import
|
13
|
+
# Copyright (C) 2017-2018 Manuel Nuno Melo
|
14
|
+
#
|
15
|
+
# This file is part of lazy_import.
|
16
|
+
#
|
17
|
+
# lazy_import is free software: you can redistribute it and/or modify
|
18
|
+
# it under the terms of the GNU General Public License as published by
|
19
|
+
# the Free Software Foundation, either version 3 of the License, or
|
20
|
+
# (at your option) any later version.
|
21
|
+
#
|
22
|
+
# lazy_import is distributed in the hope that it will be useful,
|
23
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
24
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
25
|
+
# GNU General Public License for more details.
|
26
|
+
#
|
27
|
+
# You should have received a copy of the GNU General Public License
|
28
|
+
# along with lazy_import. If not, see <http://www.gnu.org/licenses/>.
|
29
|
+
#
|
30
|
+
# lazy_import was based on code from the importing module from the PEAK
|
31
|
+
# package (see <http://peak.telecommunity.com/DevCenter/Importing>). The PEAK
|
32
|
+
# package is released under the following license, reproduced here:
|
33
|
+
#
|
34
|
+
# Copyright (C) 1996-2004 by Phillip J. Eby and Tyler C. Sarna.
|
35
|
+
# All rights reserved. This software may be used under the same terms
|
36
|
+
# as Zope or Python. THERE ARE ABSOLUTELY NO WARRANTIES OF ANY KIND.
|
37
|
+
# Code quality varies between modules, from "beta" to "experimental
|
38
|
+
# pre-alpha". :)
|
39
|
+
#
|
40
|
+
# Code pertaining to lazy loading from PEAK importing was included in
|
41
|
+
# lazy_import, modified in a number of ways. These are detailed in the
|
42
|
+
# CHANGELOG file of lazy_import. Changes mainly involved Python 3
|
43
|
+
# compatibility, extension to allow customizable behavior, and added
|
44
|
+
# functionality (lazy importing of callable objects).
|
45
|
+
#
|
46
|
+
|
47
|
+
"""
|
48
|
+
Lazy module loading
|
49
|
+
===================
|
50
|
+
|
51
|
+
Functions and classes for lazy module loading that also delay import errors.
|
52
|
+
Heavily borrowed from the `importing`_ module.
|
53
|
+
|
54
|
+
.. _`importing`: http://peak.telecommunity.com/DevCenter/Importing
|
55
|
+
|
56
|
+
Files and directories
|
57
|
+
---------------------
|
58
|
+
|
59
|
+
.. autofunction:: module
|
60
|
+
.. autofunction:: callable
|
61
|
+
|
62
|
+
"""
|
63
|
+
|
64
|
+
__all__ = ['lazy_module', 'lazy_callable', 'lazy_function', 'lazy_class',
|
65
|
+
'LazyModule', 'LazyCallable', 'module_basename', '_MSG',
|
66
|
+
'_MSG_CALLABLE']
|
67
|
+
|
68
|
+
from types import ModuleType
|
69
|
+
import sys
|
70
|
+
|
71
|
+
try:
|
72
|
+
from importlib._bootstrap import _ImportLockContext
|
73
|
+
except ImportError:
|
74
|
+
# Python 2 doesn't have the context manager. Roll it ourselves (copied from
|
75
|
+
# Python 3's importlib/_bootstrap.py)
|
76
|
+
import imp
|
77
|
+
|
78
|
+
|
79
|
+
class _ImportLockContext:
|
80
|
+
"""Context manager for the import lock."""
|
81
|
+
|
82
|
+
def __enter__(self):
|
83
|
+
imp.acquire_lock()
|
84
|
+
|
85
|
+
def __exit__(self, exc_type, exc_value, exc_traceback):
|
86
|
+
imp.release_lock()
|
87
|
+
|
88
|
+
# Adding a __spec__ doesn't really help. I'll leave the code here in case
|
89
|
+
# future python implementations start relying on it.
|
90
|
+
# try:
|
91
|
+
# from importlib.machinery import ModuleSpec
|
92
|
+
# except ImportError:
|
93
|
+
# ModuleSpec = None
|
94
|
+
|
95
|
+
import six
|
96
|
+
from six import raise_from
|
97
|
+
from six.moves import reload_module
|
98
|
+
# It is sometime useful to have access to the version number of a library.
|
99
|
+
# This is usually done through the __version__ special attribute.
|
100
|
+
# To make sure the version number is consistent between setup.py and the
|
101
|
+
# library, we read the version number from the file called VERSION that stays
|
102
|
+
# in the module directory.
|
103
|
+
import os
|
104
|
+
|
105
|
+
VERSION_FILE = os.path.join(os.path.dirname(__file__), 'VERSION')
|
106
|
+
__version__ = "1"
|
107
|
+
|
108
|
+
# Logging
|
109
|
+
import logging
|
110
|
+
|
111
|
+
# adding a TRACE level for stack debugging
|
112
|
+
_LAZY_TRACE = 1
|
113
|
+
logging.addLevelName(1, "LAZY_TRACE")
|
114
|
+
logging.basicConfig(level=logging.WARNING)
|
115
|
+
|
116
|
+
|
117
|
+
# Logs a formatted stack (takes no message or args/kwargs)
|
118
|
+
def _lazy_trace(self):
|
119
|
+
if self.isEnabledFor(_LAZY_TRACE):
|
120
|
+
import traceback
|
121
|
+
self._log(_LAZY_TRACE, " ### STACK TRACE ###", ())
|
122
|
+
for line in traceback.format_stack(sys._getframe(2)):
|
123
|
+
for subline in line.split("\n"):
|
124
|
+
self._log(_LAZY_TRACE, subline.rstrip(), ())
|
125
|
+
|
126
|
+
|
127
|
+
logging.Logger.lazy_trace = _lazy_trace
|
128
|
+
logger = logging.getLogger(__name__)
|
129
|
+
|
130
|
+
|
131
|
+
################################
|
132
|
+
# Module/function registration #
|
133
|
+
################################
|
134
|
+
|
135
|
+
#### Lazy classes ####
|
136
|
+
|
137
|
+
class LazyModule(ModuleType):
|
138
|
+
"""Class for lazily-loaded modules that triggers proper loading on access.
|
139
|
+
|
140
|
+
Instantiation should be made from a subclass of :class:`LazyModule`, with
|
141
|
+
one subclass per instantiated module. Regular attribute set/access can then
|
142
|
+
be recovered by setting the subclass's :meth:`__getattribute__` and
|
143
|
+
:meth:`__setattribute__` to those of :class:`types.ModuleType`.
|
144
|
+
"""
|
145
|
+
|
146
|
+
# peak.util.imports sets __slots__ to (), but it seems pointless because
|
147
|
+
# the base ModuleType doesn't itself set __slots__.
|
148
|
+
def __getattribute__(self, attr):
|
149
|
+
logger.debug("Getting attr {} of LazyModule instance of {}"
|
150
|
+
.format(attr, super(LazyModule, self)
|
151
|
+
.__getattribute__("__name__")))
|
152
|
+
logger.lazy_trace()
|
153
|
+
# IPython tries to be too clever and constantly inspects, asking for
|
154
|
+
# modules' attrs, which causes premature module loading and unesthetic
|
155
|
+
# internal errors if the lazily-loaded module doesn't exist.
|
156
|
+
if (run_from_ipython()
|
157
|
+
and (attr.startswith(("__", "_ipython"))
|
158
|
+
or attr == "_repr_mimebundle_")
|
159
|
+
and module_basename(_caller_name()) in ('inspect', 'IPython')):
|
160
|
+
logger.debug("Ignoring request for {}, deemed from IPython's "
|
161
|
+
"inspection.".format(super(LazyModule, self)
|
162
|
+
.__getattribute__("__name__"), attr))
|
163
|
+
raise AttributeError
|
164
|
+
if not attr in ('__name__', '__class__', '__spec__'):
|
165
|
+
# __name__ and __class__ yield their values from the LazyModule;
|
166
|
+
# __spec__ causes an AttributeError. Maybe in the future it will be
|
167
|
+
# necessary to return an actual ModuleSpec object, but it works as
|
168
|
+
# it is without that now.
|
169
|
+
|
170
|
+
# If it's an already-loaded submodule, we return it without
|
171
|
+
# triggering a full loading
|
172
|
+
try:
|
173
|
+
return sys.modules[self.__name__ + "." + attr]
|
174
|
+
except KeyError:
|
175
|
+
pass
|
176
|
+
# Check if it's one of the lazy callables
|
177
|
+
try:
|
178
|
+
_callable = type(self)._lazy_import_callables[attr]
|
179
|
+
logger.debug("Returning lazy-callable '{}'.".format(attr))
|
180
|
+
return _callable
|
181
|
+
except (AttributeError, KeyError) as err:
|
182
|
+
logger.debug("Proceeding to load module {}, "
|
183
|
+
"from requested value {}"
|
184
|
+
.format(super(LazyModule, self)
|
185
|
+
.__getattribute__("__name__"), attr))
|
186
|
+
_load_module(self)
|
187
|
+
logger.debug("Returning value '{}'.".format(super(LazyModule, self)
|
188
|
+
.__getattribute__(attr)))
|
189
|
+
return super(LazyModule, self).__getattribute__(attr)
|
190
|
+
|
191
|
+
def __setattr__(self, attr, value):
|
192
|
+
logger.debug("Setting attr {} to value {}, in LazyModule instance "
|
193
|
+
"of {}".format(attr, value, super(LazyModule, self)
|
194
|
+
.__getattribute__("__name__")))
|
195
|
+
_load_module(self)
|
196
|
+
return super(LazyModule, self).__setattr__(attr, value)
|
197
|
+
|
198
|
+
|
199
|
+
class LazyCallable(object):
|
200
|
+
"""Class for lazily-loaded callables that triggers module loading on access
|
201
|
+
|
202
|
+
"""
|
203
|
+
|
204
|
+
def __init__(self, *args):
|
205
|
+
if len(args) != 2:
|
206
|
+
# Maybe the user tried to base a class off this lazy callable?
|
207
|
+
try:
|
208
|
+
logger.debug("Got wrong number of args when init'ing "
|
209
|
+
"LazyCallable. args is '{}'".format(args))
|
210
|
+
base = args[1][0]
|
211
|
+
if isinstance(base, LazyCallable) and len(args) == 3:
|
212
|
+
raise NotImplementedError("It seems you are trying to use "
|
213
|
+
"a lazy callable as a class "
|
214
|
+
"base. This is not supported.")
|
215
|
+
except (IndexError, TypeError):
|
216
|
+
raise_from(TypeError("LazyCallable takes exactly 2 arguments: "
|
217
|
+
"a module/lazy module object and the name of "
|
218
|
+
"a callable to be lazily loaded."), None)
|
219
|
+
self.module, self.cname = args
|
220
|
+
self.modclass = type(self.module)
|
221
|
+
self.callable = None
|
222
|
+
# Need to save these, since the module-loading gets rid of them
|
223
|
+
self.error_msgs = self.modclass._lazy_import_error_msgs
|
224
|
+
self.error_strings = self.modclass._lazy_import_error_strings
|
225
|
+
|
226
|
+
def __call__(self, *args, **kwargs):
|
227
|
+
# No need to go through all the reloading more than once.
|
228
|
+
if self.callable:
|
229
|
+
return self.callable(*args, **kwargs)
|
230
|
+
try:
|
231
|
+
del self.modclass._lazy_import_callables[self.cname]
|
232
|
+
except (AttributeError, KeyError):
|
233
|
+
pass
|
234
|
+
try:
|
235
|
+
self.callable = getattr(self.module, self.cname)
|
236
|
+
except AttributeError:
|
237
|
+
msg = self.error_msgs['msg_callable']
|
238
|
+
raise_from(AttributeError(
|
239
|
+
msg.format(callable=self.cname, **self.error_strings)), None)
|
240
|
+
except ImportError as err:
|
241
|
+
# Import failed. We reset the dict and re-raise the ImportError.
|
242
|
+
try:
|
243
|
+
self.modclass._lazy_import_callables[self.cname] = self
|
244
|
+
except AttributeError:
|
245
|
+
self.modclass._lazy_import_callables = {self.cname: self}
|
246
|
+
raise_from(err, None)
|
247
|
+
else:
|
248
|
+
return self.callable(*args, **kwargs)
|
249
|
+
|
250
|
+
|
251
|
+
### Functions ###
|
252
|
+
|
253
|
+
def lazy_module(modname, error_strings=None, lazy_mod_class=LazyModule,
|
254
|
+
level='leaf'):
|
255
|
+
"""Function allowing lazy importing of a module into the namespace.
|
256
|
+
|
257
|
+
A lazy module object is created, registered in `sys.modules`, and
|
258
|
+
returned. This is a hollow module; actual loading, and `ImportErrors` if
|
259
|
+
not found, are delayed until an attempt is made to access attributes of the
|
260
|
+
lazy module.
|
261
|
+
|
262
|
+
A handy application is to use :func:`lazy_module` early in your own code
|
263
|
+
(say, in `__init__.py`) to register all modulenames you want to be lazy.
|
264
|
+
Because of registration in `sys.modules` later invocations of
|
265
|
+
`import modulename` will also return the lazy object. This means that after
|
266
|
+
initial registration the rest of your code can use regular pyhon import
|
267
|
+
statements and retain the lazyness of the modules.
|
268
|
+
|
269
|
+
Parameters
|
270
|
+
----------
|
271
|
+
modname : str
|
272
|
+
The module to import.
|
273
|
+
error_strings : dict, optional
|
274
|
+
A dictionary of strings to use when module-loading fails. Key 'msg'
|
275
|
+
sets the message to use (defaults to :attr:`lazy_import._MSG`). The
|
276
|
+
message is formatted using the remaining dictionary keys. The default
|
277
|
+
message informs the user of which module is missing (key 'module'),
|
278
|
+
what code loaded the module as lazy (key 'caller'), and which package
|
279
|
+
should be installed to solve the dependency (key 'install_name').
|
280
|
+
None of the keys is mandatory and all are given smart names by default.
|
281
|
+
lazy_mod_class: type, optional
|
282
|
+
Which class to use when instantiating the lazy module, to allow
|
283
|
+
deep customization. The default is :class:`LazyModule` and custom
|
284
|
+
alternatives **must** be a subclass thereof.
|
285
|
+
level : str, optional
|
286
|
+
Which submodule reference to return. Either a reference to the 'leaf'
|
287
|
+
module (the default) or to the 'base' module. This is useful if you'll
|
288
|
+
be using the module functionality in the same place you're calling
|
289
|
+
:func:`lazy_module` from, since then you don't need to run `import`
|
290
|
+
again. Setting *level* does not affect which names/modules get
|
291
|
+
registered in `sys.modules`.
|
292
|
+
For *level* set to 'base' and *modulename* 'aaa.bbb.ccc'::
|
293
|
+
|
294
|
+
aaa = lazy_import.lazy_module("aaa.bbb.ccc", level='base')
|
295
|
+
# 'aaa' becomes defined in the current namespace, with
|
296
|
+
# (sub)attributes 'aaa.bbb' and 'aaa.bbb.ccc'.
|
297
|
+
# It's the lazy equivalent to:
|
298
|
+
import aaa.bbb.ccc
|
299
|
+
|
300
|
+
For *level* set to 'leaf'::
|
301
|
+
|
302
|
+
ccc = lazy_import.lazy_module("aaa.bbb.ccc", level='leaf')
|
303
|
+
# Only 'ccc' becomes set in the current namespace.
|
304
|
+
# Lazy equivalent to:
|
305
|
+
from aaa.bbb import ccc
|
306
|
+
|
307
|
+
Returns
|
308
|
+
-------
|
309
|
+
module
|
310
|
+
The module specified by *modname*, or its base, depending on *level*.
|
311
|
+
The module isn't immediately imported. Instead, an instance of
|
312
|
+
*lazy_mod_class* is returned. Upon access to any of its attributes, the
|
313
|
+
module is finally loaded.
|
314
|
+
|
315
|
+
Examples
|
316
|
+
--------
|
317
|
+
>>> import lazy_import, sys
|
318
|
+
>>> np = lazy_import.lazy_module("numpy")
|
319
|
+
>>> np
|
320
|
+
Lazily-loaded module numpy
|
321
|
+
>>> np is sys.modules['numpy']
|
322
|
+
True
|
323
|
+
>>> np.pi # This causes the full loading of the module ...
|
324
|
+
3.141592653589793
|
325
|
+
>>> np # ... and the module is changed in place.
|
326
|
+
<module 'numpy' from '/usr/local/lib/python/site-packages/numpy/__init__.py'>
|
327
|
+
|
328
|
+
>>> import lazy_import, sys
|
329
|
+
>>> # The following succeeds even when asking for a module that's not available
|
330
|
+
>>> missing = lazy_import.lazy_module("missing_module")
|
331
|
+
>>> missing
|
332
|
+
Lazily-loaded module missing_module
|
333
|
+
>>> missing is sys.modules['missing_module']
|
334
|
+
True
|
335
|
+
>>> missing.some_attr # This causes the full loading of the module, which now fails.
|
336
|
+
ImportError: __main__ attempted to use a functionality that requires module missing_module, but it couldn't be loaded. Please install missing_module and retry.
|
337
|
+
|
338
|
+
See Also
|
339
|
+
--------
|
340
|
+
:func:`lazy_callable`
|
341
|
+
:class:`LazyModule`
|
342
|
+
|
343
|
+
"""
|
344
|
+
if error_strings is None:
|
345
|
+
error_strings = {}
|
346
|
+
_set_default_errornames(modname, error_strings)
|
347
|
+
|
348
|
+
mod = _lazy_module(modname, error_strings, lazy_mod_class)
|
349
|
+
if level == 'base':
|
350
|
+
return sys.modules[module_basename(modname)]
|
351
|
+
elif level == 'leaf':
|
352
|
+
return mod
|
353
|
+
else:
|
354
|
+
raise ValueError("Parameter 'level' must be one of ('base', 'leaf')")
|
355
|
+
|
356
|
+
|
357
|
+
def _lazy_module(modname, error_strings, lazy_mod_class):
|
358
|
+
with _ImportLockContext():
|
359
|
+
fullmodname = modname
|
360
|
+
fullsubmodname = None
|
361
|
+
# ensure parent module/package is in sys.modules
|
362
|
+
# and parent.modname=module, as soon as the parent is imported
|
363
|
+
while modname:
|
364
|
+
try:
|
365
|
+
mod = sys.modules[modname]
|
366
|
+
# We reached a (base) module that's already loaded. Let's stop
|
367
|
+
# the cycle. Can't use 'break' because we still want to go
|
368
|
+
# through the fullsubmodname check below.
|
369
|
+
modname = ''
|
370
|
+
except KeyError:
|
371
|
+
err_s = error_strings.copy()
|
372
|
+
err_s.setdefault('module', modname)
|
373
|
+
|
374
|
+
class _LazyModule(lazy_mod_class):
|
375
|
+
_lazy_import_error_msgs = {'msg': err_s.pop('msg')}
|
376
|
+
try:
|
377
|
+
_lazy_import_error_msgs['msg_callable'] = \
|
378
|
+
err_s.pop('msg_callable')
|
379
|
+
except KeyError:
|
380
|
+
pass
|
381
|
+
_lazy_import_error_strings = err_s
|
382
|
+
_lazy_import_callables = {}
|
383
|
+
_lazy_import_submodules = {}
|
384
|
+
|
385
|
+
def __repr__(self):
|
386
|
+
return "Lazily-loaded module {}".format(self.__name__)
|
387
|
+
|
388
|
+
# A bit of cosmetic, to make AttributeErrors read more natural
|
389
|
+
_LazyModule.__name__ = 'module'
|
390
|
+
# Actual module instantiation
|
391
|
+
mod = sys.modules[modname] = _LazyModule(modname)
|
392
|
+
# No need for __spec__. Maybe in the future.
|
393
|
+
# if ModuleSpec:
|
394
|
+
# ModuleType.__setattr__(mod, '__spec__',
|
395
|
+
# ModuleSpec(modname, None))
|
396
|
+
if fullsubmodname:
|
397
|
+
submod = sys.modules[fullsubmodname]
|
398
|
+
ModuleType.__setattr__(mod, submodname, submod)
|
399
|
+
_LazyModule._lazy_import_submodules[submodname] = submod
|
400
|
+
fullsubmodname = modname
|
401
|
+
modname, _, submodname = modname.rpartition('.')
|
402
|
+
return sys.modules[fullmodname]
|
403
|
+
|
404
|
+
|
405
|
+
def lazy_callable(modname, *names, **kwargs):
|
406
|
+
"""Performs lazy importing of one or more callables.
|
407
|
+
|
408
|
+
:func:`lazy_callable` creates functions that are thin wrappers that pass
|
409
|
+
any and all arguments straight to the target module's callables. These can
|
410
|
+
be functions or classes. The full loading of that module is only actually
|
411
|
+
triggered when the returned lazy function itself is called. This lazy
|
412
|
+
import of the target module uses the same mechanism as
|
413
|
+
:func:`lazy_module`.
|
414
|
+
|
415
|
+
If, however, the target module has already been fully imported prior
|
416
|
+
to invocation of :func:`lazy_callable`, then the target callables
|
417
|
+
themselves are returned and no lazy imports are made.
|
418
|
+
|
419
|
+
:func:`lazy_function` and :func:`lazy_function` are aliases of
|
420
|
+
:func:`lazy_callable`.
|
421
|
+
|
422
|
+
Parameters
|
423
|
+
----------
|
424
|
+
modname : str
|
425
|
+
The base module from where to import the callable(s) in *names*,
|
426
|
+
or a full 'module_name.callable_name' string.
|
427
|
+
names : str (optional)
|
428
|
+
The callable name(s) to import from the module specified by *modname*.
|
429
|
+
If left empty, *modname* is assumed to also include the callable name
|
430
|
+
to import.
|
431
|
+
error_strings : dict, optional
|
432
|
+
A dictionary of strings to use when reporting loading errors (either a
|
433
|
+
missing module, or a missing callable name in the loaded module).
|
434
|
+
*error_string* follows the same usage as described under
|
435
|
+
:func:`lazy_module`, with the exceptions that 1) a further key,
|
436
|
+
'msg_callable', can be supplied to be used as the error when a module
|
437
|
+
is successfully loaded but the target callable can't be found therein
|
438
|
+
(defaulting to :attr:`lazy_import._MSG_CALLABLE`); 2) a key 'callable'
|
439
|
+
is always added with the callable name being loaded.
|
440
|
+
lazy_mod_class : type, optional
|
441
|
+
See definition under :func:`lazy_module`.
|
442
|
+
lazy_call_class : type, optional
|
443
|
+
Analogously to *lazy_mod_class*, allows setting a custom class to
|
444
|
+
handle lazy callables, other than the default :class:`LazyCallable`.
|
445
|
+
|
446
|
+
Returns
|
447
|
+
-------
|
448
|
+
wrapper function or tuple of wrapper functions
|
449
|
+
If *names* is passed, returns a tuple of wrapper functions, one for
|
450
|
+
each element in *names*.
|
451
|
+
If only *modname* is passed it is assumed to be a full
|
452
|
+
'module_name.callable_name' string, in which case the wrapper for the
|
453
|
+
imported callable is returned directly, and not in a tuple.
|
454
|
+
|
455
|
+
Notes
|
456
|
+
-----
|
457
|
+
Unlike :func:`lazy_module`, which returns a lazy module that eventually
|
458
|
+
mutates into the fully-functional version, :func:`lazy_callable` only
|
459
|
+
returns thin wrappers that never change. This means that the returned
|
460
|
+
wrapper object never truly becomes the one under the module's namespace,
|
461
|
+
even after successful loading of the module in *modname*. This is fine for
|
462
|
+
most practical use cases, but may break code that relies on the usage of
|
463
|
+
the returned objects oter than calling them. One such example is the lazy
|
464
|
+
import of a class: it's fine to use the returned wrapper to instantiate an
|
465
|
+
object, but it can't be used, for instance, to subclass from.
|
466
|
+
|
467
|
+
Examples
|
468
|
+
--------
|
469
|
+
>>> import lazy_import, sys
|
470
|
+
>>> fn = lazy_import.lazy_callable("numpy.arange")
|
471
|
+
>>> sys.modules['numpy']
|
472
|
+
Lazily-loaded module numpy
|
473
|
+
>>> fn(10)
|
474
|
+
array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
|
475
|
+
>>> sys.modules['numpy']
|
476
|
+
<module 'numpy' from '/usr/local/lib/python3.5/site-packages/numpy/__init__.py'>
|
477
|
+
|
478
|
+
>>> import lazy_import, sys
|
479
|
+
>>> cl = lazy_import.lazy_callable("numpy.ndarray") # a class
|
480
|
+
>>> obj = cl([1, 2]) # This works OK (and also triggers the loading of numpy)
|
481
|
+
>>> class MySubclass(cl): # This fails because cls is just a wrapper,
|
482
|
+
>>> pass # not an actual class.
|
483
|
+
|
484
|
+
See Also
|
485
|
+
--------
|
486
|
+
:func:`lazy_module`
|
487
|
+
:class:`LazyCallable`
|
488
|
+
:class:`LazyModule`
|
489
|
+
|
490
|
+
"""
|
491
|
+
if not names:
|
492
|
+
modname, _, name = modname.rpartition(".")
|
493
|
+
lazy_mod_class = _setdef(kwargs, 'lazy_mod_class', LazyModule)
|
494
|
+
lazy_call_class = _setdef(kwargs, 'lazy_call_class', LazyCallable)
|
495
|
+
error_strings = _setdef(kwargs, 'error_strings', {})
|
496
|
+
_set_default_errornames(modname, error_strings, call=True)
|
497
|
+
|
498
|
+
if not names:
|
499
|
+
# We allow passing a single string as 'modname.callable_name',
|
500
|
+
# in which case the wrapper is returned directly and not as a list.
|
501
|
+
return _lazy_callable(modname, name, error_strings.copy(),
|
502
|
+
lazy_mod_class, lazy_call_class)
|
503
|
+
return tuple(_lazy_callable(modname, cname, error_strings.copy(),
|
504
|
+
lazy_mod_class, lazy_call_class) for cname in names)
|
505
|
+
|
506
|
+
|
507
|
+
lazy_function = lazy_class = lazy_callable
|
508
|
+
|
509
|
+
|
510
|
+
def _lazy_callable(modname, cname, error_strings,
|
511
|
+
lazy_mod_class, lazy_call_class):
|
512
|
+
# We could do most of this in the LazyCallable __init__, but here we can
|
513
|
+
# pre-check whether to actually be lazy or not.
|
514
|
+
module = _lazy_module(modname, error_strings, lazy_mod_class)
|
515
|
+
modclass = type(module)
|
516
|
+
if (issubclass(modclass, LazyModule) and
|
517
|
+
hasattr(modclass, '_lazy_import_callables')):
|
518
|
+
modclass._lazy_import_callables.setdefault(
|
519
|
+
cname, lazy_call_class(module, cname))
|
520
|
+
return getattr(module, cname)
|
521
|
+
|
522
|
+
|
523
|
+
#######################
|
524
|
+
# Real module loading #
|
525
|
+
#######################
|
526
|
+
|
527
|
+
def _load_module(module):
|
528
|
+
"""Ensures that a module, and its parents, are properly loaded
|
529
|
+
|
530
|
+
"""
|
531
|
+
modclass = type(module)
|
532
|
+
# We only take care of our own LazyModule instances
|
533
|
+
if not issubclass(modclass, LazyModule):
|
534
|
+
raise TypeError("Passed module is not a LazyModule instance.")
|
535
|
+
with _ImportLockContext():
|
536
|
+
parent, _, modname = module.__name__.rpartition('.')
|
537
|
+
logger.debug("loading module {}".format(modname))
|
538
|
+
# We first identify whether this is a loadable LazyModule, then we
|
539
|
+
# strip as much of lazy_import behavior as possible (keeping it cached,
|
540
|
+
# in case loading fails and we need to reset the lazy state).
|
541
|
+
if not hasattr(modclass, '_lazy_import_error_msgs'):
|
542
|
+
# Alreay loaded (no _lazy_import_error_msgs attr). Not reloading.
|
543
|
+
return
|
544
|
+
# First, ensure the parent is loaded (using recursion; *very* unlikely
|
545
|
+
# we'll ever hit a stack limit in this case).
|
546
|
+
modclass._LOADING = True
|
547
|
+
try:
|
548
|
+
if parent:
|
549
|
+
logger.debug("first loading parent module {}".format(parent))
|
550
|
+
setattr(sys.modules[parent], modname, module)
|
551
|
+
if not hasattr(modclass, '_LOADING'):
|
552
|
+
logger.debug("Module {} already loaded by the parent"
|
553
|
+
.format(modname))
|
554
|
+
# We've been loaded by the parent. Let's bail.
|
555
|
+
return
|
556
|
+
cached_data = _clean_lazymodule(module)
|
557
|
+
try:
|
558
|
+
# Get Python to do the real import!
|
559
|
+
reload_module(module)
|
560
|
+
except:
|
561
|
+
# Loading failed. We reset our lazy state.
|
562
|
+
logger.debug("Failed to load module {}. Resetting..."
|
563
|
+
.format(modname))
|
564
|
+
_reset_lazymodule(module, cached_data)
|
565
|
+
raise
|
566
|
+
else:
|
567
|
+
# Successful load
|
568
|
+
logger.debug("Successfully loaded module {}".format(modname))
|
569
|
+
delattr(modclass, '_LOADING')
|
570
|
+
_reset_lazy_submod_refs(module)
|
571
|
+
|
572
|
+
except (AttributeError, ImportError) as err:
|
573
|
+
logger.debug("Failed to load {}.\n{}: {}"
|
574
|
+
.format(modname, err.__class__.__name__, err))
|
575
|
+
logger.lazy_trace()
|
576
|
+
# Under Python 3 reloading our dummy LazyModule instances causes an
|
577
|
+
# AttributeError if the module can't be found. Would be preferrable
|
578
|
+
# if we could always rely on an ImportError. As it is we vet the
|
579
|
+
# AttributeError as thoroughly as possible.
|
580
|
+
if ((six.PY3 and isinstance(err, AttributeError)) and not
|
581
|
+
err.args[0] == "'NoneType' object has no attribute 'name'"):
|
582
|
+
# Not the AttributeError we were looking for.
|
583
|
+
raise
|
584
|
+
msg = modclass._lazy_import_error_msgs['msg']
|
585
|
+
raise_from(ImportError(
|
586
|
+
msg.format(**modclass._lazy_import_error_strings)), None)
|
587
|
+
|
588
|
+
|
589
|
+
##############################
|
590
|
+
# Helper functions/constants #
|
591
|
+
##############################
|
592
|
+
|
593
|
+
_MSG = ("{caller} attempted to use a functionality that requires module "
|
594
|
+
"{module}, but it couldn't be loaded. Please install {install_name} "
|
595
|
+
"and retry.")
|
596
|
+
|
597
|
+
_MSG_CALLABLE = ("{caller} attempted to use a functionality that requires "
|
598
|
+
"{callable}, of module {module}, but it couldn't be found in that "
|
599
|
+
"module. Please install a version of {install_name} that has "
|
600
|
+
"{module}.{callable} and retry.")
|
601
|
+
|
602
|
+
_CLS_ATTRS = ("_lazy_import_error_strings", "_lazy_import_error_msgs",
|
603
|
+
"_lazy_import_callables", "_lazy_import_submodules", "__repr__")
|
604
|
+
|
605
|
+
_DELETION_DICT = ("_lazy_import_submodules",)
|
606
|
+
|
607
|
+
|
608
|
+
def _setdef(argdict, name, defaultvalue):
|
609
|
+
"""Like dict.setdefault but sets the default value also if None is present.
|
610
|
+
|
611
|
+
"""
|
612
|
+
if not name in argdict or argdict[name] is None:
|
613
|
+
argdict[name] = defaultvalue
|
614
|
+
return argdict[name]
|
615
|
+
|
616
|
+
|
617
|
+
def module_basename(modname):
|
618
|
+
return modname.partition('.')[0]
|
619
|
+
|
620
|
+
|
621
|
+
def _set_default_errornames(modname, error_strings, call=False):
|
622
|
+
# We don't set the modulename default here because it will change for
|
623
|
+
# parents of lazily imported submodules.
|
624
|
+
error_strings.setdefault('caller', _caller_name(3, default='Python'))
|
625
|
+
error_strings.setdefault('install_name', module_basename(modname))
|
626
|
+
error_strings.setdefault('msg', _MSG)
|
627
|
+
if call:
|
628
|
+
error_strings.setdefault('msg_callable', _MSG_CALLABLE)
|
629
|
+
|
630
|
+
|
631
|
+
def _caller_name(depth=2, default=''):
|
632
|
+
"""Returns the name of the calling namespace.
|
633
|
+
|
634
|
+
"""
|
635
|
+
# the presence of sys._getframe might be implementation-dependent.
|
636
|
+
# It isn't that serious if we can't get the caller's name.
|
637
|
+
try:
|
638
|
+
return sys._getframe(depth).f_globals['__name__']
|
639
|
+
except AttributeError:
|
640
|
+
return default
|
641
|
+
|
642
|
+
|
643
|
+
def _clean_lazymodule(module):
|
644
|
+
"""Removes all lazy behavior from a module's class, for loading.
|
645
|
+
|
646
|
+
Also removes all module attributes listed under the module's class deletion
|
647
|
+
dictionaries. Deletion dictionaries are class attributes with names
|
648
|
+
specified in `_DELETION_DICT`.
|
649
|
+
|
650
|
+
Parameters
|
651
|
+
----------
|
652
|
+
module: LazyModule
|
653
|
+
|
654
|
+
Returns
|
655
|
+
-------
|
656
|
+
dict
|
657
|
+
A dictionary of deleted class attributes, that can be used to reset the
|
658
|
+
lazy state using :func:`_reset_lazymodule`.
|
659
|
+
"""
|
660
|
+
modclass = type(module)
|
661
|
+
_clean_lazy_submod_refs(module)
|
662
|
+
|
663
|
+
modclass.__getattribute__ = ModuleType.__getattribute__
|
664
|
+
modclass.__setattr__ = ModuleType.__setattr__
|
665
|
+
cls_attrs = {}
|
666
|
+
for cls_attr in _CLS_ATTRS:
|
667
|
+
try:
|
668
|
+
cls_attrs[cls_attr] = getattr(modclass, cls_attr)
|
669
|
+
delattr(modclass, cls_attr)
|
670
|
+
except AttributeError:
|
671
|
+
pass
|
672
|
+
return cls_attrs
|
673
|
+
|
674
|
+
|
675
|
+
def _clean_lazy_submod_refs(module):
|
676
|
+
modclass = type(module)
|
677
|
+
for deldict in _DELETION_DICT:
|
678
|
+
try:
|
679
|
+
delnames = getattr(modclass, deldict)
|
680
|
+
except AttributeError:
|
681
|
+
continue
|
682
|
+
for delname in delnames:
|
683
|
+
try:
|
684
|
+
super(LazyModule, module).__delattr__(delname)
|
685
|
+
except AttributeError:
|
686
|
+
# Maybe raise a warning?
|
687
|
+
pass
|
688
|
+
|
689
|
+
|
690
|
+
def _reset_lazymodule(module, cls_attrs):
|
691
|
+
"""Resets a module's lazy state from cached data.
|
692
|
+
|
693
|
+
"""
|
694
|
+
modclass = type(module)
|
695
|
+
del modclass.__getattribute__
|
696
|
+
del modclass.__setattr__
|
697
|
+
try:
|
698
|
+
del modclass._LOADING
|
699
|
+
except AttributeError:
|
700
|
+
pass
|
701
|
+
for cls_attr in _CLS_ATTRS:
|
702
|
+
try:
|
703
|
+
setattr(modclass, cls_attr, cls_attrs[cls_attr])
|
704
|
+
except KeyError:
|
705
|
+
pass
|
706
|
+
_reset_lazy_submod_refs(module)
|
707
|
+
|
708
|
+
|
709
|
+
def _reset_lazy_submod_refs(module):
|
710
|
+
modclass = type(module)
|
711
|
+
for deldict in _DELETION_DICT:
|
712
|
+
try:
|
713
|
+
resetnames = getattr(modclass, deldict)
|
714
|
+
except AttributeError:
|
715
|
+
continue
|
716
|
+
for name, submod in resetnames.items():
|
717
|
+
super(LazyModule, module).__setattr__(name, submod)
|
718
|
+
|
719
|
+
|
720
|
+
def run_from_ipython():
|
721
|
+
# Taken from https://stackoverflow.com/questions/5376837
|
722
|
+
try:
|
723
|
+
__IPYTHON__
|
724
|
+
return True
|
725
|
+
except NameError:
|
726
|
+
return False
|
727
|
+
|