nlpertools 1.0.5__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. nlpertools/__init__.py +24 -20
  2. nlpertools/algo/ac.py +18 -0
  3. nlpertools/algo/bit_ops.py +28 -0
  4. nlpertools/algo/kmp.py +94 -55
  5. nlpertools/algo/num_ops.py +12 -0
  6. nlpertools/algo/template.py +116 -0
  7. nlpertools/algo/union.py +13 -0
  8. nlpertools/data_client.py +387 -257
  9. nlpertools/data_structure/base_structure.py +109 -13
  10. nlpertools/dataprocess.py +611 -3
  11. nlpertools/default_db_config.yml +41 -0
  12. nlpertools/io/__init__.py +3 -3
  13. nlpertools/io/dir.py +54 -36
  14. nlpertools/io/file.py +277 -222
  15. nlpertools/ml.py +483 -460
  16. nlpertools/monitor/__init__.py +0 -0
  17. nlpertools/monitor/gpu.py +18 -0
  18. nlpertools/monitor/memory.py +24 -0
  19. nlpertools/movie.py +36 -0
  20. nlpertools/nlpertools_config.yml +1 -0
  21. nlpertools/{openApi.py → open_api.py} +65 -65
  22. nlpertools/other.py +364 -249
  23. nlpertools/pic.py +288 -0
  24. nlpertools/plugin.py +43 -43
  25. nlpertools/reminder.py +98 -87
  26. nlpertools/utils/__init__.py +3 -3
  27. nlpertools/utils/lazy.py +727 -0
  28. nlpertools/utils/log_util.py +20 -0
  29. nlpertools/utils/package.py +89 -76
  30. nlpertools/utils/package_v1.py +94 -0
  31. nlpertools/utils/package_v2.py +117 -0
  32. nlpertools/utils_for_nlpertools.py +93 -93
  33. nlpertools/vector_index_demo.py +108 -0
  34. nlpertools/wrapper.py +161 -96
  35. {nlpertools-1.0.5.dist-info → nlpertools-1.0.6.dev0.dist-info}/LICENSE +200 -200
  36. nlpertools-1.0.6.dev0.dist-info/METADATA +111 -0
  37. nlpertools-1.0.6.dev0.dist-info/RECORD +43 -0
  38. {nlpertools-1.0.5.dist-info → nlpertools-1.0.6.dev0.dist-info}/WHEEL +1 -1
  39. nlpertools-1.0.6.dev0.dist-info/top_level.txt +2 -0
  40. nlpertools_helper/__init__.py +10 -0
  41. nlpertools-1.0.5.dist-info/METADATA +0 -85
  42. nlpertools-1.0.5.dist-info/RECORD +0 -25
  43. nlpertools-1.0.5.dist-info/top_level.txt +0 -1
@@ -0,0 +1,727 @@
1
+ """
2
+ Write at first
3
+
4
+ This function comes from https://github.com/mnmelo/lazy_import
5
+ I can't install it property, so i add it to my project
6
+
7
+ # TODO failed in torch
8
+ """
9
+ # -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*-
10
+ # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
11
+ #
12
+ # lazy_import --- https://github.com/mnmelo/lazy_import
13
+ # Copyright (C) 2017-2018 Manuel Nuno Melo
14
+ #
15
+ # This file is part of lazy_import.
16
+ #
17
+ # lazy_import is free software: you can redistribute it and/or modify
18
+ # it under the terms of the GNU General Public License as published by
19
+ # the Free Software Foundation, either version 3 of the License, or
20
+ # (at your option) any later version.
21
+ #
22
+ # lazy_import is distributed in the hope that it will be useful,
23
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
24
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25
+ # GNU General Public License for more details.
26
+ #
27
+ # You should have received a copy of the GNU General Public License
28
+ # along with lazy_import. If not, see <http://www.gnu.org/licenses/>.
29
+ #
30
+ # lazy_import was based on code from the importing module from the PEAK
31
+ # package (see <http://peak.telecommunity.com/DevCenter/Importing>). The PEAK
32
+ # package is released under the following license, reproduced here:
33
+ #
34
+ # Copyright (C) 1996-2004 by Phillip J. Eby and Tyler C. Sarna.
35
+ # All rights reserved. This software may be used under the same terms
36
+ # as Zope or Python. THERE ARE ABSOLUTELY NO WARRANTIES OF ANY KIND.
37
+ # Code quality varies between modules, from "beta" to "experimental
38
+ # pre-alpha". :)
39
+ #
40
+ # Code pertaining to lazy loading from PEAK importing was included in
41
+ # lazy_import, modified in a number of ways. These are detailed in the
42
+ # CHANGELOG file of lazy_import. Changes mainly involved Python 3
43
+ # compatibility, extension to allow customizable behavior, and added
44
+ # functionality (lazy importing of callable objects).
45
+ #
46
+
47
+ """
48
+ Lazy module loading
49
+ ===================
50
+
51
+ Functions and classes for lazy module loading that also delay import errors.
52
+ Heavily borrowed from the `importing`_ module.
53
+
54
+ .. _`importing`: http://peak.telecommunity.com/DevCenter/Importing
55
+
56
+ Files and directories
57
+ ---------------------
58
+
59
+ .. autofunction:: module
60
+ .. autofunction:: callable
61
+
62
+ """
63
+
64
+ __all__ = ['lazy_module', 'lazy_callable', 'lazy_function', 'lazy_class',
65
+ 'LazyModule', 'LazyCallable', 'module_basename', '_MSG',
66
+ '_MSG_CALLABLE']
67
+
68
+ from types import ModuleType
69
+ import sys
70
+
71
+ try:
72
+ from importlib._bootstrap import _ImportLockContext
73
+ except ImportError:
74
+ # Python 2 doesn't have the context manager. Roll it ourselves (copied from
75
+ # Python 3's importlib/_bootstrap.py)
76
+ import imp
77
+
78
+
79
+ class _ImportLockContext:
80
+ """Context manager for the import lock."""
81
+
82
+ def __enter__(self):
83
+ imp.acquire_lock()
84
+
85
+ def __exit__(self, exc_type, exc_value, exc_traceback):
86
+ imp.release_lock()
87
+
88
+ # Adding a __spec__ doesn't really help. I'll leave the code here in case
89
+ # future python implementations start relying on it.
90
+ # try:
91
+ # from importlib.machinery import ModuleSpec
92
+ # except ImportError:
93
+ # ModuleSpec = None
94
+
95
+ import six
96
+ from six import raise_from
97
+ from six.moves import reload_module
98
+ # It is sometime useful to have access to the version number of a library.
99
+ # This is usually done through the __version__ special attribute.
100
+ # To make sure the version number is consistent between setup.py and the
101
+ # library, we read the version number from the file called VERSION that stays
102
+ # in the module directory.
103
+ import os
104
+
105
+ VERSION_FILE = os.path.join(os.path.dirname(__file__), 'VERSION')
106
+ __version__ = "1"
107
+
108
+ # Logging
109
+ import logging
110
+
111
+ # adding a TRACE level for stack debugging
112
+ _LAZY_TRACE = 1
113
+ logging.addLevelName(1, "LAZY_TRACE")
114
+ logging.basicConfig(level=logging.WARNING)
115
+
116
+
117
+ # Logs a formatted stack (takes no message or args/kwargs)
118
+ def _lazy_trace(self):
119
+ if self.isEnabledFor(_LAZY_TRACE):
120
+ import traceback
121
+ self._log(_LAZY_TRACE, " ### STACK TRACE ###", ())
122
+ for line in traceback.format_stack(sys._getframe(2)):
123
+ for subline in line.split("\n"):
124
+ self._log(_LAZY_TRACE, subline.rstrip(), ())
125
+
126
+
127
+ logging.Logger.lazy_trace = _lazy_trace
128
+ logger = logging.getLogger(__name__)
129
+
130
+
131
+ ################################
132
+ # Module/function registration #
133
+ ################################
134
+
135
+ #### Lazy classes ####
136
+
137
+ class LazyModule(ModuleType):
138
+ """Class for lazily-loaded modules that triggers proper loading on access.
139
+
140
+ Instantiation should be made from a subclass of :class:`LazyModule`, with
141
+ one subclass per instantiated module. Regular attribute set/access can then
142
+ be recovered by setting the subclass's :meth:`__getattribute__` and
143
+ :meth:`__setattribute__` to those of :class:`types.ModuleType`.
144
+ """
145
+
146
+ # peak.util.imports sets __slots__ to (), but it seems pointless because
147
+ # the base ModuleType doesn't itself set __slots__.
148
+ def __getattribute__(self, attr):
149
+ logger.debug("Getting attr {} of LazyModule instance of {}"
150
+ .format(attr, super(LazyModule, self)
151
+ .__getattribute__("__name__")))
152
+ logger.lazy_trace()
153
+ # IPython tries to be too clever and constantly inspects, asking for
154
+ # modules' attrs, which causes premature module loading and unesthetic
155
+ # internal errors if the lazily-loaded module doesn't exist.
156
+ if (run_from_ipython()
157
+ and (attr.startswith(("__", "_ipython"))
158
+ or attr == "_repr_mimebundle_")
159
+ and module_basename(_caller_name()) in ('inspect', 'IPython')):
160
+ logger.debug("Ignoring request for {}, deemed from IPython's "
161
+ "inspection.".format(super(LazyModule, self)
162
+ .__getattribute__("__name__"), attr))
163
+ raise AttributeError
164
+ if not attr in ('__name__', '__class__', '__spec__'):
165
+ # __name__ and __class__ yield their values from the LazyModule;
166
+ # __spec__ causes an AttributeError. Maybe in the future it will be
167
+ # necessary to return an actual ModuleSpec object, but it works as
168
+ # it is without that now.
169
+
170
+ # If it's an already-loaded submodule, we return it without
171
+ # triggering a full loading
172
+ try:
173
+ return sys.modules[self.__name__ + "." + attr]
174
+ except KeyError:
175
+ pass
176
+ # Check if it's one of the lazy callables
177
+ try:
178
+ _callable = type(self)._lazy_import_callables[attr]
179
+ logger.debug("Returning lazy-callable '{}'.".format(attr))
180
+ return _callable
181
+ except (AttributeError, KeyError) as err:
182
+ logger.debug("Proceeding to load module {}, "
183
+ "from requested value {}"
184
+ .format(super(LazyModule, self)
185
+ .__getattribute__("__name__"), attr))
186
+ _load_module(self)
187
+ logger.debug("Returning value '{}'.".format(super(LazyModule, self)
188
+ .__getattribute__(attr)))
189
+ return super(LazyModule, self).__getattribute__(attr)
190
+
191
+ def __setattr__(self, attr, value):
192
+ logger.debug("Setting attr {} to value {}, in LazyModule instance "
193
+ "of {}".format(attr, value, super(LazyModule, self)
194
+ .__getattribute__("__name__")))
195
+ _load_module(self)
196
+ return super(LazyModule, self).__setattr__(attr, value)
197
+
198
+
199
+ class LazyCallable(object):
200
+ """Class for lazily-loaded callables that triggers module loading on access
201
+
202
+ """
203
+
204
+ def __init__(self, *args):
205
+ if len(args) != 2:
206
+ # Maybe the user tried to base a class off this lazy callable?
207
+ try:
208
+ logger.debug("Got wrong number of args when init'ing "
209
+ "LazyCallable. args is '{}'".format(args))
210
+ base = args[1][0]
211
+ if isinstance(base, LazyCallable) and len(args) == 3:
212
+ raise NotImplementedError("It seems you are trying to use "
213
+ "a lazy callable as a class "
214
+ "base. This is not supported.")
215
+ except (IndexError, TypeError):
216
+ raise_from(TypeError("LazyCallable takes exactly 2 arguments: "
217
+ "a module/lazy module object and the name of "
218
+ "a callable to be lazily loaded."), None)
219
+ self.module, self.cname = args
220
+ self.modclass = type(self.module)
221
+ self.callable = None
222
+ # Need to save these, since the module-loading gets rid of them
223
+ self.error_msgs = self.modclass._lazy_import_error_msgs
224
+ self.error_strings = self.modclass._lazy_import_error_strings
225
+
226
+ def __call__(self, *args, **kwargs):
227
+ # No need to go through all the reloading more than once.
228
+ if self.callable:
229
+ return self.callable(*args, **kwargs)
230
+ try:
231
+ del self.modclass._lazy_import_callables[self.cname]
232
+ except (AttributeError, KeyError):
233
+ pass
234
+ try:
235
+ self.callable = getattr(self.module, self.cname)
236
+ except AttributeError:
237
+ msg = self.error_msgs['msg_callable']
238
+ raise_from(AttributeError(
239
+ msg.format(callable=self.cname, **self.error_strings)), None)
240
+ except ImportError as err:
241
+ # Import failed. We reset the dict and re-raise the ImportError.
242
+ try:
243
+ self.modclass._lazy_import_callables[self.cname] = self
244
+ except AttributeError:
245
+ self.modclass._lazy_import_callables = {self.cname: self}
246
+ raise_from(err, None)
247
+ else:
248
+ return self.callable(*args, **kwargs)
249
+
250
+
251
+ ### Functions ###
252
+
253
+ def lazy_module(modname, error_strings=None, lazy_mod_class=LazyModule,
254
+ level='leaf'):
255
+ """Function allowing lazy importing of a module into the namespace.
256
+
257
+ A lazy module object is created, registered in `sys.modules`, and
258
+ returned. This is a hollow module; actual loading, and `ImportErrors` if
259
+ not found, are delayed until an attempt is made to access attributes of the
260
+ lazy module.
261
+
262
+ A handy application is to use :func:`lazy_module` early in your own code
263
+ (say, in `__init__.py`) to register all modulenames you want to be lazy.
264
+ Because of registration in `sys.modules` later invocations of
265
+ `import modulename` will also return the lazy object. This means that after
266
+ initial registration the rest of your code can use regular pyhon import
267
+ statements and retain the lazyness of the modules.
268
+
269
+ Parameters
270
+ ----------
271
+ modname : str
272
+ The module to import.
273
+ error_strings : dict, optional
274
+ A dictionary of strings to use when module-loading fails. Key 'msg'
275
+ sets the message to use (defaults to :attr:`lazy_import._MSG`). The
276
+ message is formatted using the remaining dictionary keys. The default
277
+ message informs the user of which module is missing (key 'module'),
278
+ what code loaded the module as lazy (key 'caller'), and which package
279
+ should be installed to solve the dependency (key 'install_name').
280
+ None of the keys is mandatory and all are given smart names by default.
281
+ lazy_mod_class: type, optional
282
+ Which class to use when instantiating the lazy module, to allow
283
+ deep customization. The default is :class:`LazyModule` and custom
284
+ alternatives **must** be a subclass thereof.
285
+ level : str, optional
286
+ Which submodule reference to return. Either a reference to the 'leaf'
287
+ module (the default) or to the 'base' module. This is useful if you'll
288
+ be using the module functionality in the same place you're calling
289
+ :func:`lazy_module` from, since then you don't need to run `import`
290
+ again. Setting *level* does not affect which names/modules get
291
+ registered in `sys.modules`.
292
+ For *level* set to 'base' and *modulename* 'aaa.bbb.ccc'::
293
+
294
+ aaa = lazy_import.lazy_module("aaa.bbb.ccc", level='base')
295
+ # 'aaa' becomes defined in the current namespace, with
296
+ # (sub)attributes 'aaa.bbb' and 'aaa.bbb.ccc'.
297
+ # It's the lazy equivalent to:
298
+ import aaa.bbb.ccc
299
+
300
+ For *level* set to 'leaf'::
301
+
302
+ ccc = lazy_import.lazy_module("aaa.bbb.ccc", level='leaf')
303
+ # Only 'ccc' becomes set in the current namespace.
304
+ # Lazy equivalent to:
305
+ from aaa.bbb import ccc
306
+
307
+ Returns
308
+ -------
309
+ module
310
+ The module specified by *modname*, or its base, depending on *level*.
311
+ The module isn't immediately imported. Instead, an instance of
312
+ *lazy_mod_class* is returned. Upon access to any of its attributes, the
313
+ module is finally loaded.
314
+
315
+ Examples
316
+ --------
317
+ >>> import lazy_import, sys
318
+ >>> np = lazy_import.lazy_module("numpy")
319
+ >>> np
320
+ Lazily-loaded module numpy
321
+ >>> np is sys.modules['numpy']
322
+ True
323
+ >>> np.pi # This causes the full loading of the module ...
324
+ 3.141592653589793
325
+ >>> np # ... and the module is changed in place.
326
+ <module 'numpy' from '/usr/local/lib/python/site-packages/numpy/__init__.py'>
327
+
328
+ >>> import lazy_import, sys
329
+ >>> # The following succeeds even when asking for a module that's not available
330
+ >>> missing = lazy_import.lazy_module("missing_module")
331
+ >>> missing
332
+ Lazily-loaded module missing_module
333
+ >>> missing is sys.modules['missing_module']
334
+ True
335
+ >>> missing.some_attr # This causes the full loading of the module, which now fails.
336
+ ImportError: __main__ attempted to use a functionality that requires module missing_module, but it couldn't be loaded. Please install missing_module and retry.
337
+
338
+ See Also
339
+ --------
340
+ :func:`lazy_callable`
341
+ :class:`LazyModule`
342
+
343
+ """
344
+ if error_strings is None:
345
+ error_strings = {}
346
+ _set_default_errornames(modname, error_strings)
347
+
348
+ mod = _lazy_module(modname, error_strings, lazy_mod_class)
349
+ if level == 'base':
350
+ return sys.modules[module_basename(modname)]
351
+ elif level == 'leaf':
352
+ return mod
353
+ else:
354
+ raise ValueError("Parameter 'level' must be one of ('base', 'leaf')")
355
+
356
+
357
+ def _lazy_module(modname, error_strings, lazy_mod_class):
358
+ with _ImportLockContext():
359
+ fullmodname = modname
360
+ fullsubmodname = None
361
+ # ensure parent module/package is in sys.modules
362
+ # and parent.modname=module, as soon as the parent is imported
363
+ while modname:
364
+ try:
365
+ mod = sys.modules[modname]
366
+ # We reached a (base) module that's already loaded. Let's stop
367
+ # the cycle. Can't use 'break' because we still want to go
368
+ # through the fullsubmodname check below.
369
+ modname = ''
370
+ except KeyError:
371
+ err_s = error_strings.copy()
372
+ err_s.setdefault('module', modname)
373
+
374
+ class _LazyModule(lazy_mod_class):
375
+ _lazy_import_error_msgs = {'msg': err_s.pop('msg')}
376
+ try:
377
+ _lazy_import_error_msgs['msg_callable'] = \
378
+ err_s.pop('msg_callable')
379
+ except KeyError:
380
+ pass
381
+ _lazy_import_error_strings = err_s
382
+ _lazy_import_callables = {}
383
+ _lazy_import_submodules = {}
384
+
385
+ def __repr__(self):
386
+ return "Lazily-loaded module {}".format(self.__name__)
387
+
388
+ # A bit of cosmetic, to make AttributeErrors read more natural
389
+ _LazyModule.__name__ = 'module'
390
+ # Actual module instantiation
391
+ mod = sys.modules[modname] = _LazyModule(modname)
392
+ # No need for __spec__. Maybe in the future.
393
+ # if ModuleSpec:
394
+ # ModuleType.__setattr__(mod, '__spec__',
395
+ # ModuleSpec(modname, None))
396
+ if fullsubmodname:
397
+ submod = sys.modules[fullsubmodname]
398
+ ModuleType.__setattr__(mod, submodname, submod)
399
+ _LazyModule._lazy_import_submodules[submodname] = submod
400
+ fullsubmodname = modname
401
+ modname, _, submodname = modname.rpartition('.')
402
+ return sys.modules[fullmodname]
403
+
404
+
405
+ def lazy_callable(modname, *names, **kwargs):
406
+ """Performs lazy importing of one or more callables.
407
+
408
+ :func:`lazy_callable` creates functions that are thin wrappers that pass
409
+ any and all arguments straight to the target module's callables. These can
410
+ be functions or classes. The full loading of that module is only actually
411
+ triggered when the returned lazy function itself is called. This lazy
412
+ import of the target module uses the same mechanism as
413
+ :func:`lazy_module`.
414
+
415
+ If, however, the target module has already been fully imported prior
416
+ to invocation of :func:`lazy_callable`, then the target callables
417
+ themselves are returned and no lazy imports are made.
418
+
419
+ :func:`lazy_function` and :func:`lazy_function` are aliases of
420
+ :func:`lazy_callable`.
421
+
422
+ Parameters
423
+ ----------
424
+ modname : str
425
+ The base module from where to import the callable(s) in *names*,
426
+ or a full 'module_name.callable_name' string.
427
+ names : str (optional)
428
+ The callable name(s) to import from the module specified by *modname*.
429
+ If left empty, *modname* is assumed to also include the callable name
430
+ to import.
431
+ error_strings : dict, optional
432
+ A dictionary of strings to use when reporting loading errors (either a
433
+ missing module, or a missing callable name in the loaded module).
434
+ *error_string* follows the same usage as described under
435
+ :func:`lazy_module`, with the exceptions that 1) a further key,
436
+ 'msg_callable', can be supplied to be used as the error when a module
437
+ is successfully loaded but the target callable can't be found therein
438
+ (defaulting to :attr:`lazy_import._MSG_CALLABLE`); 2) a key 'callable'
439
+ is always added with the callable name being loaded.
440
+ lazy_mod_class : type, optional
441
+ See definition under :func:`lazy_module`.
442
+ lazy_call_class : type, optional
443
+ Analogously to *lazy_mod_class*, allows setting a custom class to
444
+ handle lazy callables, other than the default :class:`LazyCallable`.
445
+
446
+ Returns
447
+ -------
448
+ wrapper function or tuple of wrapper functions
449
+ If *names* is passed, returns a tuple of wrapper functions, one for
450
+ each element in *names*.
451
+ If only *modname* is passed it is assumed to be a full
452
+ 'module_name.callable_name' string, in which case the wrapper for the
453
+ imported callable is returned directly, and not in a tuple.
454
+
455
+ Notes
456
+ -----
457
+ Unlike :func:`lazy_module`, which returns a lazy module that eventually
458
+ mutates into the fully-functional version, :func:`lazy_callable` only
459
+ returns thin wrappers that never change. This means that the returned
460
+ wrapper object never truly becomes the one under the module's namespace,
461
+ even after successful loading of the module in *modname*. This is fine for
462
+ most practical use cases, but may break code that relies on the usage of
463
+ the returned objects oter than calling them. One such example is the lazy
464
+ import of a class: it's fine to use the returned wrapper to instantiate an
465
+ object, but it can't be used, for instance, to subclass from.
466
+
467
+ Examples
468
+ --------
469
+ >>> import lazy_import, sys
470
+ >>> fn = lazy_import.lazy_callable("numpy.arange")
471
+ >>> sys.modules['numpy']
472
+ Lazily-loaded module numpy
473
+ >>> fn(10)
474
+ array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
475
+ >>> sys.modules['numpy']
476
+ <module 'numpy' from '/usr/local/lib/python3.5/site-packages/numpy/__init__.py'>
477
+
478
+ >>> import lazy_import, sys
479
+ >>> cl = lazy_import.lazy_callable("numpy.ndarray") # a class
480
+ >>> obj = cl([1, 2]) # This works OK (and also triggers the loading of numpy)
481
+ >>> class MySubclass(cl): # This fails because cls is just a wrapper,
482
+ >>> pass # not an actual class.
483
+
484
+ See Also
485
+ --------
486
+ :func:`lazy_module`
487
+ :class:`LazyCallable`
488
+ :class:`LazyModule`
489
+
490
+ """
491
+ if not names:
492
+ modname, _, name = modname.rpartition(".")
493
+ lazy_mod_class = _setdef(kwargs, 'lazy_mod_class', LazyModule)
494
+ lazy_call_class = _setdef(kwargs, 'lazy_call_class', LazyCallable)
495
+ error_strings = _setdef(kwargs, 'error_strings', {})
496
+ _set_default_errornames(modname, error_strings, call=True)
497
+
498
+ if not names:
499
+ # We allow passing a single string as 'modname.callable_name',
500
+ # in which case the wrapper is returned directly and not as a list.
501
+ return _lazy_callable(modname, name, error_strings.copy(),
502
+ lazy_mod_class, lazy_call_class)
503
+ return tuple(_lazy_callable(modname, cname, error_strings.copy(),
504
+ lazy_mod_class, lazy_call_class) for cname in names)
505
+
506
+
507
+ lazy_function = lazy_class = lazy_callable
508
+
509
+
510
+ def _lazy_callable(modname, cname, error_strings,
511
+ lazy_mod_class, lazy_call_class):
512
+ # We could do most of this in the LazyCallable __init__, but here we can
513
+ # pre-check whether to actually be lazy or not.
514
+ module = _lazy_module(modname, error_strings, lazy_mod_class)
515
+ modclass = type(module)
516
+ if (issubclass(modclass, LazyModule) and
517
+ hasattr(modclass, '_lazy_import_callables')):
518
+ modclass._lazy_import_callables.setdefault(
519
+ cname, lazy_call_class(module, cname))
520
+ return getattr(module, cname)
521
+
522
+
523
+ #######################
524
+ # Real module loading #
525
+ #######################
526
+
527
+ def _load_module(module):
528
+ """Ensures that a module, and its parents, are properly loaded
529
+
530
+ """
531
+ modclass = type(module)
532
+ # We only take care of our own LazyModule instances
533
+ if not issubclass(modclass, LazyModule):
534
+ raise TypeError("Passed module is not a LazyModule instance.")
535
+ with _ImportLockContext():
536
+ parent, _, modname = module.__name__.rpartition('.')
537
+ logger.debug("loading module {}".format(modname))
538
+ # We first identify whether this is a loadable LazyModule, then we
539
+ # strip as much of lazy_import behavior as possible (keeping it cached,
540
+ # in case loading fails and we need to reset the lazy state).
541
+ if not hasattr(modclass, '_lazy_import_error_msgs'):
542
+ # Alreay loaded (no _lazy_import_error_msgs attr). Not reloading.
543
+ return
544
+ # First, ensure the parent is loaded (using recursion; *very* unlikely
545
+ # we'll ever hit a stack limit in this case).
546
+ modclass._LOADING = True
547
+ try:
548
+ if parent:
549
+ logger.debug("first loading parent module {}".format(parent))
550
+ setattr(sys.modules[parent], modname, module)
551
+ if not hasattr(modclass, '_LOADING'):
552
+ logger.debug("Module {} already loaded by the parent"
553
+ .format(modname))
554
+ # We've been loaded by the parent. Let's bail.
555
+ return
556
+ cached_data = _clean_lazymodule(module)
557
+ try:
558
+ # Get Python to do the real import!
559
+ reload_module(module)
560
+ except:
561
+ # Loading failed. We reset our lazy state.
562
+ logger.debug("Failed to load module {}. Resetting..."
563
+ .format(modname))
564
+ _reset_lazymodule(module, cached_data)
565
+ raise
566
+ else:
567
+ # Successful load
568
+ logger.debug("Successfully loaded module {}".format(modname))
569
+ delattr(modclass, '_LOADING')
570
+ _reset_lazy_submod_refs(module)
571
+
572
+ except (AttributeError, ImportError) as err:
573
+ logger.debug("Failed to load {}.\n{}: {}"
574
+ .format(modname, err.__class__.__name__, err))
575
+ logger.lazy_trace()
576
+ # Under Python 3 reloading our dummy LazyModule instances causes an
577
+ # AttributeError if the module can't be found. Would be preferrable
578
+ # if we could always rely on an ImportError. As it is we vet the
579
+ # AttributeError as thoroughly as possible.
580
+ if ((six.PY3 and isinstance(err, AttributeError)) and not
581
+ err.args[0] == "'NoneType' object has no attribute 'name'"):
582
+ # Not the AttributeError we were looking for.
583
+ raise
584
+ msg = modclass._lazy_import_error_msgs['msg']
585
+ raise_from(ImportError(
586
+ msg.format(**modclass._lazy_import_error_strings)), None)
587
+
588
+
589
+ ##############################
590
+ # Helper functions/constants #
591
+ ##############################
592
+
593
+ _MSG = ("{caller} attempted to use a functionality that requires module "
594
+ "{module}, but it couldn't be loaded. Please install {install_name} "
595
+ "and retry.")
596
+
597
+ _MSG_CALLABLE = ("{caller} attempted to use a functionality that requires "
598
+ "{callable}, of module {module}, but it couldn't be found in that "
599
+ "module. Please install a version of {install_name} that has "
600
+ "{module}.{callable} and retry.")
601
+
602
+ _CLS_ATTRS = ("_lazy_import_error_strings", "_lazy_import_error_msgs",
603
+ "_lazy_import_callables", "_lazy_import_submodules", "__repr__")
604
+
605
+ _DELETION_DICT = ("_lazy_import_submodules",)
606
+
607
+
608
+ def _setdef(argdict, name, defaultvalue):
609
+ """Like dict.setdefault but sets the default value also if None is present.
610
+
611
+ """
612
+ if not name in argdict or argdict[name] is None:
613
+ argdict[name] = defaultvalue
614
+ return argdict[name]
615
+
616
+
617
+ def module_basename(modname):
618
+ return modname.partition('.')[0]
619
+
620
+
621
+ def _set_default_errornames(modname, error_strings, call=False):
622
+ # We don't set the modulename default here because it will change for
623
+ # parents of lazily imported submodules.
624
+ error_strings.setdefault('caller', _caller_name(3, default='Python'))
625
+ error_strings.setdefault('install_name', module_basename(modname))
626
+ error_strings.setdefault('msg', _MSG)
627
+ if call:
628
+ error_strings.setdefault('msg_callable', _MSG_CALLABLE)
629
+
630
+
631
+ def _caller_name(depth=2, default=''):
632
+ """Returns the name of the calling namespace.
633
+
634
+ """
635
+ # the presence of sys._getframe might be implementation-dependent.
636
+ # It isn't that serious if we can't get the caller's name.
637
+ try:
638
+ return sys._getframe(depth).f_globals['__name__']
639
+ except AttributeError:
640
+ return default
641
+
642
+
643
+ def _clean_lazymodule(module):
644
+ """Removes all lazy behavior from a module's class, for loading.
645
+
646
+ Also removes all module attributes listed under the module's class deletion
647
+ dictionaries. Deletion dictionaries are class attributes with names
648
+ specified in `_DELETION_DICT`.
649
+
650
+ Parameters
651
+ ----------
652
+ module: LazyModule
653
+
654
+ Returns
655
+ -------
656
+ dict
657
+ A dictionary of deleted class attributes, that can be used to reset the
658
+ lazy state using :func:`_reset_lazymodule`.
659
+ """
660
+ modclass = type(module)
661
+ _clean_lazy_submod_refs(module)
662
+
663
+ modclass.__getattribute__ = ModuleType.__getattribute__
664
+ modclass.__setattr__ = ModuleType.__setattr__
665
+ cls_attrs = {}
666
+ for cls_attr in _CLS_ATTRS:
667
+ try:
668
+ cls_attrs[cls_attr] = getattr(modclass, cls_attr)
669
+ delattr(modclass, cls_attr)
670
+ except AttributeError:
671
+ pass
672
+ return cls_attrs
673
+
674
+
675
+ def _clean_lazy_submod_refs(module):
676
+ modclass = type(module)
677
+ for deldict in _DELETION_DICT:
678
+ try:
679
+ delnames = getattr(modclass, deldict)
680
+ except AttributeError:
681
+ continue
682
+ for delname in delnames:
683
+ try:
684
+ super(LazyModule, module).__delattr__(delname)
685
+ except AttributeError:
686
+ # Maybe raise a warning?
687
+ pass
688
+
689
+
690
+ def _reset_lazymodule(module, cls_attrs):
691
+ """Resets a module's lazy state from cached data.
692
+
693
+ """
694
+ modclass = type(module)
695
+ del modclass.__getattribute__
696
+ del modclass.__setattr__
697
+ try:
698
+ del modclass._LOADING
699
+ except AttributeError:
700
+ pass
701
+ for cls_attr in _CLS_ATTRS:
702
+ try:
703
+ setattr(modclass, cls_attr, cls_attrs[cls_attr])
704
+ except KeyError:
705
+ pass
706
+ _reset_lazy_submod_refs(module)
707
+
708
+
709
+ def _reset_lazy_submod_refs(module):
710
+ modclass = type(module)
711
+ for deldict in _DELETION_DICT:
712
+ try:
713
+ resetnames = getattr(modclass, deldict)
714
+ except AttributeError:
715
+ continue
716
+ for name, submod in resetnames.items():
717
+ super(LazyModule, module).__setattr__(name, submod)
718
+
719
+
720
+ def run_from_ipython():
721
+ # Taken from https://stackoverflow.com/questions/5376837
722
+ try:
723
+ __IPYTHON__
724
+ return True
725
+ except NameError:
726
+ return False
727
+