cdxcore 0.1.24__py3-none-any.whl → 0.1.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cdxcore might be problematic. Click here for more details.
- cdxcore/__init__.py +1 -1
- cdxcore/subdir.py +447 -396
- cdxcore/uniquehash.py +1 -1
- {cdxcore-0.1.24.dist-info → cdxcore-0.1.26.dist-info}/METADATA +1 -1
- {cdxcore-0.1.24.dist-info → cdxcore-0.1.26.dist-info}/RECORD +8 -8
- {cdxcore-0.1.24.dist-info → cdxcore-0.1.26.dist-info}/WHEEL +0 -0
- {cdxcore-0.1.24.dist-info → cdxcore-0.1.26.dist-info}/licenses/LICENSE +0 -0
- {cdxcore-0.1.24.dist-info → cdxcore-0.1.26.dist-info}/top_level.txt +0 -0
cdxcore/__init__.py
CHANGED
cdxcore/subdir.py
CHANGED
|
@@ -94,8 +94,7 @@ File Format
|
|
|
94
94
|
* "BLOSC" uses `blosc <https://github.com/blosc/python-blosc>`__
|
|
95
95
|
to read/write compressed binary data. The blosc compression algorithm is very fast,
|
|
96
96
|
hence using this mode will not usually lead to notably slower performance than using
|
|
97
|
-
"PICKLE" but will generate smaller files, depending on your data structure.
|
|
98
|
-
|
|
97
|
+
"PICKLE" but will generate smaller files, depending on your data structure.
|
|
99
98
|
The default extension for "BLOSC" is "zbsc".
|
|
100
99
|
|
|
101
100
|
* "GZIP": uses :mod:`gzip` to
|
|
@@ -324,19 +323,19 @@ Caching
|
|
|
324
323
|
A :class:`cdxcore.subdir.SubDir` object offers an advanced context for caching calls to :class:`collection.abc.Callable``
|
|
325
324
|
objects with :dec:`cdxcore.subdir.SubDir.cache`.
|
|
326
325
|
|
|
327
|
-
|
|
326
|
+
.. code-block:: python
|
|
328
327
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
328
|
+
from cdxcore.subdir import SubDir
|
|
329
|
+
cache = SubDir("!/.cache")
|
|
330
|
+
cache.delete_all_content() # for illustration
|
|
331
|
+
|
|
332
|
+
@cache.cache("0.1")
|
|
333
|
+
def f(x,y):
|
|
334
|
+
return x*y
|
|
335
|
+
|
|
336
|
+
_ = f(1,2) # function gets computed and the result cached
|
|
337
|
+
_ = f(1,2) # restore result from cache
|
|
338
|
+
_ = f(2,2) # different parameters: compute and store result
|
|
340
339
|
|
|
341
340
|
This involves keying the cache by the function name and its current parameters using :class:`cdxcore.uniquehash.UniqueHash`,
|
|
342
341
|
and monitoring the functions version using :dec:`cdxcore.version.version`. The caching behaviour itself can be controlled by
|
|
@@ -377,8 +376,7 @@ from .pretty import PrettyObject
|
|
|
377
376
|
from .verbose import Context
|
|
378
377
|
from .version import Version, version as version_decorator, VersionError
|
|
379
378
|
from .util import fmt_list, fmt_filename, DEF_FILE_NAME_MAP, plain, is_filename
|
|
380
|
-
from .uniquehash import unique_hash48, UniqueLabel, NamedUniqueHash
|
|
381
|
-
|
|
379
|
+
from .uniquehash import unique_hash48, UniqueLabel, NamedUniqueHash, named_unique_filename48_8
|
|
382
380
|
|
|
383
381
|
"""
|
|
384
382
|
:meta private:
|
|
@@ -637,41 +635,34 @@ class CacheController( object ):
|
|
|
637
635
|
|
|
638
636
|
Parameters
|
|
639
637
|
----------
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
Defaults to ``[Context]``.
|
|
644
|
-
|
|
645
|
-
cache_mode : CacheMode, optional
|
|
646
|
-
Top level cache control.
|
|
647
|
-
Set to "OFF" to turn off all caching.
|
|
648
|
-
Default is "ON".
|
|
649
|
-
|
|
650
|
-
max_filename_length : int, optional
|
|
651
|
-
Maximum filename length. If unique id's exceed the file name a hash of length
|
|
652
|
-
``hash_length`` will be intergated into the file name.
|
|
653
|
-
See :class:`cdxcore.uniquehash.NamedUniqueHash`.
|
|
654
|
-
Default is ``48``.
|
|
655
|
-
|
|
656
|
-
hash_length : int, optional
|
|
657
|
-
Length of the hash used to make sure each filename is unique
|
|
658
|
-
See :class:`cdxcore.uniquehash.NamedUniqueHash`.
|
|
659
|
-
Default is ``8``.
|
|
660
|
-
|
|
661
|
-
debug_verbose : :class:`cdxcore.verbose.Context`, optional
|
|
662
|
-
If not ``None`` print caching process messages to this object.
|
|
663
|
-
|
|
664
|
-
Default is ``None``.
|
|
665
|
-
|
|
666
|
-
keep_last_arguments : bool, optional
|
|
667
|
-
Keep a dictionary of all parameters as string representations after each function call.
|
|
668
|
-
If the function ``F`` was decorated using :meth:``cdxcore.subdir.SubDir.cache``,
|
|
669
|
-
you can access this information via ``F.cache_info.last_arguments``.
|
|
638
|
+
exclude_arg_types : list[type], optional
|
|
639
|
+
List of types to exclude from producing unique ids from function arguments.
|
|
670
640
|
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
641
|
+
Defaults to ``[Context]``.
|
|
642
|
+
|
|
643
|
+
cache_mode : CacheMode, default ``ON``
|
|
644
|
+
Top level cache control.
|
|
645
|
+
Set to "OFF" to turn off all caching.
|
|
646
|
+
|
|
647
|
+
max_filename_length : int, default ``48``
|
|
648
|
+
Maximum filename length. If unique id's exceed the file name a hash of length
|
|
649
|
+
``hash_length`` will be intergated into the file name.
|
|
650
|
+
See :class:`cdxcore.uniquehash.NamedUniqueHash`.
|
|
651
|
+
|
|
652
|
+
hash_length : int, default ``8``
|
|
653
|
+
Length of the hash used to make sure each filename is unique
|
|
654
|
+
See :class:`cdxcore.uniquehash.NamedUniqueHash`.
|
|
655
|
+
|
|
656
|
+
debug_verbose : :class:`cdxcore.verbose.Context` | None, default ``None``
|
|
657
|
+
If not ``None`` print caching process messages to this object.
|
|
658
|
+
|
|
659
|
+
keep_last_arguments : bool, default ``False``
|
|
660
|
+
Keep a dictionary of all parameters as string representations after each function call.
|
|
661
|
+
If the function ``F`` was decorated using :meth:``cdxcore.subdir.SubDir.cache``,
|
|
662
|
+
you can access this information via ``F.cache_info.last_arguments``.
|
|
663
|
+
|
|
664
|
+
Note that strings are limited to 100 characters per argument to avoid memory
|
|
665
|
+
overload when large objects are passed.
|
|
675
666
|
"""
|
|
676
667
|
|
|
677
668
|
def __init__(self, *,
|
|
@@ -790,84 +781,74 @@ class SubDir(object):
|
|
|
790
781
|
|
|
791
782
|
Parameters
|
|
792
783
|
----------
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
784
|
+
name : str:
|
|
785
|
+
Name of the directory.
|
|
786
|
+
|
|
787
|
+
The name may start with any of the following special characters:
|
|
797
788
|
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
789
|
+
* ``'.'`` for current directory.
|
|
790
|
+
* ``'~'`` for home directory.
|
|
791
|
+
* ``'!'`` for system default temp directory.
|
|
792
|
+
* ``'?'`` for a temporary temp directory. In this case ``delete_everything_upon_exit`` is always ``True``.
|
|
802
793
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
ext : str, optional
|
|
819
|
-
Extension for files managed by this ``SubDir``. All files will share the same extension.
|
|
794
|
+
The directory name may also contain a formatting string for defining ``ext`` on the fly:
|
|
795
|
+
for example use ``"!/test;*.bin"`` to specify a directory ``"test"`` in the user's
|
|
796
|
+
temp directory with extension ``"bin"``.
|
|
797
|
+
|
|
798
|
+
The directory name can be set to ``None`` in which case it is always empty
|
|
799
|
+
and attempts to write to it fail with :class:`EOFError`.
|
|
800
|
+
|
|
801
|
+
parent : str | SubDir | None, default ``None``
|
|
802
|
+
Parent directory.
|
|
803
|
+
|
|
804
|
+
If ``parent`` is a :class:`cdxcore.subdir.SubDir` then its parameters are used
|
|
805
|
+
as default values here.
|
|
806
|
+
|
|
807
|
+
ext : str | None, default ``None``
|
|
808
|
+
Extension for files managed by this ``SubDir``. All files will share the same extension.
|
|
820
809
|
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
* 'pck' for the default PICKLE format.
|
|
828
|
-
* 'json' for JSON_PLAIN.
|
|
829
|
-
* 'jpck' for JSON_PICKLE.
|
|
830
|
-
* 'zbsc' for BLOSC.
|
|
831
|
-
* 'pgz' for GZIP.
|
|
832
|
-
|
|
833
|
-
Default is ``None``.
|
|
810
|
+
If set to ``""`` no extension is assigned to this directory. That means, for example, that
|
|
811
|
+
:meth:`cdxcore.subdir.SubDir.files` returns all files contained in the directory, not
|
|
812
|
+
just files with a specific extension.
|
|
813
|
+
|
|
814
|
+
If ``None``, use an extension depending on ``fmt``:
|
|
834
815
|
|
|
835
|
-
|
|
816
|
+
* 'pck' for the default PICKLE format.
|
|
817
|
+
* 'json' for JSON_PLAIN.
|
|
818
|
+
* 'jpck' for JSON_PICKLE.
|
|
819
|
+
* 'zbsc' for BLOSC.
|
|
820
|
+
* 'pgz' for GZIP.
|
|
821
|
+
|
|
822
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``Format.PICKLE``
|
|
836
823
|
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
Default is ``Format.PICKLE``.
|
|
824
|
+
One of the :class:`cdxcore.subdir.Format` codes.
|
|
825
|
+
If ``ext`` is left to ``None`` then setting the a format will also set the corrsponding ``ext``.
|
|
841
826
|
|
|
842
|
-
|
|
827
|
+
create_directory : bool | None, default ``False``
|
|
828
|
+
|
|
829
|
+
Whether to create the directory upon creation of the ``SubDir`` object; otherwise it will be created upon first
|
|
830
|
+
:meth:`cdxcore.subdir.SubDir.write`.
|
|
843
831
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
Set to ``None`` to use the setting of the parent directory, or ``False`` if no parent
|
|
848
|
-
is specified.
|
|
849
|
-
|
|
850
|
-
Default is ``False``.
|
|
832
|
+
Set to ``None`` to use the setting of the parent directory, or ``False`` if no parent
|
|
833
|
+
is specified.
|
|
851
834
|
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
835
|
+
cache_controller : :class:`cdxcore.subdir.CacheController` | None, default ``None``
|
|
836
|
+
|
|
837
|
+
An object which fine-tunes the behaviour of :meth:`cdxcore.subdir.SubDir.cache`.
|
|
838
|
+
See that function's documentation for further details. Default is ``None``.
|
|
856
839
|
|
|
857
|
-
|
|
840
|
+
delete_everything : bool, default ``False``
|
|
841
|
+
|
|
842
|
+
Delete all contents in the newly defined sub directory upon creation.
|
|
858
843
|
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
delete_everything_upon_exit : bool, optional
|
|
844
|
+
delete_everything_upon_exit : bool, default ``False``
|
|
845
|
+
|
|
846
|
+
Delete all contents of the current exist if ``self`` is deleted.
|
|
847
|
+
This is the always ``True`` if the ``"?/"`` pretext was used.
|
|
864
848
|
|
|
865
|
-
|
|
866
|
-
This is the always ``True`` if the ``"?/"`` pretext was used.
|
|
867
|
-
|
|
868
|
-
Note, however, that this will only be executed once the object is garbage collected.
|
|
849
|
+
Note, however, that this will only be executed once the object is garbage collected.
|
|
869
850
|
|
|
870
|
-
|
|
851
|
+
Default is, for some good reason, ``False``.
|
|
871
852
|
"""
|
|
872
853
|
|
|
873
854
|
class __RETURN_SUB_DIRECTORY(object):
|
|
@@ -997,8 +978,11 @@ class SubDir(object):
|
|
|
997
978
|
self._crt = bool(create_directory)
|
|
998
979
|
|
|
999
980
|
# cache controller
|
|
1000
|
-
|
|
1001
|
-
|
|
981
|
+
if cache_controller is None:
|
|
982
|
+
self._cctrl = parent._cctrl if not parent is None else None
|
|
983
|
+
else:
|
|
984
|
+
assert type(cache_controller).__name__ == CacheController.__name__, ("'cache_controller' should be of type 'CacheController'", type(cache_controller))
|
|
985
|
+
self._cctrl = cache_controller
|
|
1002
986
|
|
|
1003
987
|
# name
|
|
1004
988
|
self._tclean = delete_everything_upon_exit # delete directory upon completion
|
|
@@ -1170,13 +1154,13 @@ class SubDir(object):
|
|
|
1170
1154
|
|
|
1171
1155
|
Parameters
|
|
1172
1156
|
----------
|
|
1173
|
-
|
|
1174
|
-
|
|
1157
|
+
ext_or_fmt : str | :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
1158
|
+
An extension or a format.
|
|
1175
1159
|
|
|
1176
1160
|
Returns
|
|
1177
1161
|
-------
|
|
1178
|
-
|
|
1179
|
-
|
|
1162
|
+
ext : str
|
|
1163
|
+
The extension with leading ``'.'``.
|
|
1180
1164
|
"""
|
|
1181
1165
|
if isinstance(ext_or_fmt, Format):
|
|
1182
1166
|
return self._auto_ext(ext_or_fmt)
|
|
@@ -1193,9 +1177,9 @@ class SubDir(object):
|
|
|
1193
1177
|
|
|
1194
1178
|
Returns
|
|
1195
1179
|
-------
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1180
|
+
(ext, fmt) : tuple
|
|
1181
|
+
Here ``ext`` contains the leading ``'.'`` and ``fmt`` is
|
|
1182
|
+
of type :class:`cdxcore.subdir.Format`.
|
|
1199
1183
|
"""
|
|
1200
1184
|
if isinstance(ext, Format):
|
|
1201
1185
|
verify( fmt is None or fmt == ext, "If 'ext' is a Format, then 'fmt' must match 'ext' or be None. Found '%s' and '%s', respectively.", ext, fmt, exception=ValueError )
|
|
@@ -1255,9 +1239,10 @@ class SubDir(object):
|
|
|
1255
1239
|
def _extract_ext( ext : str ) -> str:
|
|
1256
1240
|
"""
|
|
1257
1241
|
Checks that 'ext' is an extension, and returns .ext.
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1242
|
+
|
|
1243
|
+
* Accepts '.ext' and 'ext'
|
|
1244
|
+
* Detects use of directories
|
|
1245
|
+
* Returns '*' if ext='*'
|
|
1261
1246
|
"""
|
|
1262
1247
|
assert not ext is None, ("'ext' should not be None here")
|
|
1263
1248
|
verify( isinstance(ext,str), "Extension 'ext' must be a string. Found type %s", type(ext).__name__, exception=ValueError )
|
|
@@ -1289,16 +1274,16 @@ class SubDir(object):
|
|
|
1289
1274
|
|
|
1290
1275
|
Parameters
|
|
1291
1276
|
----------
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1277
|
+
file : str
|
|
1278
|
+
Core file name without path or extension.
|
|
1279
|
+
ext : str | None, default ``None``
|
|
1280
|
+
If not ``None``, use this extension rather than :attr:`cdxcore.subdir.SubDir.ext`.
|
|
1296
1281
|
|
|
1297
1282
|
Returns
|
|
1298
1283
|
-------
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1284
|
+
Filename : str | None
|
|
1285
|
+
Fully qualified system file name.
|
|
1286
|
+
If ``self`` is ``None``, then this function returns ``None``; if ``file`` is ``None`` then this function also returns ``None``.
|
|
1302
1287
|
"""
|
|
1303
1288
|
if self._path is None or file is None:
|
|
1304
1289
|
return None
|
|
@@ -1380,22 +1365,22 @@ class SubDir(object):
|
|
|
1380
1365
|
|
|
1381
1366
|
Parameters
|
|
1382
1367
|
----------
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1368
|
+
reader( file, full_file_name, default )
|
|
1369
|
+
A function which is called to read the file once the correct directory is identified
|
|
1370
|
+
file : file (for error messages, might include '/')
|
|
1371
|
+
full_file_name : full file name
|
|
1372
|
+
default value
|
|
1373
|
+
file : str or list
|
|
1374
|
+
str: fully qualified file
|
|
1375
|
+
list: list of fully qualified names
|
|
1376
|
+
default :
|
|
1377
|
+
default value. None is a valid default value
|
|
1378
|
+
list : list of defaults for a list of keys
|
|
1379
|
+
raise_on_error : bool
|
|
1380
|
+
If True, and the file does not exist, throw exception
|
|
1381
|
+
ext :
|
|
1382
|
+
Extension or None for current extension.
|
|
1383
|
+
list : list of extensions for a list of keys
|
|
1399
1384
|
"""
|
|
1400
1385
|
# vector version
|
|
1401
1386
|
if not isinstance(file,str):
|
|
@@ -1494,7 +1479,6 @@ class SubDir(object):
|
|
|
1494
1479
|
def handle_pickle_error(e):
|
|
1495
1480
|
err = "invalid load key, '\\x03'."
|
|
1496
1481
|
if not version is None or e.args[0] != err:
|
|
1497
|
-
print("####", e.args)
|
|
1498
1482
|
raise e
|
|
1499
1483
|
raise VersionPresentError(
|
|
1500
1484
|
f"Error reading '{full_file_name}': encountered an unpickling error '{err}' "+\
|
|
@@ -1632,10 +1616,10 @@ class SubDir(object):
|
|
|
1632
1616
|
default = None,
|
|
1633
1617
|
raise_on_error : bool = False,
|
|
1634
1618
|
*,
|
|
1635
|
-
version : str = None,
|
|
1619
|
+
version : str|None = None,
|
|
1636
1620
|
delete_wrong_version : bool = True,
|
|
1637
|
-
ext : str = None,
|
|
1638
|
-
fmt : Format = None
|
|
1621
|
+
ext : str|None = None,
|
|
1622
|
+
fmt : Format|None = None
|
|
1639
1623
|
):
|
|
1640
1624
|
"""
|
|
1641
1625
|
Read data from a file if the file exists, or return ``default``.
|
|
@@ -1662,58 +1646,64 @@ class SubDir(object):
|
|
|
1662
1646
|
|
|
1663
1647
|
Parameters
|
|
1664
1648
|
----------
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1649
|
+
file : str
|
|
1650
|
+
A file name or a list thereof. ``file`` may contain subdirectories.
|
|
1651
|
+
|
|
1652
|
+
default :
|
|
1653
|
+
Default value, or default values if ``file`` is a list.
|
|
1654
|
+
|
|
1655
|
+
raise_on_error : bool, default ``False``
|
|
1656
|
+
Whether to raise an exception if reading an existing file failed.
|
|
1657
|
+
By default this function fails silently and returns the default.
|
|
1658
|
+
|
|
1659
|
+
version : str | None, default ``None``
|
|
1660
|
+
If not ``None``, specifies the version of the current code base.
|
|
1661
|
+
|
|
1662
|
+
In this case, this version will be compared to the version of the file being read.
|
|
1663
|
+
If they do not match, read fails (either by returning default or throwing a :class:`cdxcore.version.VersionError` exception).
|
|
1680
1664
|
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1665
|
+
You can specify version ``"*"`` to accept any version.
|
|
1666
|
+
Note that this is distinct
|
|
1667
|
+
to using ``None`` which stipulates that the file should not
|
|
1668
|
+
have version information.
|
|
1669
|
+
|
|
1670
|
+
delete_wrong_version : bool, default ``True``
|
|
1671
|
+
If ``True``, and if a wrong version was found, delete the file.
|
|
1688
1672
|
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
* ``None`` to use directory's default.
|
|
1695
|
-
* ``'*'`` to use the extension implied by ``fmt``.
|
|
1696
|
-
* ``""`` to turn of extension management.
|
|
1673
|
+
ext : str | None, default ``None``
|
|
1674
|
+
Extension overwrite, or a list thereof if ``file`` is a list.
|
|
1675
|
+
|
|
1676
|
+
Use:
|
|
1697
1677
|
|
|
1698
|
-
|
|
1699
|
-
|
|
1678
|
+
* ``None`` to use directory's default.
|
|
1679
|
+
* ``'*'`` to use the extension implied by ``fmt``.
|
|
1680
|
+
* ``""`` to turn of extension management.
|
|
1681
|
+
|
|
1682
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
1683
|
+
File :class:`cdxcore.subdir.Format` or ``None`` to use the directory's default.
|
|
1684
|
+
|
|
1685
|
+
Note:
|
|
1700
1686
|
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
* ``fmt`` cannot be a list even if ``file`` is.
|
|
1704
|
-
* Unless ``ext`` or the SubDir's extension is ``'*'``, changing the format does not automatically change the extension.
|
|
1687
|
+
* ``fmt`` cannot be a list even if ``file`` is.
|
|
1688
|
+
* Unless ``ext`` or the SubDir's extension is ``'*'``, changing the format does not automatically change the extension.
|
|
1705
1689
|
|
|
1706
1690
|
Returns
|
|
1707
1691
|
-------
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1692
|
+
Content : type | list
|
|
1693
|
+
For a single ``file`` returns the content of the file if successfully read, or ``default`` otherwise.
|
|
1694
|
+
If ``file``` is a list, this function returns a list of contents.
|
|
1711
1695
|
|
|
1712
1696
|
Raises
|
|
1713
1697
|
------
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1698
|
+
Version error : :class:`cdxcore.version.VersionError`:
|
|
1699
|
+
If the file's version did not match the ``version`` provided.
|
|
1700
|
+
|
|
1701
|
+
Version present : :class:`cdxcore.subdir.VersionPresentError`:
|
|
1702
|
+
When attempting to read a file without ``version`` which has a version this exception is raised.
|
|
1703
|
+
|
|
1704
|
+
I/O errors : ``Exception``
|
|
1705
|
+
Various standard I/O errors are raisedas usual.
|
|
1706
|
+
|
|
1717
1707
|
"""
|
|
1718
1708
|
return self._read( file=file,
|
|
1719
1709
|
default=default,
|
|
@@ -1730,42 +1720,43 @@ class SubDir(object):
|
|
|
1730
1720
|
|
|
1731
1721
|
Parameters
|
|
1732
1722
|
----------
|
|
1733
|
-
|
|
1734
|
-
|
|
1723
|
+
file : str
|
|
1724
|
+
A filename, or a list thereof.
|
|
1725
|
+
|
|
1735
1726
|
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1727
|
+
version : str
|
|
1728
|
+
Specifies the version to compare the file's version with.
|
|
1729
|
+
|
|
1730
|
+
You can use ``"*"`` to match any version.
|
|
1740
1731
|
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1732
|
+
raise_on_error : bool
|
|
1733
|
+
Whether to raise an exception if accessing an existing file failed (e.g. if it is a directory).
|
|
1734
|
+
By default this function fails silently and returns the default.
|
|
1744
1735
|
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
* ``None`` to use directory's default.
|
|
1754
|
-
* ``"*"`` to use the extension implied by ``fmt``.
|
|
1755
|
-
* ``""`` for no extension.
|
|
1736
|
+
delete_wrong_version : bool, default ``True``
|
|
1737
|
+
If ``True``, and if a wrong version was found, delete ``file``.
|
|
1738
|
+
|
|
1739
|
+
ext : str | None, default ``None``
|
|
1740
|
+
Extension overwrite, or a list thereof if ``file`` is a list.
|
|
1741
|
+
|
|
1742
|
+
Set to:
|
|
1756
1743
|
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1744
|
+
* ``None`` to use directory's default.
|
|
1745
|
+
* ``"*"`` to use the extension implied by ``fmt``.
|
|
1746
|
+
* ``""`` for no extension.
|
|
1747
|
+
|
|
1748
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
1749
|
+
File format or ``None`` to use the directory's default.
|
|
1750
|
+
Note that ``fmt`` cannot be a list even if ``file`` is.
|
|
1760
1751
|
|
|
1761
1752
|
Returns
|
|
1762
1753
|
-------
|
|
1763
1754
|
Status : bool
|
|
1764
|
-
Returns
|
|
1755
|
+
Returns ``True`` only if the file exists, has version information, and its version is equal to ``version``.
|
|
1765
1756
|
"""
|
|
1766
1757
|
return self._read( file=file,default=False,raise_on_error=raise_on_error,version=version,ext=ext,fmt=fmt,delete_wrong_version=delete_wrong_version,handle_version=SubDir.VER_CHECK )
|
|
1767
1758
|
|
|
1768
|
-
def get_version( self, file : str, raise_on_error : bool = False, *, ext : str = None, fmt : Format = None ):
|
|
1759
|
+
def get_version( self, file : str, raise_on_error : bool = False, *, ext : str|None = None, fmt : Format|None = None ):
|
|
1769
1760
|
"""
|
|
1770
1761
|
Returns a version stored in a file.
|
|
1771
1762
|
|
|
@@ -1774,32 +1765,33 @@ class SubDir(object):
|
|
|
1774
1765
|
|
|
1775
1766
|
Parameters
|
|
1776
1767
|
----------
|
|
1777
|
-
|
|
1778
|
-
|
|
1768
|
+
file : str
|
|
1769
|
+
A filename, or a list thereof.
|
|
1779
1770
|
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1771
|
+
raise_on_error : bool
|
|
1772
|
+
Whether to raise an exception if accessing an existing file failed (e.g. if it is a directory).
|
|
1773
|
+
By default this function fails silently and returns the default.
|
|
1783
1774
|
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
* ``None`` to use directory's default.
|
|
1793
|
-
* ``"*"`` to use the extension implied by ``fmt``.
|
|
1794
|
-
* ``""`` for no extension.
|
|
1775
|
+
delete_wrong_version : bool, default ``True``
|
|
1776
|
+
If ``True``, and if a wrong version was found, delete ``file``.
|
|
1777
|
+
|
|
1778
|
+
ext : str | None, default ``None``
|
|
1779
|
+
Extension overwrite, or a list thereof if ``file`` is a list.
|
|
1780
|
+
|
|
1781
|
+
Set to:
|
|
1795
1782
|
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1783
|
+
* ``None`` to use directory's default.
|
|
1784
|
+
* ``"*"`` to use the extension implied by ``fmt``.
|
|
1785
|
+
* ``""`` for no extension.
|
|
1786
|
+
|
|
1787
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
1788
|
+
File format or ``None`` to use the directory's default.
|
|
1789
|
+
Note that ``fmt`` cannot be a list even if ``file`` is.
|
|
1799
1790
|
|
|
1800
1791
|
Returns
|
|
1801
1792
|
-------
|
|
1802
|
-
|
|
1793
|
+
version : str
|
|
1794
|
+
The version.
|
|
1803
1795
|
"""
|
|
1804
1796
|
return self._read( file=file,default=None,raise_on_error=raise_on_error,version="",ext=ext,fmt=fmt,delete_wrong_version=False,handle_version=SubDir.VER_RETURN )
|
|
1805
1797
|
|
|
@@ -1856,9 +1848,9 @@ class SubDir(object):
|
|
|
1856
1848
|
# write to temp file, then rename into target file
|
|
1857
1849
|
# this reduces collision when i/o operations are slow
|
|
1858
1850
|
full_file_name = self.full_file_name(file,ext=ext)
|
|
1859
|
-
tmp_file
|
|
1860
|
-
tmp_i
|
|
1861
|
-
fullTmpFile
|
|
1851
|
+
tmp_file = self.temp_file_name( file )
|
|
1852
|
+
tmp_i = 0
|
|
1853
|
+
fullTmpFile = self.full_file_name(tmp_file,ext="tmp" if not ext=="tmp" else "_tmp")
|
|
1862
1854
|
while os.path.exists(fullTmpFile):
|
|
1863
1855
|
fullTmpFile = self.full_file_name(tmp_file) + "." + str(tmp_i) + ".tmp"
|
|
1864
1856
|
tmp_i += 1
|
|
@@ -1884,9 +1876,9 @@ class SubDir(object):
|
|
|
1884
1876
|
obj,
|
|
1885
1877
|
raise_on_error : bool = True,
|
|
1886
1878
|
*,
|
|
1887
|
-
version : str = None,
|
|
1888
|
-
ext : str = None,
|
|
1889
|
-
fmt : Format = None ) -> bool:
|
|
1879
|
+
version : str|None = None,
|
|
1880
|
+
ext : str|None = None,
|
|
1881
|
+
fmt : Format|None = None ) -> bool:
|
|
1890
1882
|
"""
|
|
1891
1883
|
Writes an object to file.
|
|
1892
1884
|
|
|
@@ -1917,17 +1909,17 @@ class SubDir(object):
|
|
|
1917
1909
|
raise_on_error : bool
|
|
1918
1910
|
If ``False``, this function will return ``False`` upon failure.
|
|
1919
1911
|
|
|
1920
|
-
version : str
|
|
1912
|
+
version : str | None, default ``None``
|
|
1921
1913
|
If not ``None``, specifies the version of the code which generated ``obj``.
|
|
1922
1914
|
This version will be written to the beginning of the file.
|
|
1923
1915
|
|
|
1924
|
-
ext : str
|
|
1916
|
+
ext : str | None, default ``None``
|
|
1925
1917
|
Extension, or list thereof if ``file`` is a list.
|
|
1926
1918
|
|
|
1927
1919
|
* Use ``None`` to use directory's default extension.
|
|
1928
1920
|
* Use ``"*"`` to use the extension implied by ``fmt``.
|
|
1929
1921
|
|
|
1930
|
-
fmt : :class:`cdxcore.subdir.Format`
|
|
1922
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
1931
1923
|
File format or ``None`` to use the directory's default.
|
|
1932
1924
|
Note that ``fmt`` cannot be a list even if ``file`` is.
|
|
1933
1925
|
Note that unless ``ext`` or the SubDir's extension is '*',
|
|
@@ -2027,7 +2019,7 @@ class SubDir(object):
|
|
|
2027
2019
|
return True
|
|
2028
2020
|
return self._write( writer=writer, file=file, obj=obj, raise_on_error=raise_on_error, ext=ext )
|
|
2029
2021
|
|
|
2030
|
-
def write_string( self, file : str, line : str, raise_on_error : bool = True, *, ext : str = None ) -> bool:
|
|
2022
|
+
def write_string( self, file : str, line : str, raise_on_error : bool = True, *, ext : str|None = None ) -> bool:
|
|
2031
2023
|
"""
|
|
2032
2024
|
Writes a line of text into a file.
|
|
2033
2025
|
|
|
@@ -2056,7 +2048,7 @@ class SubDir(object):
|
|
|
2056
2048
|
|
|
2057
2049
|
# -- iterate --
|
|
2058
2050
|
|
|
2059
|
-
def files(self, *, ext : str = None) -> list:
|
|
2051
|
+
def files(self, *, ext : str|None = None) -> list:
|
|
2060
2052
|
"""
|
|
2061
2053
|
Returns a list of files in this subdirectory with the current extension, or the specified extension.
|
|
2062
2054
|
|
|
@@ -2105,9 +2097,10 @@ class SubDir(object):
|
|
|
2105
2097
|
subdirs.append( entry.name )
|
|
2106
2098
|
return subdirs
|
|
2107
2099
|
|
|
2108
|
-
#
|
|
2100
|
+
# delete
|
|
2101
|
+
# ------
|
|
2109
2102
|
|
|
2110
|
-
def delete( self, file : str, raise_on_error: bool = False, *, ext : str = None ):
|
|
2103
|
+
def delete( self, file : str, raise_on_error: bool = False, *, ext : str|None = None ):
|
|
2111
2104
|
"""
|
|
2112
2105
|
Deletes ``file``.
|
|
2113
2106
|
|
|
@@ -2116,21 +2109,21 @@ class SubDir(object):
|
|
|
2116
2109
|
|
|
2117
2110
|
Parameters
|
|
2118
2111
|
----------
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2112
|
+
file :
|
|
2113
|
+
filename, or list of filenames
|
|
2114
|
+
|
|
2115
|
+
raise_on_error : bool, default ``False``
|
|
2116
|
+
If ``False``, do not throw :class:`KeyError` if file does not exist
|
|
2117
|
+
or another error occurs.
|
|
2118
|
+
|
|
2119
|
+
ext : str | None, default ``None``
|
|
2120
|
+
Extension, or list thereof if ``file`` is a list.
|
|
2121
|
+
|
|
2122
|
+
Use
|
|
2123
|
+
|
|
2124
|
+
* ``None`` for the directory default.
|
|
2125
|
+
* ``""`` to not use an automatic extension.
|
|
2126
|
+
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2134
2127
|
"""
|
|
2135
2128
|
# do not do anything if the object was deleted
|
|
2136
2129
|
if self._path is None:
|
|
@@ -2166,21 +2159,21 @@ class SubDir(object):
|
|
|
2166
2159
|
else:
|
|
2167
2160
|
os.remove(full_file_name)
|
|
2168
2161
|
|
|
2169
|
-
def delete_all_files( self, raise_on_error : bool = False, *, ext : str = None ):
|
|
2162
|
+
def delete_all_files( self, raise_on_error : bool = False, *, ext : str|None = None ):
|
|
2170
2163
|
"""
|
|
2171
2164
|
Deletes all valid keys in this sub directory with the correct extension.
|
|
2172
2165
|
|
|
2173
2166
|
Parameters
|
|
2174
2167
|
----------
|
|
2175
|
-
|
|
2176
|
-
|
|
2168
|
+
raise_on_error : bool
|
|
2169
|
+
Set to ``False`` to quietly ignore errors.
|
|
2170
|
+
|
|
2171
|
+
ext : str | None, default ``None``
|
|
2172
|
+
Extension to be used:
|
|
2177
2173
|
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
* ``None`` for the directory default.
|
|
2182
|
-
* ``""`` to not use an automatic extension.
|
|
2183
|
-
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2174
|
+
* ``None`` for the directory default.
|
|
2175
|
+
* ``""`` to not use an automatic extension.
|
|
2176
|
+
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2184
2177
|
"""
|
|
2185
2178
|
if self._path is None:
|
|
2186
2179
|
if raise_on_error: raise EOFError("Cannot delete all files: current directory not specified")
|
|
@@ -2189,7 +2182,7 @@ class SubDir(object):
|
|
|
2189
2182
|
return
|
|
2190
2183
|
self.delete( self.files(ext=ext), raise_on_error=raise_on_error, ext=ext )
|
|
2191
2184
|
|
|
2192
|
-
def delete_all_content( self, delete_self : bool = False, raise_on_error : bool = False, *, ext : str = None ):
|
|
2185
|
+
def delete_all_content( self, delete_self : bool = False, raise_on_error : bool = False, *, ext : str|None = None ):
|
|
2193
2186
|
"""
|
|
2194
2187
|
Deletes all valid keys and subdirectories in this sub directory.
|
|
2195
2188
|
|
|
@@ -2198,13 +2191,13 @@ class SubDir(object):
|
|
|
2198
2191
|
|
|
2199
2192
|
Parameters
|
|
2200
2193
|
----------
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2194
|
+
delete_self: bool
|
|
2195
|
+
Whether to delete the directory itself as well, or only its contents.
|
|
2196
|
+
raise_on_error: bool
|
|
2197
|
+
``False`` for silent failure
|
|
2198
|
+
ext : str | None, default ``None``
|
|
2199
|
+
Extension for keys, or ``None`` for the directory's default.
|
|
2200
|
+
Use ``""`` to match all files regardless of extension.
|
|
2208
2201
|
"""
|
|
2209
2202
|
# do not do anything if the object was deleted
|
|
2210
2203
|
if self._path is None:
|
|
@@ -2249,30 +2242,29 @@ class SubDir(object):
|
|
|
2249
2242
|
elif keep_directory and not os.path.exists(self._path[:-1]):
|
|
2250
2243
|
os.makedirs(self._path[:-1])
|
|
2251
2244
|
|
|
2252
|
-
#
|
|
2245
|
+
# file ops
|
|
2246
|
+
# --------
|
|
2253
2247
|
|
|
2254
|
-
def exists(self, file : str, *, ext : str = None ) -> bool:
|
|
2248
|
+
def exists(self, file : str, *, ext : str|None = None ) -> bool:
|
|
2255
2249
|
"""
|
|
2256
2250
|
Checks whether a file exists.
|
|
2257
2251
|
|
|
2258
2252
|
Parameters
|
|
2259
2253
|
----------
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
Use
|
|
2254
|
+
file :
|
|
2255
|
+
Filename, or list of filenames.
|
|
2256
|
+
|
|
2257
|
+
ext : str | None, default ``None``
|
|
2258
|
+
Extension to be used:
|
|
2267
2259
|
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2260
|
+
* ``None`` for the directory default.
|
|
2261
|
+
* ``""`` to not use an automatic extension.
|
|
2262
|
+
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2271
2263
|
|
|
2272
2264
|
Returns
|
|
2273
2265
|
-------
|
|
2274
|
-
|
|
2275
|
-
|
|
2266
|
+
Status : bool
|
|
2267
|
+
If ``file`` is a string, returns ``True`` or ``False``, else it will return a list of ``bool`` values.
|
|
2276
2268
|
"""
|
|
2277
2269
|
# vector version
|
|
2278
2270
|
if not isinstance(file,str):
|
|
@@ -2330,7 +2322,7 @@ class SubDir(object):
|
|
|
2330
2322
|
return None
|
|
2331
2323
|
return func(full_file_name)
|
|
2332
2324
|
|
|
2333
|
-
def get_creation_time( self, file : str, *, ext : str = None ) -> datetime.datetime:
|
|
2325
|
+
def get_creation_time( self, file : str, *, ext : str|None = None ) -> datetime.datetime:
|
|
2334
2326
|
"""
|
|
2335
2327
|
Returns the creation time of a file.
|
|
2336
2328
|
|
|
@@ -2338,21 +2330,21 @@ class SubDir(object):
|
|
|
2338
2330
|
|
|
2339
2331
|
Parameters
|
|
2340
2332
|
----------
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2333
|
+
file :
|
|
2334
|
+
Filename, or list of filenames.
|
|
2335
|
+
|
|
2336
|
+
ext : str | None, default ``None``
|
|
2337
|
+
Extension to be used:
|
|
2338
|
+
|
|
2339
|
+
* ``None`` for the directory default.
|
|
2340
|
+
* ``""`` to not use an automatic extension.
|
|
2341
|
+
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2350
2342
|
|
|
2351
2343
|
Returns
|
|
2352
2344
|
-------
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2345
|
+
Datetime : :class:`datetime.datetime`
|
|
2346
|
+
A single ``datetime`` if ``file`` is a string, otherwise a list of ``datetime``'s.
|
|
2347
|
+
Returns ``None`` if an error occured.
|
|
2356
2348
|
"""
|
|
2357
2349
|
return self._getFileProperty( file=file, ext=ext, func=lambda x : datetime.datetime.fromtimestamp(os.path.getctime(x)) )
|
|
2358
2350
|
|
|
@@ -2364,21 +2356,21 @@ class SubDir(object):
|
|
|
2364
2356
|
|
|
2365
2357
|
Parameters
|
|
2366
2358
|
----------
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2359
|
+
file :
|
|
2360
|
+
Filename, or list of filenames.
|
|
2361
|
+
|
|
2362
|
+
ext : str | None, default ``None``
|
|
2363
|
+
Extension to be used:
|
|
2364
|
+
|
|
2365
|
+
* ``None`` for the directory default.
|
|
2366
|
+
* ``""`` to not use an automatic extension.
|
|
2367
|
+
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2376
2368
|
|
|
2377
2369
|
Returns
|
|
2378
2370
|
-------
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2371
|
+
Datetime : :class:`datetime.datetime`
|
|
2372
|
+
A single ``datetime`` if ``file`` is a string, otherwise a list of ``datetime``'s.
|
|
2373
|
+
Returns ``None`` if an error occured.
|
|
2382
2374
|
"""
|
|
2383
2375
|
return self._getFileProperty( file=file, ext=ext, func=lambda x : datetime.datetime.fromtimestamp(os.path.getmtime(x)) )
|
|
2384
2376
|
|
|
@@ -2390,20 +2382,21 @@ class SubDir(object):
|
|
|
2390
2382
|
|
|
2391
2383
|
Parameters
|
|
2392
2384
|
----------
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2385
|
+
file :
|
|
2386
|
+
Filename, or list of filenames.
|
|
2387
|
+
|
|
2388
|
+
ext : str | None, default ``None``
|
|
2389
|
+
Extension to be used:
|
|
2390
|
+
|
|
2391
|
+
* ``None`` for the directory default.
|
|
2392
|
+
* ``""`` to not use an automatic extension.
|
|
2393
|
+
* ``"*"`` to use the extension associated with the format of the directory.
|
|
2401
2394
|
|
|
2402
2395
|
Returns
|
|
2403
2396
|
-------
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2397
|
+
Datetime : :class:`datetime.datetime`
|
|
2398
|
+
A single ``datetime`` if ``file`` is a string, otherwise a list of ``datetime``'s.
|
|
2399
|
+
Returns ``None`` if an error occured.
|
|
2407
2400
|
"""
|
|
2408
2401
|
return self._getFileProperty( file=file, ext=ext, func=lambda x : datetime.datetime.fromtimestamp(os.path.getatime(x)) )
|
|
2409
2402
|
|
|
@@ -2475,6 +2468,75 @@ class SubDir(object):
|
|
|
2475
2468
|
os.rename(src_full, tar_full)
|
|
2476
2469
|
|
|
2477
2470
|
# utilities
|
|
2471
|
+
# ---------
|
|
2472
|
+
|
|
2473
|
+
def temp_file_name( self, file : str|None = None ):
|
|
2474
|
+
"""
|
|
2475
|
+
Returns a unique temporary file name.
|
|
2476
|
+
|
|
2477
|
+
The file name is generated by applying a unique hash
|
|
2478
|
+
to the current directory, ``file``, the current process and thread IDs, and
|
|
2479
|
+
:func:`datetime.datetime.now`.
|
|
2480
|
+
|
|
2481
|
+
If ``file`` is not ``None`` it will be used as a label.
|
|
2482
|
+
|
|
2483
|
+
This function returns just the file name. Use :meth:`cdxcore.subdir.SubDir.full_temp_file_name`
|
|
2484
|
+
to get a full temporary file name including path and extension.
|
|
2485
|
+
|
|
2486
|
+
Parameters
|
|
2487
|
+
----------
|
|
2488
|
+
file : str | None, default ``None``
|
|
2489
|
+
An optional file. If provided, :func:`cdxcore.uniquehash.named_unique_filename48_8`
|
|
2490
|
+
is used to generate the temporary file which means that a portion of ``file``
|
|
2491
|
+
will head the returned temporary name.
|
|
2492
|
+
|
|
2493
|
+
If ``file`` is ``None``, :func:`cdxcore.uniquehash.unique_hash48`
|
|
2494
|
+
is used to generate a 48 character hash.
|
|
2495
|
+
|
|
2496
|
+
Returns
|
|
2497
|
+
-------
|
|
2498
|
+
Temporary file name : str
|
|
2499
|
+
The file name.
|
|
2500
|
+
"""
|
|
2501
|
+
if file is None or file=="":
|
|
2502
|
+
return unique_hash48( str(self), file, uuid.getnode(), os.getpid(), threading.get_ident(), datetime.datetime.now() )
|
|
2503
|
+
else:
|
|
2504
|
+
return named_unique_filename48_8( file, str(self), uuid.getnode(), os.getpid(), threading.get_ident(), datetime.datetime.now() )
|
|
2505
|
+
|
|
2506
|
+
def full_temp_file_name(self, file : str|None = None, *, ext : str | None = None, create_directory : bool = False ):
|
|
2507
|
+
"""
|
|
2508
|
+
Returns a fully qualified unique temporary file name with path and extension
|
|
2509
|
+
|
|
2510
|
+
The file name is generated by applying a unique hash
|
|
2511
|
+
to the current directory, ``file``, the current process and thread IDs, and
|
|
2512
|
+
:func:`datetime.datetime.now`.
|
|
2513
|
+
|
|
2514
|
+
If ``file`` is not ``None`` it will be used as a label.
|
|
2515
|
+
|
|
2516
|
+
This function returns the fully qualified file name. Use :meth:`cdxcore.subdir.SubDir.temp_file_name`
|
|
2517
|
+
to only a file name.
|
|
2518
|
+
|
|
2519
|
+
Parameters
|
|
2520
|
+
----------
|
|
2521
|
+
file : str | None, default ``None``
|
|
2522
|
+
An optional file. If provided, :func:`cdxcore.uniquehash.named_unique_filename48_8`
|
|
2523
|
+
is used to generate the temporary file which means that a portion of ``file``
|
|
2524
|
+
will head the returned temporary name.
|
|
2525
|
+
|
|
2526
|
+
If ``file`` is ``None``, :func:`cdxcore.uniquehash.unique_hash48`
|
|
2527
|
+
is used to generate a 48 character hash.
|
|
2528
|
+
|
|
2529
|
+
ext : str | None, default ``None``
|
|
2530
|
+
Extension to use, or ``None`` for the extrension of ``self``.
|
|
2531
|
+
|
|
2532
|
+
Returns
|
|
2533
|
+
-------
|
|
2534
|
+
Temporary file name : str
|
|
2535
|
+
The fully qualified file name.
|
|
2536
|
+
"""
|
|
2537
|
+
if create_directory:
|
|
2538
|
+
self.create_directory()
|
|
2539
|
+
return self.full_file_name( self.temp_file_name(file), ext=ext )
|
|
2478
2540
|
|
|
2479
2541
|
@staticmethod
|
|
2480
2542
|
def remove_bad_file_characters( file : str, by : str="default" ) -> str:
|
|
@@ -2511,17 +2573,18 @@ class SubDir(object):
|
|
|
2511
2573
|
uqf = UniqueLabel( max_length=max_length-len_ext, id_length=id_length, separator=separator )
|
|
2512
2574
|
return uqf( unique_filename )
|
|
2513
2575
|
|
|
2514
|
-
#
|
|
2576
|
+
# object interface
|
|
2577
|
+
# ----------------
|
|
2515
2578
|
|
|
2516
2579
|
def __call__(self, element : str,
|
|
2517
2580
|
default = RETURN_SUB_DIRECTORY,
|
|
2518
2581
|
raise_on_error : bool = False,
|
|
2519
2582
|
*,
|
|
2520
|
-
version : str = None,
|
|
2521
|
-
ext : str = None,
|
|
2522
|
-
fmt : Format = None,
|
|
2583
|
+
version : str|None = None,
|
|
2584
|
+
ext : str|None = None,
|
|
2585
|
+
fmt : Format|None = None,
|
|
2523
2586
|
delete_wrong_version : bool = True,
|
|
2524
|
-
create_directory : bool = None ):
|
|
2587
|
+
create_directory : bool|None = None ):
|
|
2525
2588
|
"""
|
|
2526
2589
|
Read either data from a file, or return a new sub directory.
|
|
2527
2590
|
|
|
@@ -2561,20 +2624,18 @@ class SubDir(object):
|
|
|
2561
2624
|
If ``default`` is not specified, then this function returns a new sub-directory by calling
|
|
2562
2625
|
``SubDir(element,parent=self,ext=ext,fmt=fmt)``.
|
|
2563
2626
|
|
|
2564
|
-
create_directory : bool,
|
|
2627
|
+
create_directory : bool, default ``None``
|
|
2565
2628
|
*When creating sub-directories:*
|
|
2566
2629
|
|
|
2567
2630
|
Whether or not to instantly create the sub-directory. The default, ``None``, is to inherit the behaviour from ``self``.
|
|
2568
2631
|
|
|
2569
|
-
raise_on_error : bool,
|
|
2632
|
+
raise_on_error : bool, default ``False``
|
|
2570
2633
|
*When reading files:*
|
|
2571
2634
|
|
|
2572
2635
|
Whether to raise an exception if reading an existing file failed.
|
|
2573
2636
|
By default this function fails silently and returns ``default``.
|
|
2574
2637
|
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
version : str, optional
|
|
2638
|
+
version : str | None, default ``None``
|
|
2578
2639
|
*When reading files:*
|
|
2579
2640
|
|
|
2580
2641
|
If not ``None``, specifies the version of the current code base.
|
|
@@ -2586,17 +2647,13 @@ class SubDir(object):
|
|
|
2586
2647
|
Note that this is distinct
|
|
2587
2648
|
to using ``None`` which stipulates that the file should not
|
|
2588
2649
|
have version information.
|
|
2589
|
-
|
|
2590
|
-
Default is ``None``.
|
|
2591
2650
|
|
|
2592
|
-
delete_wrong_version : bool,
|
|
2651
|
+
delete_wrong_version : bool, default ``True``.
|
|
2593
2652
|
*When reading files:*
|
|
2594
2653
|
|
|
2595
2654
|
If ``True``, and if a wrong version was found, delete the file.
|
|
2596
2655
|
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
ext : str, optional
|
|
2656
|
+
ext : str | None, default is ``None``.
|
|
2600
2657
|
*When reading files:*
|
|
2601
2658
|
|
|
2602
2659
|
Extension to be used, or a list thereof if ``element`` is a list. Defaults
|
|
@@ -2611,11 +2668,8 @@ class SubDir(object):
|
|
|
2611
2668
|
*When creating sub-directories:*
|
|
2612
2669
|
|
|
2613
2670
|
Extension for the new subdirectory; set to ``None`` to inherit the parent's extension.
|
|
2614
|
-
|
|
2615
|
-
Default is ``None``.
|
|
2616
2671
|
|
|
2617
|
-
|
|
2618
|
-
fmt : :class:`cdxcore.subdir.Format`, optional
|
|
2672
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
2619
2673
|
*When reading files:*
|
|
2620
2674
|
|
|
2621
2675
|
File format or ``None`` to use the directory's default.
|
|
@@ -2627,12 +2681,10 @@ class SubDir(object):
|
|
|
2627
2681
|
*When creating sub-directories:*
|
|
2628
2682
|
|
|
2629
2683
|
Format for the new sub-directory; set to ``None`` to inherit the parent's format.
|
|
2630
|
-
|
|
2631
|
-
Default is ``None``.
|
|
2632
2684
|
|
|
2633
2685
|
Returns
|
|
2634
2686
|
-------
|
|
2635
|
-
Object : type|SubDir
|
|
2687
|
+
Object : type | SubDir
|
|
2636
2688
|
Either the value in the file, a new sub directory, or lists thereof.
|
|
2637
2689
|
"""
|
|
2638
2690
|
if default == SubDir.RETURN_SUB_DIRECTORY:
|
|
@@ -2676,7 +2728,7 @@ class SubDir(object):
|
|
|
2676
2728
|
""" Tests whether ``file`` :meth:`cdxcore.subdir.SubDir.exists`. """
|
|
2677
2729
|
return self.exists(file)
|
|
2678
2730
|
|
|
2679
|
-
def items(self, *, ext : str = None, raise_on_error : bool = False) -> Iterable:
|
|
2731
|
+
def items(self, *, ext : str|None = None, raise_on_error : bool = False) -> Iterable:
|
|
2680
2732
|
"""
|
|
2681
2733
|
Dictionary-style iterable of filenames and their content.
|
|
2682
2734
|
|
|
@@ -2688,7 +2740,7 @@ class SubDir(object):
|
|
|
2688
2740
|
|
|
2689
2741
|
Parameters
|
|
2690
2742
|
----------
|
|
2691
|
-
ext : str
|
|
2743
|
+
ext : str | None, default ``None``
|
|
2692
2744
|
Extension or ``None`` for the directory's current extension. Use ``""``
|
|
2693
2745
|
for all file extension.
|
|
2694
2746
|
|
|
@@ -2746,7 +2798,6 @@ class SubDir(object):
|
|
|
2746
2798
|
raise LookupError(f"Unknown format name '{format_name}'. Must be one of: {fmt_list(SubDir.FORMAT_NAMES)}")
|
|
2747
2799
|
return Format[format_name]
|
|
2748
2800
|
|
|
2749
|
-
|
|
2750
2801
|
# caching
|
|
2751
2802
|
# -------
|
|
2752
2803
|
|
|
@@ -3327,8 +3378,8 @@ class SubDir(object):
|
|
|
3327
3378
|
# ========================================================================
|
|
3328
3379
|
|
|
3329
3380
|
def VersionedCacheRoot( directory : str, *,
|
|
3330
|
-
ext : str = None,
|
|
3331
|
-
fmt : Format = None,
|
|
3381
|
+
ext : str|None = None,
|
|
3382
|
+
fmt : Format|None = None,
|
|
3332
3383
|
create_directory : bool = False,
|
|
3333
3384
|
**controller_kwargs
|
|
3334
3385
|
):
|
|
@@ -3360,43 +3411,43 @@ def VersionedCacheRoot( directory : str, *,
|
|
|
3360
3411
|
|
|
3361
3412
|
Parameters
|
|
3362
3413
|
----------
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
* ``"!/dir"`` creates ``dir`` in the temporary directory.
|
|
3369
|
-
* ``"~/dir"`` creates ``dir`` in the home directory.
|
|
3370
|
-
* ``"./dir"`` creates ``dir`` relative to the current directory.
|
|
3371
|
-
|
|
3372
|
-
ext : str
|
|
3373
|
-
Extension, which will automatically be appended to file names.
|
|
3374
|
-
The default value depends on ``fmt`; for ``Format.PICKLE`` it is "pck".
|
|
3414
|
+
directory : str
|
|
3415
|
+
Name of the root directory for caching.
|
|
3416
|
+
|
|
3417
|
+
Using SubDir the following Short-cuts are supported:
|
|
3375
3418
|
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3419
|
+
* ``"!/dir"`` creates ``dir`` in the temporary directory.
|
|
3420
|
+
* ``"~/dir"`` creates ``dir`` in the home directory.
|
|
3421
|
+
* ``"./dir"`` creates ``dir`` relative to the current directory.
|
|
3422
|
+
|
|
3423
|
+
ext : str | None, default ``None``
|
|
3424
|
+
Extension, which will automatically be appended to file names.
|
|
3425
|
+
The default value depends on ``fmt`; for ``Format.PICKLE`` it is "pck".
|
|
3426
|
+
|
|
3427
|
+
fmt : :class:`cdxcore.subdir.Format` | None, default ``None``
|
|
3428
|
+
File format; if ``ext`` is not specified, the format drives the extension, too.
|
|
3429
|
+
The default ``None`` becomes ``Format.PICKLE``.
|
|
3379
3430
|
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
* ``exclude_arg_types``: list of types or names of types to exclude when auto-generating function
|
|
3389
|
-
signatures from function arguments.
|
|
3390
|
-
An example is :class:`cdxcore.verbose.Context` which is used to print progress messages.
|
|
3391
|
-
|
|
3392
|
-
* ``max_filename_length``: maximum filename length.
|
|
3431
|
+
create_directory : bool, default ``False``
|
|
3432
|
+
Whether to create the directory upon creation.
|
|
3433
|
+
|
|
3434
|
+
controller_kwargs: dict
|
|
3435
|
+
Parameters passed to :class:`cdxcore.subdir.CacheController``.
|
|
3436
|
+
|
|
3437
|
+
Common parameters used:
|
|
3393
3438
|
|
|
3394
|
-
|
|
3439
|
+
* ``exclude_arg_types``: list of types or names of types to exclude when auto-generating function
|
|
3440
|
+
signatures from function arguments.
|
|
3441
|
+
An example is :class:`cdxcore.verbose.Context` which is used to print progress messages.
|
|
3442
|
+
|
|
3443
|
+
* ``max_filename_length``: maximum filename length.
|
|
3444
|
+
|
|
3445
|
+
* ``hash_length``: length used for hashes, see :class:`cdxcore.uniquehash.UniqueHash`.
|
|
3395
3446
|
|
|
3396
3447
|
Returns
|
|
3397
3448
|
-------
|
|
3398
|
-
|
|
3399
|
-
|
|
3449
|
+
Root : :class:`cdxcore.subdir.SubDir`
|
|
3450
|
+
A root directory suitable for caching.
|
|
3400
3451
|
"""
|
|
3401
3452
|
controller = CacheController(**controller_kwargs) if len(controller_kwargs) > 0 else None
|
|
3402
3453
|
return SubDir( directory, ext=ext, fmt=fmt, create_directory=create_directory, cache_controller=controller )
|
cdxcore/uniquehash.py
CHANGED
|
@@ -1120,5 +1120,5 @@ def named_unique_filename48_8( label : str, *args, **kwargs ) -> str:
|
|
|
1120
1120
|
discussed in :class:`cdxcore.uniquehash.UniqueHash` around
|
|
1121
1121
|
elements starting with `_` or function members.
|
|
1122
1122
|
"""
|
|
1123
|
-
return NamedUniqueHash( max_length=48, id_length=8, filename_by="default" )
|
|
1123
|
+
return NamedUniqueHash( max_length=48, id_length=8, filename_by="default" )(label, *args, **kwargs)
|
|
1124
1124
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
cdxcore/__init__.py,sha256=
|
|
1
|
+
cdxcore/__init__.py,sha256=fkVQQ_XdGdXhWA35RMrTjwmBD2jNYOcKPhNKMgPF8Vw,127
|
|
2
2
|
cdxcore/config.py,sha256=RaxpAnTEyGXLiq-1O2prXFBb2sdyqB6hHDHPAFGe2k0,98472
|
|
3
3
|
cdxcore/deferred.py,sha256=5bREI2IQ3uW3EMDcAO17gJbVnbUIV__-bdEUQpZIEeU,44684
|
|
4
4
|
cdxcore/dynalimits.py,sha256=TXEwa4wCdeM5epG1ceezaHwvmhnbU-oXlXpHNbOMQR8,11293
|
|
@@ -9,12 +9,12 @@ cdxcore/jcpool.py,sha256=Vw8o8S4_qTVQNIr2sRaBR_kHz_wO0zpYs0QjlKSYFz8,27280
|
|
|
9
9
|
cdxcore/npio.py,sha256=SVpKkFt6lyRogkns-oky0wEiWgVTalgB0OdaSvyWtlU,24212
|
|
10
10
|
cdxcore/npshm.py,sha256=9buYPNJoNlw69NZQ-nLF13PEWBWNx51a0gjQw5Gc24U,18368
|
|
11
11
|
cdxcore/pretty.py,sha256=FsI62rlaqRX1E-uPCSnu0M4UoCQ5Z55TDvYPnzTNO70,17220
|
|
12
|
-
cdxcore/subdir.py,sha256=
|
|
13
|
-
cdxcore/uniquehash.py,sha256=
|
|
12
|
+
cdxcore/subdir.py,sha256=QnFgWq464KtoFXedAuo1r5PHvsN3whDs_WA_vXf5BdQ,179560
|
|
13
|
+
cdxcore/uniquehash.py,sha256=WCtieiRBavfpSVT-hW4QNYGHFrfgXG3kJuTUtAi9_b0,50729
|
|
14
14
|
cdxcore/util.py,sha256=dqCEhrDvUM4qjeUwb6n8jPfS8WC4Navcj83gDjXfRFE,39349
|
|
15
15
|
cdxcore/verbose.py,sha256=vsjGTVnAHMPg2L2RfsowWKKPjUSnQJ3F653vDTydBkI,30223
|
|
16
16
|
cdxcore/version.py,sha256=S3H0ktEZsM0zAj3EQ5AdrZ2KxWhQtemd8clzEFE9hOQ,27160
|
|
17
|
-
cdxcore-0.1.
|
|
17
|
+
cdxcore-0.1.26.dist-info/licenses/LICENSE,sha256=M-cisgK9kb1bqVRJ7vrCxHcMQQfDxdY3c2YFJJWfNQg,1090
|
|
18
18
|
docs/source/conf.py,sha256=yn3LYgw3sT45mUyll-B2emVp6jg7H6KfAHOcBg_MNv4,4182
|
|
19
19
|
tests/test_config.py,sha256=N86mH3y7k3LXEmU8uPLfrmRMZ-80VhlD35nBbpLmebg,15617
|
|
20
20
|
tests/test_deferred.py,sha256=4Xsb76r-XqHKiBuHa4jbErjMWbrgHXfPwewzzY4lf9Y,7922
|
|
@@ -34,7 +34,7 @@ tmp/npsh1.py,sha256=mNucUl2-jNmE84GlMlliB4aJ0UQ9FqdymgcY_9mLeZY,15432
|
|
|
34
34
|
tmp/sharedarray.py,sha256=dNOT1ObCc3nM3qA3OA508NcENIBnkmWMxRPCqvMVa8A,12862
|
|
35
35
|
up/git_message.py,sha256=EfSH7Pit3ZoCiRqSMwRCUN_QyuwreU4LTIyGSutBlm4,123
|
|
36
36
|
up/pip_modify_setup.py,sha256=Esaml4yA9tFsqxLhk5bWSwvKCURONjQqfyChgFV2TSY,1584
|
|
37
|
-
cdxcore-0.1.
|
|
38
|
-
cdxcore-0.1.
|
|
39
|
-
cdxcore-0.1.
|
|
40
|
-
cdxcore-0.1.
|
|
37
|
+
cdxcore-0.1.26.dist-info/METADATA,sha256=YT7I_dWXka3a7RSKznr_zZO-9WC_LFmMy0DkyGSDJdE,5939
|
|
38
|
+
cdxcore-0.1.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
39
|
+
cdxcore-0.1.26.dist-info/top_level.txt,sha256=phNSwCyJFe7UP2YMoi8o6ykhotatlIbJHjTp9EHM51k,26
|
|
40
|
+
cdxcore-0.1.26.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|