tensorneko-util 0.3.16__py3-none-any.whl → 0.3.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tensorneko_util/io/_path_conversion.py +25 -0
- tensorneko_util/io/audio/audio_reader.py +7 -4
- tensorneko_util/io/audio/audio_writer.py +9 -5
- tensorneko_util/io/hdf5/hdf5_reader.py +9 -3
- tensorneko_util/io/hdf5/hdf5_writer.py +4 -1
- tensorneko_util/io/image/image_reader.py +7 -4
- tensorneko_util/io/image/image_writer.py +13 -8
- tensorneko_util/io/json/json_reader.py +12 -7
- tensorneko_util/io/json/json_writer.py +8 -3
- tensorneko_util/io/matlab/mat_reader.py +8 -4
- tensorneko_util/io/matlab/mat_writer.py +8 -4
- tensorneko_util/io/npy/npy_reader.py +14 -8
- tensorneko_util/io/npy/npy_writer.py +19 -10
- tensorneko_util/io/pickle/pickle_reader.py +9 -3
- tensorneko_util/io/pickle/pickle_writer.py +9 -3
- tensorneko_util/io/reader.py +6 -2
- tensorneko_util/io/text/text_reader.py +11 -3
- tensorneko_util/io/text/text_writer.py +11 -3
- tensorneko_util/io/toml/toml_reader.py +9 -3
- tensorneko_util/io/toml/toml_writer.py +9 -3
- tensorneko_util/io/video/video_reader.py +15 -8
- tensorneko_util/io/video/video_writer.py +11 -6
- tensorneko_util/io/writer.py +6 -2
- tensorneko_util/io/yaml/yaml_reader.py +8 -4
- tensorneko_util/io/yaml/yaml_writer.py +9 -6
- tensorneko_util/msg/gotify.py +2 -2
- tensorneko_util/notebook/__init__.py +2 -1
- tensorneko_util/notebook/animation.py +20 -0
- tensorneko_util/notebook/display.py +6 -1
- tensorneko_util/util/misc.py +5 -1
- tensorneko_util/version.txt +1 -1
- tensorneko_util/visualization/watcher/web/dist/assets/{index.ae750248.js → index.3222e532.js} +47 -47
- tensorneko_util/visualization/watcher/web/dist/index.html +1 -1
- {tensorneko_util-0.3.16.dist-info → tensorneko_util-0.3.18.dist-info}/METADATA +1 -1
- {tensorneko_util-0.3.16.dist-info → tensorneko_util-0.3.18.dist-info}/RECORD +38 -36
- {tensorneko_util-0.3.16.dist-info → tensorneko_util-0.3.18.dist-info}/LICENSE +0 -0
- {tensorneko_util-0.3.16.dist-info → tensorneko_util-0.3.18.dist-info}/WHEEL +0 -0
- {tensorneko_util-0.3.16.dist-info → tensorneko_util-0.3.18.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _path2str(path: Union[str, Path]) -> str:
|
|
6
|
+
"""
|
|
7
|
+
Convert a path to a string.
|
|
8
|
+
|
|
9
|
+
Args:
|
|
10
|
+
path (``Union[str, Path]``): The path to be converted.
|
|
11
|
+
|
|
12
|
+
Returns:
|
|
13
|
+
``str``: The string of the path.
|
|
14
|
+
|
|
15
|
+
Examples::
|
|
16
|
+
|
|
17
|
+
>>> _path2str("a/b/c")
|
|
18
|
+
'a/b/c'
|
|
19
|
+
>>> _path2str(Path("a/b/c"))
|
|
20
|
+
'a/b/c'
|
|
21
|
+
|
|
22
|
+
"""
|
|
23
|
+
if isinstance(path, Path):
|
|
24
|
+
return str(path)
|
|
25
|
+
return path
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
from typing import Optional
|
|
1
|
+
from typing import Optional, Union
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import numpy as np
|
|
4
5
|
|
|
5
6
|
from .audio_data import AudioData
|
|
6
7
|
from .._default_backends import _default_audio_io_backend
|
|
8
|
+
from .._path_conversion import _path2str
|
|
7
9
|
from ...backend.audio_lib import AudioLib
|
|
8
10
|
|
|
9
11
|
|
|
@@ -11,12 +13,12 @@ class AudioReader:
|
|
|
11
13
|
"""AudioReader for reading audio file"""
|
|
12
14
|
|
|
13
15
|
@staticmethod
|
|
14
|
-
def of(path: str, channel_first: bool = True, backend: Optional[AudioLib] = None) -> AudioData:
|
|
16
|
+
def of(path: Union[str, Path], channel_first: bool = True, backend: Optional[AudioLib] = None) -> AudioData:
|
|
15
17
|
"""
|
|
16
18
|
Read audio tensor from given file.
|
|
17
19
|
|
|
18
20
|
Args:
|
|
19
|
-
path (``str``): Path to the audio file.
|
|
21
|
+
path (``str`` | ``pathlib.Path``): Path to the audio file.
|
|
20
22
|
channel_first (``bool``, optional): Whether the audio is channel first. The output shape is (C, T) if true
|
|
21
23
|
and (T, C) if false. Default: True.
|
|
22
24
|
backend (:class:`~tensorneko.io.audio.audio_lib.AudioLib`, optional): The audio library to use.
|
|
@@ -26,6 +28,7 @@ class AudioReader:
|
|
|
26
28
|
:class:`~.audio_data.AudioData`: The Audio data in the file.
|
|
27
29
|
"""
|
|
28
30
|
backend = backend or _default_audio_io_backend()
|
|
31
|
+
path = _path2str(path)
|
|
29
32
|
|
|
30
33
|
if backend == AudioLib.PYTORCH:
|
|
31
34
|
if not AudioLib.pytorch_available():
|
|
@@ -62,6 +65,6 @@ class AudioReader:
|
|
|
62
65
|
else:
|
|
63
66
|
raise ValueError("Unknown audio library: {}".format(backend))
|
|
64
67
|
|
|
65
|
-
def __new__(cls, path: str, channel_first: bool = True, backend: Optional[AudioLib] = None) -> AudioData:
|
|
68
|
+
def __new__(cls, path: Union[str, Path], channel_first: bool = True, backend: Optional[AudioLib] = None) -> AudioData:
|
|
66
69
|
"""Alias of :meth:`~AudioReader.of`"""
|
|
67
70
|
return cls.of(path, channel_first, backend)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
from typing import Union
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
from .audio_data import AudioData
|
|
4
5
|
from .._default_backends import _default_audio_io_backend
|
|
6
|
+
from .._path_conversion import _path2str
|
|
5
7
|
from ...backend.audio_lib import AudioLib
|
|
6
8
|
from ...util import dispatch
|
|
7
9
|
from ...util.type import T_ARRAY
|
|
@@ -12,30 +14,31 @@ class AudioWriter:
|
|
|
12
14
|
|
|
13
15
|
@classmethod
|
|
14
16
|
@dispatch
|
|
15
|
-
def to(cls, path: str, audio: AudioData, channel_first: bool = True, backend: AudioLib = None) -> None:
|
|
17
|
+
def to(cls, path: Union[str, Path], audio: AudioData, channel_first: bool = True, backend: AudioLib = None) -> None:
|
|
16
18
|
"""
|
|
17
19
|
Save wav file from :class:`~tensorneko.io.audio.audio_data.AudioData`.
|
|
18
20
|
|
|
19
21
|
Args:
|
|
20
|
-
path (``str``): The path of output file.
|
|
22
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
21
23
|
audio (:class:`~tensorneko.io.audio.AudioData`): The AudioData object for output.
|
|
22
24
|
channel_first (``bool``, optional): Whether the audio is channel first. The input shape is (C, T) if true
|
|
23
25
|
and (T, C) if false. Default: True.
|
|
24
26
|
backend (:class:`~tensorneko.io.audio.audio_lib.AudioLib`, optional): The audio library to use.
|
|
25
27
|
Default: pytorch.
|
|
26
28
|
"""
|
|
29
|
+
path = _path2str(path)
|
|
27
30
|
return cls.to(path, audio.audio, audio.sample_rate, channel_first, backend)
|
|
28
31
|
|
|
29
32
|
@classmethod
|
|
30
33
|
@dispatch
|
|
31
|
-
def to(cls, path: str, audio: T_ARRAY, sample_rate: int = 16000, channel_first: bool = True,
|
|
34
|
+
def to(cls, path: Union[str, Path], audio: T_ARRAY, sample_rate: int = 16000, channel_first: bool = True,
|
|
32
35
|
backend: AudioLib = None
|
|
33
36
|
):
|
|
34
37
|
"""
|
|
35
38
|
Save wav file from :class:`~torch.Tensor` or :class:`~numpy.ndarray` with (C, T).
|
|
36
39
|
|
|
37
40
|
Args:
|
|
38
|
-
path (``str``): The path of output file.
|
|
41
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
39
42
|
audio (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The tensor or array of audio.
|
|
40
43
|
sample_rate (``int``, optional): The sample rate of the audio. Default: 16000.
|
|
41
44
|
channel_first (``bool``, optional): Whether the audio is channel first. The input shape is (C, T) if true
|
|
@@ -44,6 +47,7 @@ class AudioWriter:
|
|
|
44
47
|
Default: pytorch.
|
|
45
48
|
"""
|
|
46
49
|
backend = backend or _default_audio_io_backend()
|
|
50
|
+
path = _path2str(path)
|
|
47
51
|
|
|
48
52
|
if backend == AudioLib.PYTORCH:
|
|
49
53
|
if not AudioLib.pytorch_available():
|
|
@@ -53,6 +57,6 @@ class AudioWriter:
|
|
|
53
57
|
else:
|
|
54
58
|
raise ValueError("Unknown audio library: {}".format(backend))
|
|
55
59
|
|
|
56
|
-
def __new__(cls, path: str, audio: Union[AudioData, T_ARRAY], *args, **kwargs):
|
|
60
|
+
def __new__(cls, path: Union[str, Path], audio: Union[AudioData, T_ARRAY], *args, **kwargs):
|
|
57
61
|
"""Alias to :meth:`~AudioWriter.to`"""
|
|
58
62
|
return cls.to(path, audio, *args, **kwargs)
|
|
@@ -1,21 +1,27 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
1
4
|
import h5py
|
|
2
5
|
|
|
6
|
+
from .._path_conversion import _path2str
|
|
7
|
+
|
|
3
8
|
|
|
4
9
|
class Hdf5Reader:
|
|
5
10
|
|
|
6
11
|
@classmethod
|
|
7
|
-
def of(cls, path: str) -> h5py.File:
|
|
12
|
+
def of(cls, path: Union[str, Path]) -> h5py.File:
|
|
8
13
|
"""
|
|
9
14
|
Open a hdf5 file.
|
|
10
15
|
|
|
11
16
|
Args:
|
|
12
|
-
path (``str``): Path to the hdf5 file.
|
|
17
|
+
path (``str`` | ``pathlib.Path``): Path to the hdf5 file.
|
|
13
18
|
|
|
14
19
|
Returns:
|
|
15
20
|
:class:`h5py.File`: The opened hdf5 file.
|
|
16
21
|
|
|
17
22
|
"""
|
|
23
|
+
path = _path2str(path)
|
|
18
24
|
return h5py.File(path, "r")
|
|
19
25
|
|
|
20
|
-
def __new__(cls, path: str):
|
|
26
|
+
def __new__(cls, path: Union[str, Path]):
|
|
21
27
|
return cls.of(path)
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
1
4
|
import h5py
|
|
2
5
|
|
|
3
6
|
|
|
4
7
|
class Hdf5Writer:
|
|
5
8
|
|
|
6
9
|
@classmethod
|
|
7
|
-
def to(cls, path: str):
|
|
10
|
+
def to(cls, path: Union[str, Path]):
|
|
8
11
|
raise NotImplementedError("Hdf5Writer is not implemented yet.")
|
|
9
12
|
|
|
10
13
|
def __new__(cls, *args, **kwargs):
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
from typing import Optional
|
|
1
|
+
from typing import Optional, Union
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import numpy as np
|
|
4
5
|
from einops import rearrange
|
|
5
6
|
|
|
7
|
+
from .._path_conversion import _path2str
|
|
6
8
|
from ...backend.visual_lib import VisualLib
|
|
7
9
|
from ...util.type import T_ARRAY
|
|
8
10
|
from .._default_backends import _default_image_io_backend
|
|
@@ -12,12 +14,12 @@ class ImageReader:
|
|
|
12
14
|
"""ImageReader for reading images as :class:`~torch.Tensor`"""
|
|
13
15
|
|
|
14
16
|
@classmethod
|
|
15
|
-
def of(cls, path: str, channel_first: bool = False, backend: Optional[VisualLib] = None) -> T_ARRAY:
|
|
17
|
+
def of(cls, path: Union[str, Path], channel_first: bool = False, backend: Optional[VisualLib] = None) -> T_ARRAY:
|
|
16
18
|
"""
|
|
17
19
|
Read image tensor of given file.
|
|
18
20
|
|
|
19
21
|
Args:
|
|
20
|
-
path (``str``): Path of the JPEG or PNG image.
|
|
22
|
+
path (``str`` | ``pathlib.Path``): Path of the JPEG or PNG image.
|
|
21
23
|
channel_first (``bool``, optional): Get image dimension (H, W, C) if False or (C, H, W) if True.
|
|
22
24
|
Default: False.
|
|
23
25
|
backend (:class:`~tensorneko_util.backend.visual_lib.VisualLib`, optional): The backend library for saving.
|
|
@@ -27,6 +29,7 @@ class ImageReader:
|
|
|
27
29
|
:class:`~numpy.ndarray` | :class:`~torch.Tensor`: A float tensor of image, with value range of 0. to 1.
|
|
28
30
|
"""
|
|
29
31
|
backend = backend or _default_image_io_backend()
|
|
32
|
+
path = _path2str(path)
|
|
30
33
|
|
|
31
34
|
if backend == VisualLib.OPENCV:
|
|
32
35
|
if not VisualLib.opencv_available():
|
|
@@ -61,6 +64,6 @@ class ImageReader:
|
|
|
61
64
|
|
|
62
65
|
return img
|
|
63
66
|
|
|
64
|
-
def __new__(cls, path: str, channel_first: bool = True, backend: Optional[VisualLib] = None) -> T_ARRAY:
|
|
67
|
+
def __new__(cls, path: Union[str, Path], channel_first: bool = True, backend: Optional[VisualLib] = None) -> T_ARRAY:
|
|
65
68
|
"""Alias of :meth:`~ImageReader.of`"""
|
|
66
69
|
return cls.of(path, channel_first, backend)
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import pathlib
|
|
2
|
-
from typing import Optional
|
|
2
|
+
from typing import Optional, Union
|
|
3
|
+
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
import numpy as np
|
|
5
6
|
from einops import rearrange
|
|
6
7
|
from numpy import ndarray
|
|
7
8
|
|
|
8
9
|
from .._default_backends import _default_image_io_backend
|
|
10
|
+
from .._path_conversion import _path2str
|
|
9
11
|
from ...backend.visual_lib import VisualLib
|
|
10
12
|
from ...util.type import T_ARRAY
|
|
11
13
|
|
|
@@ -37,14 +39,14 @@ class ImageWriter:
|
|
|
37
39
|
return img_out
|
|
38
40
|
|
|
39
41
|
@classmethod
|
|
40
|
-
def to_jpeg(cls, path: str, image: T_ARRAY, quality: int = 75, channel_first: bool = False,
|
|
42
|
+
def to_jpeg(cls, path: Union[str, Path], image: T_ARRAY, quality: int = 75, channel_first: bool = False,
|
|
41
43
|
backend: Optional[VisualLib] = None
|
|
42
44
|
) -> None:
|
|
43
45
|
"""
|
|
44
46
|
Save as jpeg files from :class:`~torch.Tensor` or :class:`~numpy.ndarray`. The value range is [0, 1].
|
|
45
47
|
|
|
46
48
|
Args:
|
|
47
|
-
path (``str``): The path of output file.
|
|
49
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
48
50
|
image (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The image tensor for output.
|
|
49
51
|
quality (``int``, optional): The quality level for jpg from 0 to 100. Higher means quality is better.
|
|
50
52
|
Default: 75.
|
|
@@ -54,6 +56,7 @@ class ImageWriter:
|
|
|
54
56
|
Default: "opencv" if installed else "matplotlib".
|
|
55
57
|
"""
|
|
56
58
|
backend = backend or _default_image_io_backend()
|
|
59
|
+
path = _path2str(path)
|
|
57
60
|
image = cls._convert_img_format(image, backend, channel_first)
|
|
58
61
|
|
|
59
62
|
if backend == VisualLib.OPENCV:
|
|
@@ -73,14 +76,14 @@ class ImageWriter:
|
|
|
73
76
|
raise ValueError("Unknown backend library.")
|
|
74
77
|
|
|
75
78
|
@classmethod
|
|
76
|
-
def to_png(cls, path: str, image: T_ARRAY, compression_level: int = 6, channel_first: bool = False,
|
|
79
|
+
def to_png(cls, path: Union[str, Path], image: T_ARRAY, compression_level: int = 6, channel_first: bool = False,
|
|
77
80
|
backend: Optional[VisualLib] = None
|
|
78
81
|
) -> None:
|
|
79
82
|
"""
|
|
80
83
|
Save as png files from :class:`~torch.Tensor` or :class:`~numpy.ndarray` with (C, H, W) or (H, W, C).
|
|
81
84
|
|
|
82
85
|
Args:
|
|
83
|
-
path (``str``): The path of output file.
|
|
86
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
84
87
|
image (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The image tensor for output.
|
|
85
88
|
compression_level (``int``, optional): The compression level png from 0 to 9. Higher means quality is worse.
|
|
86
89
|
Default: 6
|
|
@@ -90,6 +93,7 @@ class ImageWriter:
|
|
|
90
93
|
Default: "opencv" if installed else "matplotlib".
|
|
91
94
|
"""
|
|
92
95
|
backend = backend or _default_image_io_backend()
|
|
96
|
+
path = _path2str(path)
|
|
93
97
|
image = cls._convert_img_format(image, backend, channel_first)
|
|
94
98
|
if backend == VisualLib.OPENCV:
|
|
95
99
|
if not VisualLib.opencv_available():
|
|
@@ -119,16 +123,17 @@ class ImageWriter:
|
|
|
119
123
|
raise ValueError("Unknown backend library.")
|
|
120
124
|
|
|
121
125
|
@classmethod
|
|
122
|
-
def to(cls, path: str, image: T_ARRAY, *args, **kwargs) -> None:
|
|
126
|
+
def to(cls, path: Union[str, Path], image: T_ARRAY, *args, **kwargs) -> None:
|
|
123
127
|
"""
|
|
124
128
|
Save as png files from :class:`~torch.Tensor` or :class:`~numpy.ndarray` with (C, H, W).
|
|
125
129
|
|
|
126
130
|
Args:
|
|
127
|
-
path (``str``): The path of output file.
|
|
131
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
128
132
|
image (:class:`~torch.Tensor` | :class:`~numpy.ndarray`): The image tensor for output.
|
|
129
133
|
*args: The arguments for :meth:`ImageWriter.to_jpeg` or :meth:`ImageWriter.to_png`.
|
|
130
134
|
**kwargs: The keyword arguments for :meth:`ImageWriter.to_jpeg` or :meth:`ImageWriter.to_png`.
|
|
131
135
|
"""
|
|
136
|
+
path = _path2str(path)
|
|
132
137
|
ext = pathlib.Path(path).suffix
|
|
133
138
|
if ext in ('.jpg', '.jpeg'):
|
|
134
139
|
cls.to_jpeg(path, image, *args, **kwargs)
|
|
@@ -137,6 +142,6 @@ class ImageWriter:
|
|
|
137
142
|
else:
|
|
138
143
|
raise ValueError("Unknown file extension. Now only support .jpg, .jpeg and .png.")
|
|
139
144
|
|
|
140
|
-
def __new__(cls, path: str, image: T_ARRAY, *args, **kwargs) -> None:
|
|
145
|
+
def __new__(cls, path: Union[str, Path], image: T_ARRAY, *args, **kwargs) -> None:
|
|
141
146
|
"""Alias of :meth:`ImageWriter.to`"""
|
|
142
147
|
cls.to(path, image, *args, **kwargs)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import warnings
|
|
3
|
-
from typing import Union, Optional, List
|
|
3
|
+
from typing import Union, Optional, List, Union
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
|
|
6
|
+
from .._path_conversion import _path2str
|
|
5
7
|
from ...util.type import T
|
|
6
8
|
|
|
7
9
|
|
|
@@ -18,13 +20,13 @@ class JsonReader:
|
|
|
18
20
|
return orjson.loads(file.read())
|
|
19
21
|
|
|
20
22
|
@classmethod
|
|
21
|
-
def of(cls, path: str, clazz: Optional[T] = None, encoding: str = "UTF-8", fast: bool = True
|
|
23
|
+
def of(cls, path: Union[str, Path], clazz: Optional[T] = None, encoding: str = "UTF-8", fast: bool = True
|
|
22
24
|
) -> Union[T, dict, list]:
|
|
23
25
|
"""
|
|
24
26
|
Read json files to ``list`` or ``dict``.
|
|
25
27
|
|
|
26
28
|
Args:
|
|
27
|
-
path (``str``): Json file path.
|
|
29
|
+
path (``str`` | ``pathlib.Path``): Json file path.
|
|
28
30
|
clazz: (``T``, optional): The object of the json read for. The type should be decorated by
|
|
29
31
|
:func:`json_data`. This should be ``T`` or ``List[T]`` or ``List[List[T]]``
|
|
30
32
|
encoding (``str``, optional): The encoding for python ``open`` function. Default: "UTF-8"
|
|
@@ -34,6 +36,7 @@ class JsonReader:
|
|
|
34
36
|
Returns:
|
|
35
37
|
``dict`` | ``list`` | ``object``: The object of given json.
|
|
36
38
|
"""
|
|
39
|
+
path = _path2str(path)
|
|
37
40
|
if fast:
|
|
38
41
|
try:
|
|
39
42
|
import orjson
|
|
@@ -73,13 +76,13 @@ class JsonReader:
|
|
|
73
76
|
|
|
74
77
|
|
|
75
78
|
@classmethod
|
|
76
|
-
def of_jsonl(cls, path: str, clazz: Optional[T] = None, encoding: str = "UTF-8", fast: bool = True
|
|
79
|
+
def of_jsonl(cls, path: Union[str, Path], clazz: Optional[T] = None, encoding: str = "UTF-8", fast: bool = True
|
|
77
80
|
) -> List[Union[T, dict, list]]:
|
|
78
81
|
"""
|
|
79
82
|
Read jsonl files to ``list`` or ``dict``.
|
|
80
83
|
|
|
81
84
|
Args:
|
|
82
|
-
path (``str``): Jsonl file path.
|
|
85
|
+
path (``str`` | ``pathlib.Path``): Jsonl file path.
|
|
83
86
|
clazz: (``T``, optional): The object of the jsonl read for. The type should be decorated by
|
|
84
87
|
:func:`json_data`. This should be ``T`` or ``List[T]`` or ``List[List[T]]``
|
|
85
88
|
encoding (``str``, optional): The encoding for python ``open`` function. Default: "UTF-8"
|
|
@@ -89,6 +92,7 @@ class JsonReader:
|
|
|
89
92
|
Returns:
|
|
90
93
|
``List[dict]`` | ``List[list]`` | ``List[T]``: The object of given jsonl.
|
|
91
94
|
"""
|
|
95
|
+
path = _path2str(path)
|
|
92
96
|
if fast:
|
|
93
97
|
try:
|
|
94
98
|
import orjson
|
|
@@ -139,12 +143,12 @@ class JsonReader:
|
|
|
139
143
|
|
|
140
144
|
return obj
|
|
141
145
|
|
|
142
|
-
def __new__(cls, path: str, clazz: T = None, encoding: str = "UTF-8", fast: bool = True) -> Union[T, dict, list]:
|
|
146
|
+
def __new__(cls, path: Union[str, Path], clazz: T = None, encoding: str = "UTF-8", fast: bool = True) -> Union[T, dict, list]:
|
|
143
147
|
"""
|
|
144
148
|
Read json or jsonl file smartly.
|
|
145
149
|
|
|
146
150
|
Args:
|
|
147
|
-
path (``str``): Json or jsonl file path.
|
|
151
|
+
path (``str`` | ``pathlib.Path``): Json or jsonl file path.
|
|
148
152
|
clazz: (``T``, optional): The object of the json read for. The type should be decorated by
|
|
149
153
|
:func:`json_data`. This should be ``T`` or ``List[T]`` or ``List[List[T]]``
|
|
150
154
|
encoding (``str``, optional): The encoding for python ``open`` function. Default: "UTF-8"
|
|
@@ -154,6 +158,7 @@ class JsonReader:
|
|
|
154
158
|
Returns:
|
|
155
159
|
``dict`` | ``list`` | ``object``: The object of given json or jsonl.
|
|
156
160
|
"""
|
|
161
|
+
path = _path2str(path)
|
|
157
162
|
if path.endswith(".jsonl"):
|
|
158
163
|
return cls.of_jsonl(path, clazz, encoding, fast)
|
|
159
164
|
else:
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import warnings
|
|
3
3
|
from typing import Union
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .._path_conversion import _path2str
|
|
4
7
|
|
|
5
8
|
|
|
6
9
|
class JsonWriter:
|
|
@@ -25,14 +28,14 @@ class JsonWriter:
|
|
|
25
28
|
file.write(orjson.dumps(obj, option=option))
|
|
26
29
|
|
|
27
30
|
@classmethod
|
|
28
|
-
def to(cls, path: str, obj: Union[dict, list, object], encoding: str = "UTF-8", indent: int = 4,
|
|
31
|
+
def to(cls, path: Union[str, Path], obj: Union[dict, list, object], encoding: str = "UTF-8", indent: int = 4,
|
|
29
32
|
ensure_ascii: bool = False, fast: bool = True
|
|
30
33
|
) -> None:
|
|
31
34
|
"""
|
|
32
35
|
Save as Json file from a dictionary, list or json_dict.
|
|
33
36
|
|
|
34
37
|
Args:
|
|
35
|
-
path (``str``): The path of output file.
|
|
38
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
36
39
|
obj (``dict`` | ``list`` | ``object``): The json data which need to be used for output. The type should be
|
|
37
40
|
``dict``, ``list`` or the ``object`` decorated by :func:`~tensorneko.io.text.text_reader.json_data`.
|
|
38
41
|
encoding (``str``, optional): Python file IO encoding parameter. Default: "UTF-8".
|
|
@@ -41,6 +44,7 @@ class JsonWriter:
|
|
|
41
44
|
fast (``bool``, optional): Whether to use faster `orjson`. If `orjson` is not installed, use
|
|
42
45
|
`json` library. Default: True
|
|
43
46
|
"""
|
|
47
|
+
path = _path2str(path)
|
|
44
48
|
if fast:
|
|
45
49
|
try:
|
|
46
50
|
import orjson
|
|
@@ -64,8 +68,9 @@ class JsonWriter:
|
|
|
64
68
|
else:
|
|
65
69
|
raise TypeError("Not implemented type. Only support dict, list, json_data.")
|
|
66
70
|
|
|
67
|
-
def __new__(cls, path: str, obj: Union[dict, list, object], encoding: str = "UTF-8", indent: int = 4,
|
|
71
|
+
def __new__(cls, path: Union[str, Path], obj: Union[dict, list, object], encoding: str = "UTF-8", indent: int = 4,
|
|
68
72
|
ensure_ascii: bool = False, fast: bool = True
|
|
69
73
|
) -> None:
|
|
70
74
|
"""Alias of :meth:`~tensorneko.io.json.json_writer.JsonWriter.to`."""
|
|
75
|
+
path = _path2str(path)
|
|
71
76
|
cls.to(path, obj, encoding, indent, ensure_ascii, fast)
|
|
@@ -1,20 +1,24 @@
|
|
|
1
|
-
from typing import Dict, Any
|
|
1
|
+
from typing import Dict, Any, Union
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import scipy.io
|
|
4
5
|
|
|
6
|
+
from .._path_conversion import _path2str
|
|
7
|
+
|
|
5
8
|
|
|
6
9
|
class MatReader:
|
|
7
10
|
|
|
8
11
|
@classmethod
|
|
9
|
-
def of(cls, path: str) -> Dict[str, Any]:
|
|
12
|
+
def of(cls, path: Union[str, Path]) -> Dict[str, Any]:
|
|
10
13
|
"""
|
|
11
14
|
Read a mat file.
|
|
12
15
|
|
|
13
16
|
Args:
|
|
14
|
-
path (``str``): The path of the mat file.
|
|
17
|
+
path (``str`` | ``pathlib.Path``): The path of the mat file.
|
|
15
18
|
"""
|
|
19
|
+
path = _path2str(path)
|
|
16
20
|
return scipy.io.loadmat(path)
|
|
17
21
|
|
|
18
|
-
def __new__(cls, path: str) -> Dict[str, Any]:
|
|
22
|
+
def __new__(cls, path: Union[str, Path]) -> Dict[str, Any]:
|
|
19
23
|
"""Alias to :meth:`~tensorneko_util.io.matlab.MatReader.of`."""
|
|
20
24
|
return cls.of(path)
|
|
@@ -1,21 +1,25 @@
|
|
|
1
|
-
from typing import Dict, Any
|
|
1
|
+
from typing import Dict, Any, Union
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import scipy.io
|
|
4
5
|
|
|
6
|
+
from .._path_conversion import _path2str
|
|
7
|
+
|
|
5
8
|
|
|
6
9
|
class MatWriter:
|
|
7
10
|
|
|
8
11
|
@classmethod
|
|
9
|
-
def to(cls, path: str, data: Dict[str, Any]) -> None:
|
|
12
|
+
def to(cls, path: Union[str, Path], data: Dict[str, Any]) -> None:
|
|
10
13
|
"""
|
|
11
14
|
Write data to a mat file.
|
|
12
15
|
|
|
13
16
|
Args:
|
|
14
|
-
path (``str``): path to write.
|
|
17
|
+
path (``str`` | ``pathlib.Path``): path to write.
|
|
15
18
|
data (``Dict[str, Any]``): data to write.
|
|
16
19
|
"""
|
|
20
|
+
path = _path2str(path)
|
|
17
21
|
scipy.io.savemat(path, data)
|
|
18
22
|
|
|
19
|
-
def __new__(cls, path: str, data: Dict[str, Any]) -> None:
|
|
23
|
+
def __new__(cls, path: Union[str, Path], data: Dict[str, Any]) -> None:
|
|
20
24
|
"""Alias to :meth:`~tensorneko_util.io.mat_writer.MatWriter.to`."""
|
|
21
25
|
cls.to(data, path)
|
|
@@ -1,46 +1,51 @@
|
|
|
1
1
|
from typing import Union
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import numpy as np
|
|
4
5
|
from numpy.lib.npyio import NpzFile
|
|
5
6
|
|
|
7
|
+
from .._path_conversion import _path2str
|
|
8
|
+
|
|
6
9
|
|
|
7
10
|
class NpyReader:
|
|
8
11
|
|
|
9
12
|
@classmethod
|
|
10
|
-
def of(cls, path: str) -> Union[np.ndarray, NpzFile]:
|
|
13
|
+
def of(cls, path: Union[str, Path]) -> Union[np.ndarray, NpzFile]:
|
|
11
14
|
"""
|
|
12
15
|
Read numpy array from file.
|
|
13
16
|
|
|
14
17
|
Args:
|
|
15
|
-
path (``str``): Path of the numpy file.
|
|
18
|
+
path (``str`` | ``pathlib.Path``): Path of the numpy file.
|
|
16
19
|
|
|
17
20
|
Returns:
|
|
18
21
|
:class:`~numpy.ndarray` | :class:`~numpy.lib.npyio.NpzFile`: The numpy array.
|
|
19
22
|
"""
|
|
23
|
+
path = _path2str(path)
|
|
20
24
|
return np.load(path)
|
|
21
25
|
|
|
22
26
|
@classmethod
|
|
23
|
-
def of_csc(cls, path: str) -> np.ndarray:
|
|
27
|
+
def of_csc(cls, path: Union[str, Path]) -> np.ndarray:
|
|
24
28
|
"""
|
|
25
29
|
Read numpy array from file as CSC sparse matrix.
|
|
26
30
|
|
|
27
31
|
Args:
|
|
28
|
-
path (``str``): Path of the numpy file.
|
|
32
|
+
path (``str`` | ``pathlib.Path``): Path of the numpy file.
|
|
29
33
|
|
|
30
34
|
Returns:
|
|
31
35
|
:class:`~numpy.ndarray`: The numpy array.
|
|
32
36
|
|
|
33
37
|
"""
|
|
38
|
+
path = _path2str(path)
|
|
34
39
|
import scipy.sparse
|
|
35
40
|
return scipy.sparse.load_npz(path).toarray()
|
|
36
41
|
|
|
37
42
|
@classmethod
|
|
38
|
-
def of_txt(cls, path: str, delimiter: str = ' ', dtype: type = float) -> np.ndarray:
|
|
43
|
+
def of_txt(cls, path: Union[str, Path], delimiter: str = ' ', dtype: type = float) -> np.ndarray:
|
|
39
44
|
"""
|
|
40
45
|
Read numpy array from file as text format.
|
|
41
46
|
|
|
42
47
|
Args:
|
|
43
|
-
path (``str``): Path of the numpy file.
|
|
48
|
+
path (``str`` | ``pathlib.Path``): Path of the numpy file.
|
|
44
49
|
delimiter (``str``, optional): The delimiter for each element.
|
|
45
50
|
Default: ' '
|
|
46
51
|
dtype (``type``, optional): The data type for each element.
|
|
@@ -49,10 +54,11 @@ class NpyReader:
|
|
|
49
54
|
Returns:
|
|
50
55
|
:class:`~numpy.ndarray`: The numpy array.
|
|
51
56
|
"""
|
|
57
|
+
path = _path2str(path)
|
|
52
58
|
return np.loadtxt(path, delimiter=delimiter, dtype=dtype)
|
|
53
59
|
|
|
54
|
-
|
|
55
|
-
|
|
60
|
+
def __new__(cls, path: Union[str, Path], *args, **kwargs) -> Union[np.ndarray, NpzFile]:
|
|
61
|
+
path = _path2str(path)
|
|
56
62
|
ext = path.split(".")[-1]
|
|
57
63
|
if ext in ("npz", "npy"):
|
|
58
64
|
return cls.of(path)
|
|
@@ -1,64 +1,73 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
1
4
|
import numpy as np
|
|
2
5
|
|
|
6
|
+
from .._path_conversion import _path2str
|
|
7
|
+
|
|
3
8
|
|
|
4
9
|
class NpyWriter:
|
|
5
10
|
|
|
6
11
|
@classmethod
|
|
7
|
-
def to(cls, path: str, arr: np.ndarray) -> None:
|
|
12
|
+
def to(cls, path: Union[str, Path], arr: np.ndarray) -> None:
|
|
8
13
|
"""
|
|
9
14
|
Save numpy array to file.
|
|
10
15
|
|
|
11
16
|
Args:
|
|
12
|
-
path (``str``): The path of output file.
|
|
17
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
13
18
|
arr (:class:`~numpy.ndarray`): The numpy array for output.
|
|
14
19
|
"""
|
|
20
|
+
path = _path2str(path)
|
|
15
21
|
np.save(path, arr)
|
|
16
22
|
|
|
17
23
|
@classmethod
|
|
18
|
-
def to_csc(cls, path: str, arr: np.ndarray) -> None:
|
|
24
|
+
def to_csc(cls, path: Union[str, Path], arr: np.ndarray) -> None:
|
|
19
25
|
"""
|
|
20
26
|
Save numpy array to file as CSC sparse matrix.
|
|
21
27
|
|
|
22
28
|
Args:
|
|
23
|
-
path (``str``): The path of output file.
|
|
29
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
24
30
|
arr (:class:`~numpy.ndarray`): The numpy array for output.
|
|
25
31
|
"""
|
|
32
|
+
path = _path2str(path)
|
|
26
33
|
import scipy.sparse
|
|
27
34
|
scipy.sparse.save_npz(path, scipy.sparse.csc_matrix(arr))
|
|
28
35
|
|
|
29
36
|
@classmethod
|
|
30
|
-
def to_npz(cls, path: str, compressed: bool = False, **kwargs) -> None:
|
|
37
|
+
def to_npz(cls, path: Union[str, Path], compressed: bool = False, **kwargs) -> None:
|
|
31
38
|
"""
|
|
32
39
|
Save numpy array to file as npz format.
|
|
33
40
|
|
|
34
41
|
Args:
|
|
35
|
-
path (``str``): The path of output file.
|
|
42
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
36
43
|
compressed (``bool``, optional): The flag for compressed npz file.
|
|
37
44
|
Default: False
|
|
38
45
|
**kwargs: The numpy arrays for output.
|
|
39
46
|
"""
|
|
47
|
+
path = _path2str(path)
|
|
40
48
|
if compressed:
|
|
41
49
|
np.savez_compressed(path, **kwargs)
|
|
42
50
|
else:
|
|
43
51
|
np.savez(path, **kwargs)
|
|
44
52
|
|
|
45
53
|
@classmethod
|
|
46
|
-
def to_txt(cls, path: str, arr: np.ndarray, delimiter: str = ' ', newline: str = '\n') -> None:
|
|
54
|
+
def to_txt(cls, path: Union[str, Path], arr: np.ndarray, delimiter: str = ' ', newline: str = '\n') -> None:
|
|
47
55
|
"""
|
|
48
56
|
Save numpy array to file as text format.
|
|
49
57
|
|
|
50
58
|
Args:
|
|
51
|
-
path (``str``): The path of output file.
|
|
59
|
+
path (``str`` | ``pathlib.Path``): The path of output file.
|
|
52
60
|
arr (:class:`~numpy.ndarray`): The numpy array for output.
|
|
53
61
|
delimiter (``str``, optional): The delimiter for each element.
|
|
54
62
|
Default: ' '
|
|
55
63
|
newline (``str``, optional): The newline for each row.
|
|
56
64
|
Default: '\n'
|
|
57
65
|
"""
|
|
66
|
+
path = _path2str(path)
|
|
58
67
|
np.savetxt(path, arr, delimiter=delimiter, newline=newline)
|
|
59
68
|
|
|
60
|
-
|
|
61
|
-
|
|
69
|
+
def __new__(cls, path: Union[str, Path], *args, **kwargs) -> None:
|
|
70
|
+
path = _path2str(path)
|
|
62
71
|
ext = path.split(".")[-1]
|
|
63
72
|
if ext == "npy":
|
|
64
73
|
return cls.to(path, *args, **kwargs)
|