stouputils 1.0.20__tar.gz → 1.0.21__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stouputils-1.0.20 → stouputils-1.0.21}/.github/workflows/documentation.yml +41 -29
- {stouputils-1.0.20 → stouputils-1.0.21}/PKG-INFO +2 -1
- {stouputils-1.0.20 → stouputils-1.0.21}/README.md +1 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/pyproject.toml +1 -1
- {stouputils-1.0.20 → stouputils-1.0.21}/scripts/create_docs.py +146 -5
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/ctx.py +2 -1
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/parallel.py +12 -12
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/print.py +21 -11
- {stouputils-1.0.20 → stouputils-1.0.21}/.gitignore +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/.python-version +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/1_upgrades.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/2_build.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/3_upload.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/LICENSE +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/all_in_one.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/build_all_in_one.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/copy_in_local.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/doctests.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/examples/delta_backup.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/github_release.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/__init__.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/all_doctests.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/archive.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/backup.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/collections.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/continuous_delivery/__init__.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/continuous_delivery/cd_utils.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/continuous_delivery/github.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/decorators.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/dont_look/zip_file_override.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/io.py +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/src/stouputils/py.typed +0 -0
- {stouputils-1.0.20 → stouputils-1.0.21}/upgrade.py +0 -0
|
@@ -1,29 +1,41 @@
|
|
|
1
|
-
name: documentation
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
- name:
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
1
|
+
name: documentation
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
tags:
|
|
8
|
+
- 'v*'
|
|
9
|
+
pull_request:
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: write
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
docs:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
- name: Install dependencies
|
|
22
|
+
run: |
|
|
23
|
+
pip install hatch stouputils sphinx sphinx_rtd_theme myst_parser
|
|
24
|
+
hatch build
|
|
25
|
+
- name: Build latest docs
|
|
26
|
+
if: github.ref == 'refs/heads/main'
|
|
27
|
+
run: |
|
|
28
|
+
python scripts/create_docs.py
|
|
29
|
+
- name: Build version docs
|
|
30
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
31
|
+
run: |
|
|
32
|
+
python scripts/create_docs.py ${GITHUB_REF#refs/tags/v}
|
|
33
|
+
- name: Deploy to GitHub Pages
|
|
34
|
+
uses: peaceiris/actions-gh-pages@v3
|
|
35
|
+
if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/tags/v') }}
|
|
36
|
+
with:
|
|
37
|
+
publish_branch: gh-pages
|
|
38
|
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
39
|
+
publish_dir: docs/build/html
|
|
40
|
+
force_orphan: false
|
|
41
|
+
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stouputils
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.21
|
|
4
4
|
Summary: Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Stoupy51/stouputils
|
|
6
6
|
Project-URL: Issues, https://github.com/Stoupy51/stouputils/issues
|
|
@@ -19,6 +19,7 @@ Description-Content-Type: text/markdown
|
|
|
19
19
|
# 🛠️ Project Badges
|
|
20
20
|
[](https://github.com/Stoupy51/stouputils/releases/latest)
|
|
21
21
|
[](https://pypi.org/project/stouputils/)
|
|
22
|
+
[](https://stoupy51.github.io/stouputils/latest/)
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
# 📚 Project Overview
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# 🛠️ Project Badges
|
|
3
3
|
[](https://github.com/Stoupy51/stouputils/releases/latest)
|
|
4
4
|
[](https://pypi.org/project/stouputils/)
|
|
5
|
+
[](https://stoupy51.github.io/stouputils/latest/)
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
# 📚 Project Overview
|
|
@@ -7,6 +7,82 @@ import sys
|
|
|
7
7
|
from stouputils import clean_path, handle_error
|
|
8
8
|
clean_exec: str = clean_path(sys.executable)
|
|
9
9
|
|
|
10
|
+
conf_content: str = """
|
|
11
|
+
# Imports
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from typing import Any
|
|
15
|
+
sys.path.insert(0, os.path.abspath('../..'))
|
|
16
|
+
sys.path.insert(0, os.path.abspath('../../src'))
|
|
17
|
+
from upgrade import current_version # Get version from pyproject.toml
|
|
18
|
+
|
|
19
|
+
# Project information
|
|
20
|
+
project: str = 'stouputils'
|
|
21
|
+
copyright: str = '2024, Stoupy'
|
|
22
|
+
author: str = 'Stoupy'
|
|
23
|
+
release: str = current_version
|
|
24
|
+
|
|
25
|
+
# General configuration
|
|
26
|
+
extensions: list[str] = [
|
|
27
|
+
'sphinx.ext.autodoc',
|
|
28
|
+
'sphinx.ext.napoleon',
|
|
29
|
+
'sphinx.ext.viewcode',
|
|
30
|
+
'sphinx.ext.githubpages',
|
|
31
|
+
'sphinx.ext.intersphinx',
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
templates_path: list[str] = ['_templates']
|
|
35
|
+
exclude_patterns: list[str] = []
|
|
36
|
+
|
|
37
|
+
# HTML output options
|
|
38
|
+
html_theme: str = 'sphinx_rtd_theme'
|
|
39
|
+
html_static_path: list[str] = ['_static']
|
|
40
|
+
|
|
41
|
+
# Theme options
|
|
42
|
+
html_theme_options: dict[str, Any] = {
|
|
43
|
+
'style_external_links': True,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# Add any paths that contain custom static files
|
|
47
|
+
html_static_path: list[str] = ['_static']
|
|
48
|
+
|
|
49
|
+
# Autodoc settings
|
|
50
|
+
autodoc_default_options: dict[str, bool | str] = {
|
|
51
|
+
'members': True,
|
|
52
|
+
'member-order': 'bysource',
|
|
53
|
+
'special-members': False,
|
|
54
|
+
'undoc-members': False,
|
|
55
|
+
'private-members': False,
|
|
56
|
+
'show-inheritance': True,
|
|
57
|
+
'ignore-module-all': True,
|
|
58
|
+
'exclude-members': '__weakref__'
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
# Tell autodoc to prefer source code over installed package
|
|
62
|
+
autodoc_mock_imports = []
|
|
63
|
+
always_document_param_types = True
|
|
64
|
+
add_module_names = False
|
|
65
|
+
|
|
66
|
+
# Tell Sphinx to look for source code in src directory
|
|
67
|
+
html_context = {
|
|
68
|
+
'display_github': True,
|
|
69
|
+
'github_user': 'Stoupy51',
|
|
70
|
+
'github_repo': 'stouputils',
|
|
71
|
+
'github_version': 'main',
|
|
72
|
+
'conf_py_path': '/docs/source/',
|
|
73
|
+
'source_suffix': '.rst',
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
# Only document items with docstrings
|
|
77
|
+
def skip_undocumented(app: Any, what: str, name: str, obj: Any, skip: bool, *args: Any, **kwargs: Any) -> bool:
|
|
78
|
+
if not obj.__doc__:
|
|
79
|
+
return True
|
|
80
|
+
return skip
|
|
81
|
+
|
|
82
|
+
def setup(app: Any) -> None:
|
|
83
|
+
app.connect('autodoc-skip-member', skip_undocumented)
|
|
84
|
+
"""
|
|
85
|
+
|
|
10
86
|
def generate_index_rst(readme_path: str, index_path: str) -> None:
|
|
11
87
|
""" Generate index.rst from README.md content.
|
|
12
88
|
|
|
@@ -26,6 +102,30 @@ def generate_index_rst(readme_path: str, index_path: str) -> None:
|
|
|
26
102
|
:target: https://pypi.org/project/stouputils/
|
|
27
103
|
"""
|
|
28
104
|
|
|
105
|
+
# Generate version selector
|
|
106
|
+
version_selector: str = """
|
|
107
|
+
|
|
108
|
+
**Versions**: """
|
|
109
|
+
|
|
110
|
+
# Add versions from html_context
|
|
111
|
+
version_list: list[str] = []
|
|
112
|
+
html_dir = "docs/build/html"
|
|
113
|
+
if os.path.exists(html_dir):
|
|
114
|
+
version_list = [d[1:] for d in os.listdir(html_dir) if d.startswith('v')]
|
|
115
|
+
from stouputils.continuous_delivery.github import version_to_float
|
|
116
|
+
version_list.sort(key=version_to_float, reverse=True)
|
|
117
|
+
version_list.insert(0, 'latest')
|
|
118
|
+
|
|
119
|
+
# Create version links
|
|
120
|
+
version_links: list[str] = []
|
|
121
|
+
for version in version_list:
|
|
122
|
+
if version == 'latest':
|
|
123
|
+
version_links.append("`latest <../latest/index.html>`_")
|
|
124
|
+
else:
|
|
125
|
+
version_links.append(f"`v{version} <../v{version}/index.html>`_")
|
|
126
|
+
|
|
127
|
+
version_selector += ", ".join(version_links)
|
|
128
|
+
|
|
29
129
|
# Extract sections while preserving emojis
|
|
30
130
|
overview_section: str = readme_content.split('# 📚 Project Overview')[1].split('\n#')[0].strip()
|
|
31
131
|
file_tree_section: str = readme_content.split('# 🚀 Project File Tree')[1].split('\n#')[0].strip()
|
|
@@ -45,6 +145,8 @@ def generate_index_rst(readme_path: str, index_path: str) -> None:
|
|
|
45
145
|
|
|
46
146
|
{badges_rst}
|
|
47
147
|
|
|
148
|
+
{version_selector}
|
|
149
|
+
|
|
48
150
|
📚 Overview
|
|
49
151
|
-----------
|
|
50
152
|
{overview_section.replace("<br>", " ")}
|
|
@@ -71,13 +173,25 @@ def generate_index_rst(readme_path: str, index_path: str) -> None:
|
|
|
71
173
|
with open(index_path, 'w', encoding="utf-8") as f:
|
|
72
174
|
f.write(rst_content)
|
|
73
175
|
|
|
176
|
+
def generate_conf_py(conf_path: str) -> None:
|
|
177
|
+
""" Generate conf.py file.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
conf_path (str): Path where conf.py should be created
|
|
181
|
+
"""
|
|
182
|
+
with open(conf_path, 'w', encoding="utf-8") as f:
|
|
183
|
+
f.write(conf_content)
|
|
184
|
+
|
|
74
185
|
@handle_error()
|
|
75
|
-
def update_documentation() -> None:
|
|
186
|
+
def update_documentation(version: str | None = None) -> None:
|
|
76
187
|
""" Update the Sphinx documentation.
|
|
77
188
|
This script will:
|
|
78
189
|
1. Create necessary directories if they don't exist
|
|
79
190
|
2. Generate module documentation using sphinx-apidoc
|
|
80
|
-
3. Build HTML documentation
|
|
191
|
+
3. Build HTML documentation for specific version if provided
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
version (str | None): Version to build documentation for. If None, builds for latest
|
|
81
195
|
"""
|
|
82
196
|
# Get the project root directory (parent of scripts folder)
|
|
83
197
|
root_dir: str = clean_path(os.path.dirname(os.path.dirname(__file__)))
|
|
@@ -85,6 +199,9 @@ def update_documentation() -> None:
|
|
|
85
199
|
source_dir: str = clean_path(os.path.join(docs_dir, "source"))
|
|
86
200
|
modules_dir: str = clean_path(os.path.join(source_dir, "modules"))
|
|
87
201
|
|
|
202
|
+
# Modify build directory if version is specified
|
|
203
|
+
build_dir: str = "html/latest" if not version else f"html/v{version}"
|
|
204
|
+
|
|
88
205
|
# Create directories if they don't exist
|
|
89
206
|
os.makedirs(modules_dir, exist_ok=True)
|
|
90
207
|
os.makedirs(clean_path(os.path.join(source_dir, "_static")), exist_ok=True)
|
|
@@ -100,6 +217,25 @@ def update_documentation() -> None:
|
|
|
100
217
|
shutil.rmtree(modules_dir)
|
|
101
218
|
os.makedirs(modules_dir)
|
|
102
219
|
|
|
220
|
+
# Update conf.py to include version selector
|
|
221
|
+
version_list: list[str] = []
|
|
222
|
+
if os.path.exists(clean_path(f"{docs_dir}/build/html")):
|
|
223
|
+
version_list = [d.replace("v", "") for d in os.listdir(clean_path(f"{docs_dir}/build/html"))
|
|
224
|
+
if d.startswith("v")] + ["latest"]
|
|
225
|
+
|
|
226
|
+
# Update html_context in conf.py
|
|
227
|
+
global conf_content
|
|
228
|
+
conf_content = conf_content.replace(
|
|
229
|
+
"html_context = {",
|
|
230
|
+
f"""html_context = {{
|
|
231
|
+
'versions': {version_list},
|
|
232
|
+
'current_version': 'latest' if not {repr(version)} else {repr(version)},"""
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
# Generate docs/source/conf.py
|
|
236
|
+
conf_path: str = clean_path(os.path.join(source_dir, "conf.py"))
|
|
237
|
+
generate_conf_py(conf_path)
|
|
238
|
+
|
|
103
239
|
# Generate module documentation using python -m
|
|
104
240
|
subprocess.run([
|
|
105
241
|
clean_exec,
|
|
@@ -122,12 +258,17 @@ def update_documentation() -> None:
|
|
|
122
258
|
"-b", "html", # Build HTML
|
|
123
259
|
"-a", # Write all files
|
|
124
260
|
source_dir, # Source directory
|
|
125
|
-
clean_path(
|
|
261
|
+
clean_path(f"{docs_dir}/build/{build_dir}"), # Output directory
|
|
126
262
|
], check=True)
|
|
127
263
|
|
|
128
264
|
print("Documentation updated successfully!")
|
|
129
|
-
print(f"You can view the documentation by opening {docs_dir}/build/
|
|
265
|
+
print(f"You can view the documentation by opening {docs_dir}/build/{build_dir}/index.html")
|
|
130
266
|
|
|
131
267
|
if __name__ == "__main__":
|
|
132
|
-
|
|
268
|
+
if len(sys.argv) == 2:
|
|
269
|
+
update_documentation(sys.argv[1].replace("v", "")) # Remove "v" from version just in case
|
|
270
|
+
elif len(sys.argv) == 1:
|
|
271
|
+
update_documentation()
|
|
272
|
+
else:
|
|
273
|
+
raise ValueError("Usage: python create_docs.py [version]")
|
|
133
274
|
|
|
@@ -93,7 +93,8 @@ class LogToFile:
|
|
|
93
93
|
# Build log file path
|
|
94
94
|
file_basename: str = os.path.splitext(os.path.basename(filepath))[0]
|
|
95
95
|
date_time: str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
96
|
-
|
|
96
|
+
date_str, time_str = date_time.split("_")
|
|
97
|
+
log_filepath: str = f"{logs_folder}/{file_basename}/{date_str}/{time_str}.log"
|
|
97
98
|
|
|
98
99
|
# Launch function with arguments if any
|
|
99
100
|
with LogToFile(log_filepath):
|
|
@@ -81,7 +81,7 @@ def __handle_parameters(
|
|
|
81
81
|
return desc, func, args
|
|
82
82
|
|
|
83
83
|
@handle_error(error_log=LogLevels.ERROR_TRACEBACK)
|
|
84
|
-
def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = False, chunksize: int = 1, desc: str = "", max_workers: int = CPU_COUNT, delay_first_calls: float = 0,
|
|
84
|
+
def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = False, chunksize: int = 1, desc: str = "", max_workers: int = CPU_COUNT, delay_first_calls: float = 0, verbose: int = 0) -> list[R]:
|
|
85
85
|
r""" Method to execute a function in parallel using multiprocessing, you should use it:
|
|
86
86
|
|
|
87
87
|
- For CPU-bound operations where the GIL (Global Interpreter Lock) is a bottleneck.
|
|
@@ -96,7 +96,7 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
|
|
|
96
96
|
desc (str): Description of the function execution displayed in the progress bar
|
|
97
97
|
max_workers (int): Number of workers to use (Defaults to CPU_COUNT)
|
|
98
98
|
delay_first_calls (float): Apply i*delay_first_calls seconds delay to the first "max_workers" calls. For instance, the first process will be delayed by 0 seconds, the second by 1 second, etc. (Defaults to 0): This can be useful to avoid functions being called in the same second.
|
|
99
|
-
|
|
99
|
+
verbose (int): Level of verbosity, decrease by 1 for each depth
|
|
100
100
|
Returns:
|
|
101
101
|
list[object]: Results of the function execution
|
|
102
102
|
Examples:
|
|
@@ -107,11 +107,11 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
|
|
|
107
107
|
[2, 12, 30]
|
|
108
108
|
|
|
109
109
|
>>> # Will process in parallel with progress bar
|
|
110
|
-
>>> multiprocessing(doctest_slow, list(range(10)), desc="Processing",
|
|
110
|
+
>>> multiprocessing(doctest_slow, list(range(10)), desc="Processing", verbose=1)
|
|
111
111
|
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
|
112
112
|
|
|
113
113
|
>>> # Will process in parallel with progress bar and delay the first threads
|
|
114
|
-
>>> multiprocessing(doctest_slow, list(range(10)), desc="Processing with delay", max_workers=2, delay_first_calls=1.2,
|
|
114
|
+
>>> multiprocessing(doctest_slow, list(range(10)), desc="Processing with delay", max_workers=2, delay_first_calls=1.2, verbose=1)
|
|
115
115
|
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
|
116
116
|
"""
|
|
117
117
|
# Handle parameters
|
|
@@ -119,7 +119,7 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
|
|
|
119
119
|
|
|
120
120
|
# Do multiprocessing only if there is more than 1 argument and more than 1 CPU
|
|
121
121
|
if max_workers > 1 and len(args) > 1:
|
|
122
|
-
if
|
|
122
|
+
if verbose > 0:
|
|
123
123
|
return list(process_map(func, args, max_workers=max_workers, chunksize=chunksize, desc=desc, bar_format=BAR_FORMAT)) # type: ignore
|
|
124
124
|
else:
|
|
125
125
|
with Pool(max_workers) as pool:
|
|
@@ -127,14 +127,14 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
|
|
|
127
127
|
|
|
128
128
|
# Single process execution
|
|
129
129
|
else:
|
|
130
|
-
if
|
|
130
|
+
if verbose > 0:
|
|
131
131
|
return [func(arg) for arg in tqdm(args, total=len(args), desc=desc, bar_format=BAR_FORMAT)]
|
|
132
132
|
else:
|
|
133
133
|
return [func(arg) for arg in args]
|
|
134
134
|
|
|
135
135
|
|
|
136
136
|
@handle_error(error_log=LogLevels.ERROR_TRACEBACK)
|
|
137
|
-
def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = False, desc: str = "", max_workers: int = CPU_COUNT, delay_first_calls: float = 0,
|
|
137
|
+
def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = False, desc: str = "", max_workers: int = CPU_COUNT, delay_first_calls: float = 0, verbose: int = 0) -> list[R]:
|
|
138
138
|
r""" Method to execute a function in parallel using multithreading, you should use it:
|
|
139
139
|
|
|
140
140
|
- For I/O-bound operations where the GIL is not a bottleneck, such as network requests or disk operations.
|
|
@@ -148,7 +148,7 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
|
|
|
148
148
|
desc (str): Description of the function execution displayed in the progress bar
|
|
149
149
|
max_workers (int): Number of workers to use (Defaults to CPU_COUNT)
|
|
150
150
|
delay_first_calls (float): Apply i*delay_first_calls seconds delay to the first "max_workers" calls. For instance with value to 1, the first thread will be delayed by 0 seconds, the second by 1 second, etc. (Defaults to 0): This can be useful to avoid functions being called in the same second.
|
|
151
|
-
|
|
151
|
+
verbose (int): Level of verbosity, decrease by 1 for each depth
|
|
152
152
|
Returns:
|
|
153
153
|
list[object]: Results of the function execution
|
|
154
154
|
Examples:
|
|
@@ -159,11 +159,11 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
|
|
|
159
159
|
[2, 12, 30]
|
|
160
160
|
|
|
161
161
|
>>> # Will process in parallel with progress bar
|
|
162
|
-
>>> multithreading(doctest_slow, list(range(10)), desc="Threading",
|
|
162
|
+
>>> multithreading(doctest_slow, list(range(10)), desc="Threading", verbose=1)
|
|
163
163
|
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
|
164
164
|
|
|
165
165
|
>>> # Will process in parallel with progress bar and delay the first threads
|
|
166
|
-
>>> multithreading(doctest_slow, list(range(10)), desc="Threading with delay", max_workers=2, delay_first_calls=1.2,
|
|
166
|
+
>>> multithreading(doctest_slow, list(range(10)), desc="Threading with delay", max_workers=2, delay_first_calls=1.2, verbose=1)
|
|
167
167
|
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
|
168
168
|
"""
|
|
169
169
|
# Handle parameters
|
|
@@ -171,7 +171,7 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
|
|
|
171
171
|
|
|
172
172
|
# Do multithreading only if there is more than 1 argument and more than 1 CPU
|
|
173
173
|
if max_workers > 1 and len(args) > 1:
|
|
174
|
-
if
|
|
174
|
+
if verbose > 0:
|
|
175
175
|
with ThreadPoolExecutor(max_workers) as executor:
|
|
176
176
|
return list(tqdm(executor.map(func, args), total=len(args), desc=desc, bar_format=BAR_FORMAT))
|
|
177
177
|
else:
|
|
@@ -180,7 +180,7 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
|
|
|
180
180
|
|
|
181
181
|
# Single process execution
|
|
182
182
|
else:
|
|
183
|
-
if
|
|
183
|
+
if verbose > 0:
|
|
184
184
|
return [func(arg) for arg in tqdm(args, total=len(args), desc=desc, bar_format=BAR_FORMAT)]
|
|
185
185
|
else:
|
|
186
186
|
return [func(arg) for arg in args]
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
"""
|
|
2
|
-
This module provides utility functions for printing messages with different levels of importance,
|
|
2
|
+
This module provides utility functions for printing messages with different levels of importance,
|
|
3
|
+
If a message is printed multiple times, it will be displayed as "(xN) message" where N is the number of times the message has been printed.
|
|
4
|
+
|
|
5
|
+
The functions are such as:
|
|
3
6
|
|
|
4
7
|
- info()
|
|
5
8
|
- debug()
|
|
@@ -7,12 +10,14 @@ This module provides utility functions for printing messages with different leve
|
|
|
7
10
|
- progress()
|
|
8
11
|
- warning()
|
|
9
12
|
- error()
|
|
10
|
-
|
|
11
|
-
It also includes:
|
|
12
|
-
|
|
13
13
|
- whatisit(): a function to print the type of each value and the value itself (and few other things)
|
|
14
14
|
- breakpoint(): a breakpoint function to pause the program while calling whatisit()
|
|
15
15
|
- logging_to: a set of file-like objects that will receive log messages without ANSI color codes, see stouputils.ctx.LogToFile for easy logging
|
|
16
|
+
|
|
17
|
+
Here is a demonstration gif showing examples of uses:
|
|
18
|
+
|
|
19
|
+
.. image:: https://i.imgur.com/EIeiLwa.gif
|
|
20
|
+
:alt: stouputils print examples
|
|
16
21
|
"""
|
|
17
22
|
|
|
18
23
|
# Imports
|
|
@@ -54,8 +59,13 @@ def is_same_print(*args: Any, **kwargs: Any) -> bool:
|
|
|
54
59
|
nb_values = 1
|
|
55
60
|
return False
|
|
56
61
|
|
|
62
|
+
import_time: float = time.time()
|
|
57
63
|
def current_time() -> str:
|
|
58
|
-
return
|
|
64
|
+
# If the import time is more than 24 hours, return the full datetime
|
|
65
|
+
if (time.time() - import_time) > (24 * 60 * 60):
|
|
66
|
+
return time.strftime("%Y-%m-%d %H:%M:%S")
|
|
67
|
+
else:
|
|
68
|
+
return time.strftime("%H:%M:%S")
|
|
59
69
|
|
|
60
70
|
def info(*values: Any, color: str = GREEN, text: str = "INFO ", prefix: str = "", file: TextIO|list[TextIO] = sys.stdout, **print_kwargs: Any) -> None:
|
|
61
71
|
""" Print an information message looking like "[INFO HH:MM:SS] message" in green by default.
|
|
@@ -211,15 +221,15 @@ def breakpoint(*values: Any, print_function: Callable[..., None] = warning, **pr
|
|
|
211
221
|
|
|
212
222
|
if __name__ == "__main__":
|
|
213
223
|
info("Hello", "World")
|
|
214
|
-
time.sleep(
|
|
215
|
-
info("Hello", "World")
|
|
216
|
-
time.sleep(1)
|
|
224
|
+
time.sleep(0.5)
|
|
217
225
|
info("Hello", "World")
|
|
218
|
-
time.sleep(
|
|
226
|
+
time.sleep(0.5)
|
|
219
227
|
info("Hello", "World")
|
|
220
|
-
time.sleep(
|
|
228
|
+
time.sleep(0.5)
|
|
229
|
+
info("Not Hello World !")
|
|
230
|
+
time.sleep(0.5)
|
|
221
231
|
info("Hello", "World")
|
|
222
|
-
time.sleep(
|
|
232
|
+
time.sleep(0.5)
|
|
223
233
|
info("Hello", "World")
|
|
224
234
|
|
|
225
235
|
# All remaining print functions
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|