lockss-debugpanel 0.8.2__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/CHANGELOG.rst +23 -5
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/LICENSE +1 -1
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/PKG-INFO +13 -13
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/README.rst +6 -6
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/pyproject.toml +8 -8
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/src/lockss/debugpanel/__init__.py +6 -34
- {lockss_debugpanel-0.8.2 → lockss_debugpanel-0.9.0}/src/lockss/debugpanel/__main__.py +1 -1
- lockss_debugpanel-0.9.0/src/lockss/debugpanel/cli.py +438 -0
- lockss_debugpanel-0.8.2/src/lockss/debugpanel/cli.py +0 -379
|
@@ -2,13 +2,35 @@
|
|
|
2
2
|
Release Notes
|
|
3
3
|
=============
|
|
4
4
|
|
|
5
|
+
-----
|
|
6
|
+
0.9.0
|
|
7
|
+
-----
|
|
8
|
+
|
|
9
|
+
Released: 2026-03-18
|
|
10
|
+
|
|
11
|
+
Requires Python 3.10 or greater.
|
|
12
|
+
|
|
13
|
+
* **Features**
|
|
14
|
+
|
|
15
|
+
* New command line infrastructure based on `Click Extra <https://kdeldycke.github.io/click-extra>`_, `Cloup <https://cloup.readthedocs.io/>`_ and `Click <https://click.palletsprojects.com/>`_, including expanded tabular output styles, progress bar, command sections and aliases.
|
|
16
|
+
|
|
17
|
+
* New ``--headings``//``--no-headings``, ``--progress``//``--no-progress`` output styles.
|
|
18
|
+
|
|
19
|
+
* **Changes**
|
|
20
|
+
|
|
21
|
+
* The alias ``-u`` of ``--username`` and ``-p`` of ``--password`` are deprecated in favor of ``-U`` and ``-P`` respectively.
|
|
22
|
+
|
|
23
|
+
* ``--process-pool`` and ``--thread-pool`` are deprecated in favor of ``--pool-type=process-pool`` and ``--pool-type=thread-pool`` respectively.
|
|
24
|
+
|
|
25
|
+
* ``--output-format`` has been renamed to ``--table-format``/``-T``.
|
|
26
|
+
|
|
5
27
|
-----
|
|
6
28
|
0.8.2
|
|
7
29
|
-----
|
|
8
30
|
|
|
9
31
|
Released: 2026-02-03
|
|
10
32
|
|
|
11
|
-
|
|
33
|
+
Requires Python 3.9-3.13.
|
|
12
34
|
|
|
13
35
|
-----
|
|
14
36
|
0.8.1
|
|
@@ -20,10 +42,6 @@ Released: 2025-08-13
|
|
|
20
42
|
|
|
21
43
|
* Fixed bug in the processing of ``--nodes`` and ``--auids`` options.
|
|
22
44
|
|
|
23
|
-
* **Version 0.8.1-post1** (released: 2026-02-03)
|
|
24
|
-
|
|
25
|
-
* Requires Python 3.9-3.13.
|
|
26
|
-
|
|
27
45
|
-----
|
|
28
46
|
0.8.0
|
|
29
47
|
-----
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
Copyright (c) 2000-
|
|
1
|
+
Copyright (c) 2000-2026, Board of Trustees of Leland Stanford Jr. University
|
|
2
2
|
|
|
3
3
|
Redistribution and use in source and binary forms, with or without
|
|
4
4
|
modification, are permitted provided that the following conditions are met:
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lockss-debugpanel
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Command line tool and Python library to interact with the LOCKSS 1.x DebugPanel servlet
|
|
5
5
|
License: BSD-3-Clause
|
|
6
6
|
License-File: LICENSE
|
|
7
7
|
Author: Thib Guicherd-Callin
|
|
8
8
|
Author-email: thib@cs.stanford.edu
|
|
9
9
|
Maintainer: Thib Guicherd-Callin
|
|
10
10
|
Maintainer-email: thib@cs.stanford.edu
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.10,<4.0
|
|
12
12
|
Classifier: Development Status :: 5 - Production/Stable
|
|
13
13
|
Classifier: Environment :: Console
|
|
14
14
|
Classifier: Framework :: Pydantic :: 2
|
|
@@ -19,10 +19,10 @@ Classifier: Programming Language :: Python
|
|
|
19
19
|
Classifier: Topic :: Software Development :: Libraries
|
|
20
20
|
Classifier: Topic :: System :: Archiving
|
|
21
21
|
Classifier: Topic :: Utilities
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist:
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist:
|
|
22
|
+
Requires-Dist: click-command-tree (>=1.2.0,<1.3.0)
|
|
23
|
+
Requires-Dist: click-extra[pygments] (>=7.5.0,<7.6.0)
|
|
24
|
+
Requires-Dist: click-plugins (>=1.1.1.2,<1.2.0)
|
|
25
|
+
Requires-Dist: lockss-pybasic (>=0.2.0,<0.3.0)
|
|
26
26
|
Project-URL: Documentation, https://docs.lockss.org/en/latest/software/debugpanel
|
|
27
27
|
Project-URL: Repository, https://github.com/lockss/lockss-debugpanel
|
|
28
28
|
Project-URL: changelog, https://github.com/lockss/lockss-debugpanel/blob/main/CHANGELOG.rst
|
|
@@ -33,8 +33,8 @@ Description-Content-Type: text/x-rst
|
|
|
33
33
|
Debugpanel
|
|
34
34
|
==========
|
|
35
35
|
|
|
36
|
-
.. |RELEASE| replace:: 0.
|
|
37
|
-
.. |RELEASE_DATE| replace:: 2026-
|
|
36
|
+
.. |RELEASE| replace:: 0.9.0
|
|
37
|
+
.. |RELEASE_DATE| replace:: 2026-03-18
|
|
38
38
|
.. |DEBUGPANEL| replace:: **Debugpanel**
|
|
39
39
|
|
|
40
40
|
.. image:: https://assets.lockss.org/images/logos/debugpanel/debugpanel_128x128.png
|
|
@@ -54,7 +54,7 @@ Debugpanel
|
|
|
54
54
|
|
|
55
55
|
Quick Start::
|
|
56
56
|
|
|
57
|
-
# Requires Python 3.
|
|
57
|
+
# Requires Python 3.10 or greater
|
|
58
58
|
python --version
|
|
59
59
|
|
|
60
60
|
# Install with pipx
|
|
@@ -67,9 +67,9 @@ Quick Start::
|
|
|
67
67
|
debugpanel reload-config -n lockss1.example.edu:8081
|
|
68
68
|
|
|
69
69
|
# Crawl AUIDs from list.txt on lockss1.example.edu:8081 and lockss2.example.edu:8081
|
|
70
|
-
# ...First alternative: each node gets a -n
|
|
71
70
|
debugpanel crawl -A list.txt -n lockss1.example.edu:8081 -n lockss2.example.edu:8081
|
|
72
71
|
|
|
73
|
-
#
|
|
74
|
-
debugpanel crawl -A list.txt -
|
|
72
|
+
# Alternatively, list lockss1.example.edu:8081 and lockss2.example.edu:8081 in nodes.txt
|
|
73
|
+
debugpanel crawl -A list.txt -N nodes.txt
|
|
74
|
+
|
|
75
75
|
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
Debugpanel
|
|
3
3
|
==========
|
|
4
4
|
|
|
5
|
-
.. |RELEASE| replace:: 0.
|
|
6
|
-
.. |RELEASE_DATE| replace:: 2026-
|
|
5
|
+
.. |RELEASE| replace:: 0.9.0
|
|
6
|
+
.. |RELEASE_DATE| replace:: 2026-03-18
|
|
7
7
|
.. |DEBUGPANEL| replace:: **Debugpanel**
|
|
8
8
|
|
|
9
9
|
.. image:: https://assets.lockss.org/images/logos/debugpanel/debugpanel_128x128.png
|
|
@@ -23,7 +23,7 @@ Debugpanel
|
|
|
23
23
|
|
|
24
24
|
Quick Start::
|
|
25
25
|
|
|
26
|
-
# Requires Python 3.
|
|
26
|
+
# Requires Python 3.10 or greater
|
|
27
27
|
python --version
|
|
28
28
|
|
|
29
29
|
# Install with pipx
|
|
@@ -36,8 +36,8 @@ Quick Start::
|
|
|
36
36
|
debugpanel reload-config -n lockss1.example.edu:8081
|
|
37
37
|
|
|
38
38
|
# Crawl AUIDs from list.txt on lockss1.example.edu:8081 and lockss2.example.edu:8081
|
|
39
|
-
# ...First alternative: each node gets a -n
|
|
40
39
|
debugpanel crawl -A list.txt -n lockss1.example.edu:8081 -n lockss2.example.edu:8081
|
|
41
40
|
|
|
42
|
-
#
|
|
43
|
-
debugpanel crawl -A list.txt -
|
|
41
|
+
# Alternatively, list lockss1.example.edu:8081 and lockss2.example.edu:8081 in nodes.txt
|
|
42
|
+
debugpanel crawl -A list.txt -N nodes.txt
|
|
43
|
+
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright (c) 2000-
|
|
1
|
+
# Copyright (c) 2000-2026, Board of Trustees of Leland Stanford Jr. University
|
|
2
2
|
#
|
|
3
3
|
# Redistribution and use in source and binary forms, with or without
|
|
4
4
|
# modification, are permitted provided that the following conditions are met:
|
|
@@ -28,11 +28,11 @@
|
|
|
28
28
|
|
|
29
29
|
[project]
|
|
30
30
|
name = "lockss-debugpanel"
|
|
31
|
-
version = "0.
|
|
32
|
-
description = "
|
|
31
|
+
version = "0.9.0" # Always change in __init__.py, and at release time in README.rst and CHANGELOG.rst
|
|
32
|
+
description = "Command line tool and Python library to interact with the LOCKSS 1.x DebugPanel servlet"
|
|
33
33
|
license = { text = "BSD-3-Clause" }
|
|
34
34
|
readme = "README.rst"
|
|
35
|
-
requires-python = ">=3.
|
|
35
|
+
requires-python = ">=3.10,<4.0"
|
|
36
36
|
authors = [
|
|
37
37
|
{ name = "Thib Guicherd-Callin", email = "thib@cs.stanford.edu" },
|
|
38
38
|
]
|
|
@@ -40,10 +40,10 @@ maintainers = [
|
|
|
40
40
|
{ name = "Thib Guicherd-Callin", email = "thib@cs.stanford.edu" }
|
|
41
41
|
]
|
|
42
42
|
dependencies = [
|
|
43
|
-
"
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
43
|
+
"click-command-tree (>=1.2.0,<1.3.0)",
|
|
44
|
+
"click-extra[pygments] (>=7.5.0,<7.6.0)",
|
|
45
|
+
"click-plugins (>=1.1.1.2,<1.2.0)",
|
|
46
|
+
"lockss-pybasic (>=0.2.0,<0.3.0)",
|
|
47
47
|
]
|
|
48
48
|
classifiers = [
|
|
49
49
|
"Development Status :: 5 - Production/Stable",
|
|
@@ -1,41 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
|
|
3
|
-
# Copyright (c) 2000-2025, Board of Trustees of Leland Stanford Jr. University
|
|
4
|
-
#
|
|
5
|
-
# Redistribution and use in source and binary forms, with or without
|
|
6
|
-
# modification, are permitted provided that the following conditions are met:
|
|
7
|
-
#
|
|
8
|
-
# 1. Redistributions of source code must retain the above copyright notice,
|
|
9
|
-
# this list of conditions and the following disclaimer.
|
|
10
|
-
#
|
|
11
|
-
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
-
# this list of conditions and the following disclaimer in the documentation
|
|
13
|
-
# and/or other materials provided with the distribution.
|
|
14
|
-
#
|
|
15
|
-
# 3. Neither the name of the copyright holder nor the names of its contributors
|
|
16
|
-
# may be used to endorse or promote products derived from this software without
|
|
17
|
-
# specific prior written permission.
|
|
18
|
-
#
|
|
19
|
-
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
-
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
-
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
22
|
-
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
23
|
-
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
24
|
-
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
25
|
-
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
26
|
-
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
27
|
-
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
28
|
-
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
29
|
-
# POSSIBILITY OF SUCH DAMAGE.
|
|
30
|
-
|
|
31
3
|
"""
|
|
32
4
|
Library and command line tool to interact with the LOCKSS 1.x DebugPanel servlet.
|
|
33
5
|
"""
|
|
34
6
|
|
|
35
|
-
__version__ = '0.
|
|
7
|
+
__version__ = '0.9.0'
|
|
36
8
|
|
|
37
9
|
__copyright__ = '''
|
|
38
|
-
Copyright (c) 2000-
|
|
10
|
+
Copyright (c) 2000-2026, Board of Trustees of Leland Stanford Jr. University
|
|
39
11
|
'''.strip()
|
|
40
12
|
|
|
41
13
|
__license__ = __copyright__ + '\n\n' + '''
|
|
@@ -69,7 +41,7 @@ POSSIBILITY OF SUCH DAMAGE.
|
|
|
69
41
|
|
|
70
42
|
from base64 import b64encode
|
|
71
43
|
from urllib.request import Request, urlopen
|
|
72
|
-
from typing import Any
|
|
44
|
+
from typing import Any
|
|
73
45
|
|
|
74
46
|
|
|
75
47
|
type RequestUrlOpenT = Any
|
|
@@ -303,7 +275,7 @@ def _auid_action(node: Node, auid: str, action: str, **kwargs) -> RequestUrlOpen
|
|
|
303
275
|
``Force Deep Crawl``.
|
|
304
276
|
:type action: str
|
|
305
277
|
:param kwargs: Key-value pairs of additional query string arguments.
|
|
306
|
-
:type kwargs:
|
|
278
|
+
:type kwargs: dict[str, Any]
|
|
307
279
|
:return: The result of calling `urllib.request.urlopen`` on an appropriate
|
|
308
280
|
URL.
|
|
309
281
|
:rtype: RequestUrlOpenT
|
|
@@ -326,7 +298,7 @@ def _make_request(node: Node, query: str, **kwargs) -> Request:
|
|
|
326
298
|
:type query: str
|
|
327
299
|
:param kwargs: Key-value pairs of additional query string arguments, e.g.
|
|
328
300
|
``(..., depth=99)`` to add ``"&depth=99"``.
|
|
329
|
-
:type kwargs:
|
|
301
|
+
:type kwargs: dict[str, Any]
|
|
330
302
|
:return: An authenticated ``Request`` instance (before
|
|
331
303
|
``urllib.request.urlopen`` is called).
|
|
332
304
|
:rtype: Request
|
|
@@ -350,7 +322,7 @@ def _node_action(node: Node, action: str, **kwargs) -> RequestUrlOpenT:
|
|
|
350
322
|
:type action: str
|
|
351
323
|
:param kwargs: Key-value pairs of additional query string arguments, e.g.
|
|
352
324
|
``(..., depth=99)`` to add ``"&depth=99"``.
|
|
353
|
-
:type kwargs:
|
|
325
|
+
:type kwargs: dict[str, Any]
|
|
354
326
|
:return: The result of calling `urllib.request.urlopen`` on an appropriate
|
|
355
327
|
URL.
|
|
356
328
|
:rtype: RequestUrlOpenT
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
|
|
3
|
-
# Copyright (c) 2000-
|
|
3
|
+
# Copyright (c) 2000-2026, Board of Trustees of Leland Stanford Jr. University
|
|
4
4
|
#
|
|
5
5
|
# Redistribution and use in source and binary forms, with or without
|
|
6
6
|
# modification, are permitted provided that the following conditions are met:
|
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2000-2026, Board of Trustees of Leland Stanford Jr. University
|
|
4
|
+
#
|
|
5
|
+
# Redistribution and use in source and binary forms, with or without
|
|
6
|
+
# modification, are permitted provided that the following conditions are met:
|
|
7
|
+
#
|
|
8
|
+
# 1. Redistributions of source code must retain the above copyright notice,
|
|
9
|
+
# this list of conditions and the following disclaimer.
|
|
10
|
+
#
|
|
11
|
+
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
# this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
# and/or other materials provided with the distribution.
|
|
14
|
+
#
|
|
15
|
+
# 3. Neither the name of the copyright holder nor the names of its contributors
|
|
16
|
+
# may be used to endorse or promote products derived from this software without
|
|
17
|
+
# specific prior written permission.
|
|
18
|
+
#
|
|
19
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
22
|
+
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
23
|
+
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
24
|
+
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
25
|
+
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
26
|
+
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
27
|
+
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
28
|
+
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
29
|
+
# POSSIBILITY OF SUCH DAMAGE.
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
Command line tool to interact with the LOCKSS 1.x DebugPanel servlet.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from collections.abc import Callable, Iterator
|
|
36
|
+
from concurrent.futures import Executor, Future, ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
|
37
|
+
from contextlib import nullcontext
|
|
38
|
+
from dataclasses import dataclass, field
|
|
39
|
+
from enum import Enum
|
|
40
|
+
from importlib.metadata import entry_points
|
|
41
|
+
from inspect import ismethod
|
|
42
|
+
from itertools import chain
|
|
43
|
+
from pathlib import Path
|
|
44
|
+
from typing import Any, Optional
|
|
45
|
+
|
|
46
|
+
from click_extra import ChoiceSource, EnumChoice, ExtraContext, Section, TableFormat, color_option, echo, group, option, option_group, pass_context, pass_obj, print_table, progressbar, prompt, show_params_option
|
|
47
|
+
from click_plugins import with_plugins
|
|
48
|
+
from cloup.constraints import mutually_exclusive
|
|
49
|
+
|
|
50
|
+
from lockss.pybasic.cliutil import NonNegativeInt, click_path, compose_decorators, make_extra_context_settings, make_table_format_option
|
|
51
|
+
from lockss.pybasic.errorutil import InternalError
|
|
52
|
+
from lockss.pybasic.fileutil import file_lines
|
|
53
|
+
from . import Node, RequestUrlOpenT, check_substance, crawl, crawl_plugins, deep_crawl, disable_indexing, poll, reload_config, reindex_metadata, validate_files, DEFAULT_DEPTH, __copyright__, __license__, __version__
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _JobPoolType(Enum):
|
|
57
|
+
"""An enum of job pool types."""
|
|
58
|
+
THREAD_POOL = 'thread-pool'
|
|
59
|
+
PROCESS_POOL = 'process-pool'
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
#: The default ``_JobPoolType``.
|
|
63
|
+
_DEFAULT_JOB_POOL_TYPE: _JobPoolType = _JobPoolType.THREAD_POOL
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(kw_only=True)
|
|
67
|
+
class _Opts:
|
|
68
|
+
"""Data class to hold parsed command line options."""
|
|
69
|
+
# Node operation
|
|
70
|
+
node: tuple[str, ...] = ()
|
|
71
|
+
nodes: tuple[Path, ...] = ()
|
|
72
|
+
u: Optional[str] = None # DEPRECATED
|
|
73
|
+
username: Optional[str] = None
|
|
74
|
+
p: Optional[str] = field(default=None, repr=False) # DEPRECATED
|
|
75
|
+
password: Optional[str] = field(default=None, repr=False)
|
|
76
|
+
# AUID operation
|
|
77
|
+
auid: tuple[str, ...] = ()
|
|
78
|
+
auids: tuple[Path, ...] = ()
|
|
79
|
+
# Depth
|
|
80
|
+
depth: Optional[int] = None
|
|
81
|
+
# Job pool
|
|
82
|
+
pool_size: Optional[int] = None
|
|
83
|
+
pool_type: Optional[_JobPoolType] = None
|
|
84
|
+
process_pool: bool = False # DEPRECATED
|
|
85
|
+
thread_pool: bool = False # DEPRECATED
|
|
86
|
+
# Output
|
|
87
|
+
headings: Optional[bool] = None
|
|
88
|
+
progress: Optional[bool] = None
|
|
89
|
+
table_format: Optional[TableFormat] = None
|
|
90
|
+
|
|
91
|
+
def __post_init__(self):
|
|
92
|
+
"""Post-initialization method, to handle deprecated options."""
|
|
93
|
+
if self.u:
|
|
94
|
+
self.username, self.u = self.u, None
|
|
95
|
+
if self.p:
|
|
96
|
+
self.password, self.p = self.p, None
|
|
97
|
+
if self.process_pool:
|
|
98
|
+
self.pool_type, self.process_pool = _JobPoolType.PROCESS_POOL, False
|
|
99
|
+
if self.thread_pool:
|
|
100
|
+
self.pool_type, self.thread_pool = _JobPoolType.THREAD_POOL, False
|
|
101
|
+
if not self.username:
|
|
102
|
+
self.username = prompt('UI username')
|
|
103
|
+
if not self.password:
|
|
104
|
+
self.password = prompt('UI password', hide_input=True, confirmation_prompt=False)
|
|
105
|
+
if not self.pool_type:
|
|
106
|
+
self.pool_type = _DEFAULT_JOB_POOL_TYPE
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class _DebugPanelCli(object):
|
|
110
|
+
"""DebugPanel command line application."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, ctx: ExtraContext):
|
|
113
|
+
"""
|
|
114
|
+
Constructor.
|
|
115
|
+
|
|
116
|
+
:param ctx: The Click Extra context.
|
|
117
|
+
:type ctx: ExtraContext
|
|
118
|
+
"""
|
|
119
|
+
super().__init__()
|
|
120
|
+
self._ctx: ExtraContext = ctx
|
|
121
|
+
self._opts: Optional[_Opts] = None
|
|
122
|
+
self._auids: Optional[list[str]] = None
|
|
123
|
+
self._executor: Optional[Executor] = None
|
|
124
|
+
self._nodes: Optional[list[str]] = None
|
|
125
|
+
|
|
126
|
+
def check_substance(self) -> None:
|
|
127
|
+
"""Implementation of the ``check-substance`` command."""
|
|
128
|
+
self._do_auid_command(check_substance)
|
|
129
|
+
|
|
130
|
+
def crawl(self) -> None:
|
|
131
|
+
"""Implementation of the ``crawl`` command."""
|
|
132
|
+
self._do_auid_command(crawl)
|
|
133
|
+
|
|
134
|
+
def crawl_plugins(self) -> None:
|
|
135
|
+
"""Implementation of the ``crawl-plugins`` command."""
|
|
136
|
+
self._do_node_command(crawl_plugins)
|
|
137
|
+
|
|
138
|
+
def deep_crawl(self) -> None:
|
|
139
|
+
"""Implementation of the ``deep-crawl`` command."""
|
|
140
|
+
self._do_auid_command(deep_crawl, depth=self._opts.depth)
|
|
141
|
+
|
|
142
|
+
def disable_indexing(self) -> None:
|
|
143
|
+
"""Implementation of the ``disable-indexing`` command."""
|
|
144
|
+
self._do_auid_command(disable_indexing)
|
|
145
|
+
|
|
146
|
+
def dispatch(self, method: Callable[[], None], **cli_kwargs) -> None:
|
|
147
|
+
"""
|
|
148
|
+
Initializes from the given command line options and invokes the given
|
|
149
|
+
(bound) method.
|
|
150
|
+
|
|
151
|
+
:param method: A (bound) method.
|
|
152
|
+
:type method: Callable[[], None]
|
|
153
|
+
:param cli_kwargs: The command line arguments passed by Click Extra.
|
|
154
|
+
:type cli_kwargs: dict[str, Any]
|
|
155
|
+
"""
|
|
156
|
+
if not ismethod(method):
|
|
157
|
+
raise InternalError() from ValueError(method)
|
|
158
|
+
self._opts = _Opts(**cli_kwargs)
|
|
159
|
+
method()
|
|
160
|
+
|
|
161
|
+
def poll(self) -> None:
|
|
162
|
+
"""Implementation of the ``poll`` command."""
|
|
163
|
+
self._do_auid_command(poll)
|
|
164
|
+
|
|
165
|
+
def reindex_metadata(self) -> None:
|
|
166
|
+
"""Implementation of the ``reindex-metadata`` command."""
|
|
167
|
+
self._do_auid_command(reindex_metadata)
|
|
168
|
+
|
|
169
|
+
def reload_config(self) -> None:
|
|
170
|
+
"""Implementation of the ``reload-config`` command."""
|
|
171
|
+
self._do_node_command(reload_config)
|
|
172
|
+
|
|
173
|
+
def validate_files(self) -> None:
|
|
174
|
+
"""Implementation of the ``validate-files`` command."""
|
|
175
|
+
self._do_auid_command(validate_files)
|
|
176
|
+
|
|
177
|
+
def _do_auid_command(self,
|
|
178
|
+
node_auid_func: Callable[[Node, str], RequestUrlOpenT],
|
|
179
|
+
**kwargs) -> None:
|
|
180
|
+
"""
|
|
181
|
+
Performs one AUID-centric command.
|
|
182
|
+
|
|
183
|
+
:param node_auid_func: A function that applies to a ``Node`` and an AUID
|
|
184
|
+
and returns what ``urllib.request.urlopen``
|
|
185
|
+
returns.
|
|
186
|
+
:type node_auid_func: Callable[[Node, str], RequestUrlOpenT]
|
|
187
|
+
"""
|
|
188
|
+
self._initialize_auid_operation()
|
|
189
|
+
opts = self._opts
|
|
190
|
+
node_objects = [Node(node, opts.username, opts.password) for node in self._nodes]
|
|
191
|
+
futures: dict[Future, tuple[str, str]] = {self._executor.submit(node_auid_func, node_object, auid, **kwargs): (node, auid) for auid in self._auids for node, node_object in zip(self._nodes, node_objects)}
|
|
192
|
+
completed: Iterator[Future] = as_completed(futures)
|
|
193
|
+
results: dict[tuple[str, str], Any] = {}
|
|
194
|
+
with progressbar(completed, length=len(futures), label='Progress') if opts.progress else nullcontext(completed) as bar:
|
|
195
|
+
for future in bar:
|
|
196
|
+
node_auid = futures[future]
|
|
197
|
+
try:
|
|
198
|
+
with future.result() as resp:
|
|
199
|
+
status: int = resp.status
|
|
200
|
+
reason: str = resp.reason
|
|
201
|
+
results[node_auid] = 'Requested' if status == 200 else reason
|
|
202
|
+
except Exception as exc:
|
|
203
|
+
results[node_auid] = exc
|
|
204
|
+
print_table([[auid, *[results[(node, auid)] for node in self._nodes]] for auid in self._auids],
|
|
205
|
+
headers=['AUID', *self._nodes] if opts.headings else None,
|
|
206
|
+
table_format=opts.table_format)
|
|
207
|
+
|
|
208
|
+
def _do_node_command(self,
|
|
209
|
+
node_func: Callable[[Node], RequestUrlOpenT],
|
|
210
|
+
**kwargs) -> None:
|
|
211
|
+
"""
|
|
212
|
+
Performs one node-centric command.
|
|
213
|
+
|
|
214
|
+
:param node_func: A function that applies to a ``Node`` and returns
|
|
215
|
+
what ``urllib.request.urlopen`` returns.
|
|
216
|
+
:type node_func: Callable[[Node], RequestUrlOpenT]
|
|
217
|
+
"""
|
|
218
|
+
self._initialize_node_operation()
|
|
219
|
+
opts = self._opts
|
|
220
|
+
node_objects = [Node(node, opts.username, opts.password) for node in self._nodes]
|
|
221
|
+
futures: dict[Future, str] = {self._executor.submit(node_func, node_object, **kwargs): node for node, node_object in zip(self._nodes, node_objects)}
|
|
222
|
+
completed: Iterator[Future] = as_completed(futures)
|
|
223
|
+
results: dict[str, Any] = {}
|
|
224
|
+
with progressbar(completed, length=len(futures), label='Progress') if opts.progress else nullcontext(completed) as bar:
|
|
225
|
+
for future in bar:
|
|
226
|
+
node = futures[future]
|
|
227
|
+
try:
|
|
228
|
+
with future.result() as resp:
|
|
229
|
+
status: int = resp.status
|
|
230
|
+
reason: str = resp.reason
|
|
231
|
+
results[node] = 'Requested' if status == 200 else reason
|
|
232
|
+
except Exception as exc:
|
|
233
|
+
results[node] = exc
|
|
234
|
+
print_table([[node, results[node]] for node in self._nodes],
|
|
235
|
+
headers=['Node', 'Result'] if opts.headings else None,
|
|
236
|
+
table_format=self._opts.table_format)
|
|
237
|
+
|
|
238
|
+
def _initialize_auid_operation(self) -> None:
|
|
239
|
+
"""
|
|
240
|
+
Initializes for an AUID-centric operation. Fails if the list of AUIDs
|
|
241
|
+
ends up being empty.
|
|
242
|
+
"""
|
|
243
|
+
self._initialize_node_operation()
|
|
244
|
+
self._auids = [*(opts := self._opts).auid, *chain.from_iterable(file_lines(file_path) for file_path in opts.auids)]
|
|
245
|
+
if len(self._auids) == 0:
|
|
246
|
+
self._ctx.fail('The list of AUIDs to process is empty')
|
|
247
|
+
|
|
248
|
+
def _initialize_node_operation(self) -> None:
|
|
249
|
+
"""
|
|
250
|
+
Initializes for a node-centric operation. Fails if the list of nodes
|
|
251
|
+
ends up being empty.
|
|
252
|
+
"""
|
|
253
|
+
self._nodes = [*(opts := self._opts).node, *chain.from_iterable(file_lines(file_path) for file_path in opts.nodes)]
|
|
254
|
+
if len(self._nodes) == 0:
|
|
255
|
+
self._ctx.fail('The list of nodes to process is empty')
|
|
256
|
+
match opts.pool_type:
|
|
257
|
+
case _JobPoolType.PROCESS_POOL:
|
|
258
|
+
self._executor = ProcessPoolExecutor(max_workers=opts.pool_size)
|
|
259
|
+
case _JobPoolType.THREAD_POOL:
|
|
260
|
+
self._executor = ThreadPoolExecutor(max_workers=opts.pool_size)
|
|
261
|
+
case _:
|
|
262
|
+
raise InternalError() from ValueError(opts.pool_type)
|
|
263
|
+
if opts.username is None:
|
|
264
|
+
opts.username = prompt('UI username')
|
|
265
|
+
if opts.password is None:
|
|
266
|
+
opts.password = prompt('UI password', hide_input=True)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
#: The AUID option group: --auid/-a, --auids/-A
|
|
270
|
+
_auid_option_group = option_group(
|
|
271
|
+
'AUID options',
|
|
272
|
+
option('--auid', '-a', metavar='AUID', multiple=True, help='Add AUID to the list of AUIDs to process.'),
|
|
273
|
+
option('--auids', '-A', metavar='FILE', type=click_path('ferz'), multiple=True, help='Add the AUIDs in FILE to the list of AUIDs to process.')
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
#: The depth option group: --depth/-d
|
|
278
|
+
_depth_option_group = option_group(
|
|
279
|
+
'Depth options',
|
|
280
|
+
option('--depth', '-d', metavar='DEPTH', type=NonNegativeInt, default=DEFAULT_DEPTH, help='Set the crawl depth to DEPTH.')
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
#: The node option group: --node/-n, --nodes/-N, --username/-U, --password/-P
|
|
285
|
+
_node_option_group = option_group(
|
|
286
|
+
'Node options',
|
|
287
|
+
option('--node', '-n', metavar='NODE', multiple=True, help='Add NODE to the list of nodes to process.'),
|
|
288
|
+
option('--nodes', '-N', metavar='FILE', type=click_path('ferz'), multiple=True, help='Add the nodes in FILE to the list of nodes to process.'),
|
|
289
|
+
mutually_exclusive(
|
|
290
|
+
# option('--username', '-U', metavar='USER', show_default='interactive prompt', help='Set the UI username to USER.', prompt='UI username'),
|
|
291
|
+
option('--username', '-U', metavar='USER', show_default='interactive prompt', help='Set the UI username to USER.'),
|
|
292
|
+
option('-u', metavar='USER', deprecated='Use -U instead.')
|
|
293
|
+
),
|
|
294
|
+
mutually_exclusive(
|
|
295
|
+
# password_option('--password', '-P', metavar='PASS', show_default='interactive prompt', help='Set the UI password to PASS.', prompt='UI password', confirmation_prompt=False),
|
|
296
|
+
option('--password', '-P', metavar='PASS', show_default='interactive prompt', help='Set the UI password to PASS.'),
|
|
297
|
+
option('-p', metavar='PASS', deprecated='Use -P instead.')
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
#: The output option group: --headings/--no-headings, --progress/--no-progress, --table-format/-T
|
|
303
|
+
_output_option_group = option_group(
|
|
304
|
+
'Output options',
|
|
305
|
+
option('--headings/--no-headings', is_flag=True, default=True, help='Set whether to include column headings in tabular output.'),
|
|
306
|
+
option('--progress/--no-progress', is_flag=True, default=True, help='Set whether to display a progress bar during processing.'),
|
|
307
|
+
make_table_format_option()
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
#: The job pool option group: --pool-size, --pool-type
|
|
312
|
+
_pool_option_group = option_group(
|
|
313
|
+
'Job pool options',
|
|
314
|
+
option('--pool-size', metavar='SIZE', type=Optional[NonNegativeInt], default=None, help='Set the job pool size to SIZE.', show_default='CPU-dependent'),
|
|
315
|
+
mutually_exclusive(
|
|
316
|
+
# option('--pool-type', type=EnumChoice(choices=_JobPoolType, choice_source=ChoiceSource.VALUE), default=_DEFAULT_JOB_POOL_TYPE, help=f'Set the job pool type to the given type.'),
|
|
317
|
+
option('--pool-type', type=EnumChoice(choices=_JobPoolType, choice_source=ChoiceSource.VALUE), show_default=_DEFAULT_JOB_POOL_TYPE, help=f'Set the job pool type to the given type.'),
|
|
318
|
+
option('--process-pool', is_flag=True, deprecated='Use --pool-type=process-pool instead.'),
|
|
319
|
+
option('--thread-pool', is_flag=True, deprecated='Use --pool-type=thread-pool instead.')
|
|
320
|
+
)
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
#: The composite AUID operation decorator.
|
|
325
|
+
_auid_operation = compose_decorators(_node_option_group, _auid_option_group, _pool_option_group, _output_option_group, pass_obj)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
#: The composite node operation decorator.
|
|
329
|
+
_node_operation = compose_decorators(_node_option_group, _pool_option_group, _output_option_group, pass_obj)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
@with_plugins(entry_points(module='click_command_tree')) # adds a 'tree' command
|
|
333
|
+
@group('debugpanel', params=None, context_settings=make_extra_context_settings())
|
|
334
|
+
@color_option
|
|
335
|
+
@show_params_option
|
|
336
|
+
@pass_context
|
|
337
|
+
def _debugpanel(ctx: ExtraContext, **kwargs):
|
|
338
|
+
"""Command line tool to interact with the LOCKSS 1.x DebugPanel servlet."""
|
|
339
|
+
ctx.obj = _DebugPanelCli(ctx)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
#: A subcommand section for AUID commands.
|
|
343
|
+
_AUID_COMMANDS = Section('AUID commands')
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
#: A subcommand section for node commands.
|
|
347
|
+
_NODE_COMMANDS = Section('Node commands')
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
@_debugpanel.command('check-substance', aliases=['cs'], section=_AUID_COMMANDS, help='Cause nodes to check the substance of AUs.')
|
|
351
|
+
@_auid_operation
|
|
352
|
+
def _check_substance(cli: _DebugPanelCli, **kwargs) -> None:
|
|
353
|
+
"""Cause nodes to check the substance of AUs."""
|
|
354
|
+
cli.dispatch(cli.check_substance, **kwargs)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
@_debugpanel.command('copyright', help='Show the copyright and exit.')
|
|
358
|
+
def _copyright() -> None:
|
|
359
|
+
"""Show the copyright and exit."""
|
|
360
|
+
echo(__copyright__)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@_debugpanel.command('crawl', aliases=['cr'], section=_AUID_COMMANDS, help='Cause nodes to crawl AUs.')
|
|
364
|
+
@_auid_operation
|
|
365
|
+
def _crawl(cli: _DebugPanelCli, **kwargs) -> None:
|
|
366
|
+
"""Cause nodes to crawl AUs."""
|
|
367
|
+
cli.dispatch(cli.crawl, **kwargs)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
@_debugpanel.command('crawl-plugins', aliases=['cp'], section=_NODE_COMMANDS, help='Cause nodes to crawl plugins.')
|
|
371
|
+
@_node_operation
|
|
372
|
+
def _crawl_plugins(cli: _DebugPanelCli, **kwargs) -> None:
|
|
373
|
+
"""Cause nodes to crawl plugins."""
|
|
374
|
+
cli.dispatch(cli.crawl_plugins, **kwargs)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@_debugpanel.command('deep-crawl', aliases=['dc'], section=_AUID_COMMANDS, help='Cause nodes to deep-crawl AUs.')
|
|
378
|
+
@compose_decorators(_node_option_group, _auid_option_group, _depth_option_group, _pool_option_group, _output_option_group, pass_obj)
|
|
379
|
+
def _deep_crawl(cli: _DebugPanelCli, **kwargs) -> None:
|
|
380
|
+
"""Cause nodes to deep-crawl AUs."""
|
|
381
|
+
cli.dispatch(cli.deep_crawl, **kwargs)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@_debugpanel.command('disable-indexing', aliases=['di'], section=_AUID_COMMANDS, help='Cause nodes to disable metadata indexing for AUs.')
|
|
385
|
+
@_auid_operation
|
|
386
|
+
def _disable_indexing(cli: _DebugPanelCli, **kwargs) -> None:
|
|
387
|
+
"""Cause nodes to disable metadata indexing for AUs."""
|
|
388
|
+
cli.dispatch(cli.disable_indexing, **kwargs)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
@_debugpanel.command('license', help='Show the software license and exit.')
|
|
392
|
+
def license() -> None:
|
|
393
|
+
"""Show the software license and exit."""
|
|
394
|
+
echo(__license__)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@_debugpanel.command('poll', aliases=['po'], section=_AUID_COMMANDS, help='Cause nodes to poll AUs.')
|
|
398
|
+
@_auid_operation
|
|
399
|
+
def _poll(cli: _DebugPanelCli, **kwargs) -> None:
|
|
400
|
+
"""Cause nodes to poll AUs."""
|
|
401
|
+
cli.dispatch(cli.poll, **kwargs)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@_debugpanel.command('reload-config', aliases=['rc'], section=_NODE_COMMANDS, help='Cause nodes to reload their configuration.')
|
|
405
|
+
@_node_operation
|
|
406
|
+
def _reload_config(cli: _DebugPanelCli, **kwargs) -> None:
|
|
407
|
+
"""Cause nodes to reload their configuration."""
|
|
408
|
+
cli.dispatch(cli.reload_config, **kwargs)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
@_debugpanel.command('reindex-metadata', aliases=['ri'], section=_AUID_COMMANDS, help='Cause nodes to reindex the metadata of AUs.')
|
|
412
|
+
@_auid_operation
|
|
413
|
+
def _reindex_metadata(cli: _DebugPanelCli, **kwargs) -> None:
|
|
414
|
+
"""Cause nodes to reindex the metadata of AUs."""
|
|
415
|
+
cli.dispatch(cli.reindex_metadata, **kwargs)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
@_debugpanel.command('validate-files', aliases=['vf'], section=_AUID_COMMANDS, help='Cause nodes to validate the files of AUs.')
|
|
419
|
+
@_auid_operation
|
|
420
|
+
def _validate_files(cli: _DebugPanelCli, **kwargs) -> None:
|
|
421
|
+
"""Cause nodes to validate the files of AUs."""
|
|
422
|
+
cli.dispatch(cli.validate_files, **kwargs)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@_debugpanel.command('version', help='Show the version number and exit.')
|
|
426
|
+
def version() -> None:
|
|
427
|
+
"""Show the version number and exit."""
|
|
428
|
+
echo(__version__)
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def main() -> None:
|
|
432
|
+
"""Main entry point of the module."""
|
|
433
|
+
_debugpanel()
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# Main entry point of the module.
|
|
437
|
+
if __name__ == '__main__':
|
|
438
|
+
main()
|
|
@@ -1,379 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
|
|
3
|
-
# Copyright (c) 2000-2025, Board of Trustees of Leland Stanford Jr. University
|
|
4
|
-
#
|
|
5
|
-
# Redistribution and use in source and binary forms, with or without
|
|
6
|
-
# modification, are permitted provided that the following conditions are met:
|
|
7
|
-
#
|
|
8
|
-
# 1. Redistributions of source code must retain the above copyright notice,
|
|
9
|
-
# this list of conditions and the following disclaimer.
|
|
10
|
-
#
|
|
11
|
-
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
-
# this list of conditions and the following disclaimer in the documentation
|
|
13
|
-
# and/or other materials provided with the distribution.
|
|
14
|
-
#
|
|
15
|
-
# 3. Neither the name of the copyright holder nor the names of its contributors
|
|
16
|
-
# may be used to endorse or promote products derived from this software without
|
|
17
|
-
# specific prior written permission.
|
|
18
|
-
#
|
|
19
|
-
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
-
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
-
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
22
|
-
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
23
|
-
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
24
|
-
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
25
|
-
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
26
|
-
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
27
|
-
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
28
|
-
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
29
|
-
# POSSIBILITY OF SUCH DAMAGE.
|
|
30
|
-
|
|
31
|
-
"""
|
|
32
|
-
Command line tool to interact with the LOCKSS 1.x DebugPanel servlet.
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
from collections.abc import Callable
|
|
36
|
-
from concurrent.futures import Executor, Future, ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
|
37
|
-
from enum import Enum
|
|
38
|
-
from getpass import getpass
|
|
39
|
-
from itertools import chain
|
|
40
|
-
from pathlib import Path
|
|
41
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
42
|
-
|
|
43
|
-
from pydantic.v1 import BaseModel, Field, FilePath, root_validator, validator
|
|
44
|
-
from pydantic.v1.types import PositiveInt
|
|
45
|
-
from tabulate import tabulate
|
|
46
|
-
|
|
47
|
-
from lockss.pybasic.cliutil import BaseCli, StringCommand, at_most_one_from_enum, get_from_enum, COPYRIGHT_DESCRIPTION, LICENSE_DESCRIPTION, VERSION_DESCRIPTION
|
|
48
|
-
from lockss.pybasic.errorutil import InternalError
|
|
49
|
-
from lockss.pybasic.fileutil import file_lines, path
|
|
50
|
-
from lockss.pybasic.outpututil import OutputFormatOptions
|
|
51
|
-
from . import Node, RequestUrlOpenT, check_substance, crawl, crawl_plugins, deep_crawl, disable_indexing, poll, reload_config, reindex_metadata, validate_files, DEFAULT_DEPTH, __copyright__, __license__, __version__
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
class JobPool(Enum):
|
|
55
|
-
"""
|
|
56
|
-
An enum of job pool types.
|
|
57
|
-
|
|
58
|
-
See also ``DEFAULT_POOL_TYPE``.
|
|
59
|
-
"""
|
|
60
|
-
thread_pool = 'thread-pool'
|
|
61
|
-
process_pool = 'process-pool'
|
|
62
|
-
|
|
63
|
-
@staticmethod
|
|
64
|
-
def from_option(name: str) -> str:
|
|
65
|
-
"""
|
|
66
|
-
Given an option name with hyphens, return the enum constant name with
|
|
67
|
-
underscores.
|
|
68
|
-
|
|
69
|
-
:param name: An option name with hyphens.
|
|
70
|
-
:type name: str
|
|
71
|
-
:return: The corresponding enum constant name with underscores.
|
|
72
|
-
:rtype: str
|
|
73
|
-
"""
|
|
74
|
-
return JobPool(name.replace('-', '_'))
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
DEFAULT_POOL_SIZE: Optional[int] = None
|
|
78
|
-
DEFAULT_POOL_TYPE: JobPool = JobPool.thread_pool
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
class NodesOptions(BaseModel):
|
|
82
|
-
"""
|
|
83
|
-
The --node/-n, --nodes/-N, --password/-p and --username/-u options.
|
|
84
|
-
"""
|
|
85
|
-
node: Optional[List[str]] = Field([], aliases=['-n'], description='(nodes) add one or more nodes to the set of nodes to process')
|
|
86
|
-
nodes: Optional[List[FilePath]] = Field([], aliases=['-N'], description='(nodes) add the nodes listed in one or more files to the set of nodes to process')
|
|
87
|
-
password: Optional[str] = Field(aliases=['-p'], description='(nodes) UI password; interactive prompt if not specified')
|
|
88
|
-
username: Optional[str] = Field(aliases=['-u'], description='(nodes) UI username; interactive prompt if not unspecified')
|
|
89
|
-
|
|
90
|
-
@validator('nodes', each_item=True, pre=True)
|
|
91
|
-
def _expand_each_nodes_path(cls, v: Path):
|
|
92
|
-
return path(v)
|
|
93
|
-
|
|
94
|
-
def get_nodes(self):
|
|
95
|
-
ret = [*self.node, *chain.from_iterable(file_lines(file_path) for file_path in self.nodes)]
|
|
96
|
-
if len(ret) == 0:
|
|
97
|
-
raise RuntimeError('empty list of nodes')
|
|
98
|
-
return ret
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
class AuidsOptions(BaseModel):
|
|
102
|
-
"""
|
|
103
|
-
The --auid/-a and --auids/-A options.
|
|
104
|
-
"""
|
|
105
|
-
auid: Optional[List[str]] = Field([], aliases=['-a'], description='(AUIDs) add one or more AUIDs to the set of AUIDs to process')
|
|
106
|
-
auids: Optional[List[FilePath]] = Field([], aliases=['-A'], description='(AUIDs) add the AUIDs listed in one or more files to the set of AUIDs to process')
|
|
107
|
-
|
|
108
|
-
@validator('auids', each_item=True, pre=True)
|
|
109
|
-
def _expand_each_auids_path(cls, v: Path):
|
|
110
|
-
return path(v)
|
|
111
|
-
|
|
112
|
-
def get_auids(self):
|
|
113
|
-
ret = [*self.auid, *chain.from_iterable(file_lines(file_path) for file_path in self.auids)]
|
|
114
|
-
if len(ret) == 0:
|
|
115
|
-
raise RuntimeError('empty list of AUIDs')
|
|
116
|
-
return ret
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
class DepthOptions(BaseModel):
|
|
120
|
-
"""
|
|
121
|
-
The --depth/-d option.
|
|
122
|
-
"""
|
|
123
|
-
depth: Optional[int] = Field(DEFAULT_DEPTH, aliases=['-d'], description='(deep crawl) set crawl depth')
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
class JobPoolOptions(BaseModel):
|
|
127
|
-
"""
|
|
128
|
-
The --pool-size, --process-pool and --thread-pool options.
|
|
129
|
-
"""
|
|
130
|
-
pool_size: Optional[PositiveInt] = Field(description='(job pool) set the job pool size')
|
|
131
|
-
process_pool: Optional[bool] = Field(False, description='(job pool) use a process pool', enum=JobPool)
|
|
132
|
-
thread_pool: Optional[bool] = Field(False, description='(job pool) use a thread pool', enum=JobPool)
|
|
133
|
-
|
|
134
|
-
@root_validator
|
|
135
|
-
def _at_most_one_pool_type(cls, values):
|
|
136
|
-
return at_most_one_from_enum(cls, values, JobPool)
|
|
137
|
-
|
|
138
|
-
def get_pool_size(self) -> Optional[int]:
|
|
139
|
-
return self.pool_size if hasattr(self, 'pool_size') else DEFAULT_POOL_SIZE
|
|
140
|
-
|
|
141
|
-
def get_pool_type(self) -> JobPool:
|
|
142
|
-
return get_from_enum(self, JobPool, DEFAULT_POOL_TYPE)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
class NodeCommand(OutputFormatOptions, JobPoolOptions, NodesOptions):
|
|
146
|
-
"""
|
|
147
|
-
A pydantic-argparse command for node commands.
|
|
148
|
-
"""
|
|
149
|
-
pass
|
|
150
|
-
|
|
151
|
-
class AuidCommand(NodeCommand, OutputFormatOptions, JobPoolOptions, AuidsOptions, NodesOptions):
|
|
152
|
-
"""
|
|
153
|
-
A pydantic-argparse command for AUID commands except deep-crawl.
|
|
154
|
-
"""
|
|
155
|
-
pass
|
|
156
|
-
|
|
157
|
-
class DeepCrawlCommand(AuidCommand, OutputFormatOptions, JobPoolOptions, DepthOptions, AuidsOptions, NodesOptions):
|
|
158
|
-
"""
|
|
159
|
-
A pydantic-argparse command for deep-crawl.
|
|
160
|
-
"""
|
|
161
|
-
pass
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
class DebugPanelCommand(BaseModel):
|
|
165
|
-
"""
|
|
166
|
-
The pydantic-argparse model for the top-level debugpanel command.
|
|
167
|
-
"""
|
|
168
|
-
check_substance: Optional[AuidCommand] = Field(description='cause nodes to check the substance of AUs', alias='check-substance')
|
|
169
|
-
copyright: Optional[StringCommand.type(__copyright__)] = Field(description=COPYRIGHT_DESCRIPTION)
|
|
170
|
-
cp: Optional[NodeCommand] = Field(description='synonym for: crawl-plugins')
|
|
171
|
-
cr: Optional[AuidCommand] = Field(description='synonym for: crawl')
|
|
172
|
-
crawl: Optional[AuidCommand] = Field(description='cause nodes to crawl AUs')
|
|
173
|
-
crawl_plugins: Optional[NodeCommand] = Field(description='cause nodes to crawl plugins', alias='crawl-plugins')
|
|
174
|
-
cs: Optional[AuidCommand] = Field(description='synonym for: check-substance')
|
|
175
|
-
dc: Optional[DeepCrawlCommand] = Field(description='synonym for: deep-crawl')
|
|
176
|
-
deep_crawl: Optional[DeepCrawlCommand] = Field(description='cause nodes to deeply crawl AUs', alias='deep-crawl')
|
|
177
|
-
di: Optional[AuidCommand] = Field(description='synonym for: disable-indexing')
|
|
178
|
-
disable_indexing: Optional[AuidCommand] = Field(description='cause nodes to disable metadata indexing for AUs', alias='disable-indexing')
|
|
179
|
-
license: Optional[StringCommand.type(__license__)] = Field(description=LICENSE_DESCRIPTION)
|
|
180
|
-
po: Optional[AuidCommand] = Field(description='synonym for: poll')
|
|
181
|
-
poll: Optional[AuidCommand] = Field(description='cause nodes to poll AUs')
|
|
182
|
-
rc: Optional[NodeCommand] = Field(description='synonym for: reload-config')
|
|
183
|
-
reindex_metadata: Optional[AuidCommand] = Field(description='cause nodes to reindex the metadata of AUs', alias='reindex-metadata')
|
|
184
|
-
reload_config: Optional[NodeCommand] = Field(description='cause nodes to reload their configuration', alias='reload-config')
|
|
185
|
-
ri: Optional[AuidCommand] = Field(description='synonym for: reindex-metadata')
|
|
186
|
-
validate_files: Optional[AuidCommand] = Field(description='cause nodes to validate the files of AUs', alias='validate-files')
|
|
187
|
-
version: Optional[StringCommand.type(__version__)] = Field(description=VERSION_DESCRIPTION)
|
|
188
|
-
vf: Optional[AuidCommand] = Field(description='synonym for: validate-files')
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
class DebugPanelCli(BaseCli[DebugPanelCommand]):
|
|
192
|
-
"""
|
|
193
|
-
The debugpanel command line tool.
|
|
194
|
-
"""
|
|
195
|
-
|
|
196
|
-
def __init__(self):
|
|
197
|
-
"""
|
|
198
|
-
Constructs a new ``DebugPanelCli`` instance.
|
|
199
|
-
"""
|
|
200
|
-
super().__init__(model=DebugPanelCommand,
|
|
201
|
-
prog='debugpanel',
|
|
202
|
-
description='Tool to interact with the LOCKSS 1.x DebugPanel servlet')
|
|
203
|
-
self._auids: Optional[List[str]] = None
|
|
204
|
-
self._auth: Optional[Any] = None
|
|
205
|
-
self._executor: Optional[Executor] = None
|
|
206
|
-
self._nodes: Optional[List[str]] = None
|
|
207
|
-
|
|
208
|
-
def _check_substance(self, auid_command: AuidCommand) -> None:
|
|
209
|
-
self._do_auid_command(auid_command, check_substance)
|
|
210
|
-
|
|
211
|
-
def _copyright(self, string_command: StringCommand) -> None:
|
|
212
|
-
self._do_string_command(string_command)
|
|
213
|
-
|
|
214
|
-
def _cp(self, node_command: NodeCommand) -> None:
|
|
215
|
-
self._crawl_plugins(node_command)
|
|
216
|
-
|
|
217
|
-
def _cr(self, auid_command: AuidCommand) -> None:
|
|
218
|
-
self._crawl(auid_command)
|
|
219
|
-
|
|
220
|
-
def _crawl(self, auid_command: AuidCommand) -> None:
|
|
221
|
-
self._do_auid_command(auid_command, crawl)
|
|
222
|
-
|
|
223
|
-
def _crawl_plugins(self, node_command: NodeCommand) -> None:
|
|
224
|
-
self._do_node_command(node_command, crawl_plugins)
|
|
225
|
-
|
|
226
|
-
def _cs(self, auid_command: AuidCommand) -> None:
|
|
227
|
-
self._check_substance(auid_command)
|
|
228
|
-
|
|
229
|
-
def _dc(self, deep_crawl_command: DeepCrawlCommand) -> None:
|
|
230
|
-
self._deep_crawl(deep_crawl_command)
|
|
231
|
-
|
|
232
|
-
def _deep_crawl(self, deep_crawl_command: DeepCrawlCommand) -> None:
|
|
233
|
-
self._do_auid_command(deep_crawl_command, deep_crawl, depth=deep_crawl_command.depth)
|
|
234
|
-
|
|
235
|
-
def _di(self, auid_command: AuidCommand) -> None:
|
|
236
|
-
self._disable_indexing(auid_command)
|
|
237
|
-
|
|
238
|
-
def _disable_indexing(self, auid_command: AuidCommand) -> None:
|
|
239
|
-
self._do_auid_command(auid_command, disable_indexing)
|
|
240
|
-
|
|
241
|
-
def _do_auid_command(self, auid_command: AuidCommand, node_auid_func: Callable[[Node, str], RequestUrlOpenT], **kwargs: Dict[str, Any]) -> None:
|
|
242
|
-
"""
|
|
243
|
-
Performs one AUID-centric command.
|
|
244
|
-
|
|
245
|
-
:param auid_command: An ``AuidCommand`` model.
|
|
246
|
-
:type auid_command: AuidCommand
|
|
247
|
-
:param node_auid_func: A function that applies to a ``Node`` and an AUID
|
|
248
|
-
and returns what ``urllib.request.urlopen``
|
|
249
|
-
returns.
|
|
250
|
-
:type node_auid_func: ``RequestUrlOpenT``
|
|
251
|
-
:param kwargs: Keyword arguments (needed for the ``depth`` command).
|
|
252
|
-
:type kwargs: Dict[str, Any]
|
|
253
|
-
"""
|
|
254
|
-
self._initialize_auth(auid_command)
|
|
255
|
-
self._initialize_executor(auid_command)
|
|
256
|
-
self._nodes = auid_command.get_nodes()
|
|
257
|
-
self._auids = auid_command.get_auids()
|
|
258
|
-
node_objects = [Node(node, *self._auth) for node in self._nodes]
|
|
259
|
-
futures: Dict[Future, Tuple[str, str]] = {self._executor.submit(node_auid_func, node_object, auid, **kwargs): (node, auid) for auid in self._auids for node, node_object in zip(self._nodes, node_objects)}
|
|
260
|
-
results: Dict[Tuple[str, str], Any] = {}
|
|
261
|
-
for future in as_completed(futures):
|
|
262
|
-
node_auid = futures[future]
|
|
263
|
-
try:
|
|
264
|
-
resp: RequestUrlOpenT = future.result()
|
|
265
|
-
status: int = resp.status
|
|
266
|
-
reason: str = resp.reason
|
|
267
|
-
results[node_auid] = 'Requested' if status == 200 else reason
|
|
268
|
-
except Exception as exc:
|
|
269
|
-
results[node_auid] = exc
|
|
270
|
-
print(tabulate([[auid, *[results[(node, auid)] for node in self._nodes]] for auid in self._auids],
|
|
271
|
-
headers=['AUID', *self._nodes],
|
|
272
|
-
tablefmt=auid_command.output_format))
|
|
273
|
-
|
|
274
|
-
def _do_node_command(self, node_command: NodeCommand, node_func: Callable[[Node], RequestUrlOpenT], **kwargs: Dict[str, Any]) -> None:
|
|
275
|
-
"""
|
|
276
|
-
Performs one node-centric command.
|
|
277
|
-
|
|
278
|
-
:param node_command: A ``NodeCommand`` model.
|
|
279
|
-
:type auid_command: NodeCommand
|
|
280
|
-
:param node_func: A function that applies to a ``Node`` and returns
|
|
281
|
-
what ``urllib.request.urlopen`` returns.
|
|
282
|
-
:type node_auid_func: ``RequestUrlOpenT``
|
|
283
|
-
:param kwargs: Keyword arguments (not currently needed by any command).
|
|
284
|
-
:type kwargs: Dict[str, Any]
|
|
285
|
-
"""
|
|
286
|
-
self._initialize_auth(node_command)
|
|
287
|
-
self._initialize_executor(node_command)
|
|
288
|
-
self._nodes = node_command.get_nodes()
|
|
289
|
-
node_objects = [Node(node, *self._auth) for node in self._nodes]
|
|
290
|
-
futures: Dict[Future, str] = {self._executor.submit(node_func, node_object, **kwargs): node for node, node_object in zip(self._nodes, node_objects)}
|
|
291
|
-
results: Dict[str, Any] = {}
|
|
292
|
-
for future in as_completed(futures):
|
|
293
|
-
node = futures[future]
|
|
294
|
-
try:
|
|
295
|
-
resp: RequestUrlOpenT = future.result()
|
|
296
|
-
status: int = resp.status
|
|
297
|
-
reason: str = resp.reason
|
|
298
|
-
results[node] = 'Requested' if status == 200 else reason
|
|
299
|
-
except Exception as exc:
|
|
300
|
-
results[node] = exc
|
|
301
|
-
print(tabulate([[node, results[node]] for node in self._nodes],
|
|
302
|
-
headers=['Node', 'Result'],
|
|
303
|
-
tablefmt=node_command.output_format))
|
|
304
|
-
|
|
305
|
-
def _do_string_command(self, string_command: StringCommand) -> None:
|
|
306
|
-
"""
|
|
307
|
-
Performs one string command.
|
|
308
|
-
|
|
309
|
-
:param string_command: A ``StringCommand`` model.
|
|
310
|
-
:type auid_command: StringCommand
|
|
311
|
-
"""
|
|
312
|
-
string_command()
|
|
313
|
-
|
|
314
|
-
def _initialize_auth(self, nodes_options: NodesOptions) -> None:
|
|
315
|
-
"""
|
|
316
|
-
Computes the ``self._auth`` value, possibly after asking for interactive
|
|
317
|
-
input.
|
|
318
|
-
|
|
319
|
-
:param nodes_options: A ``NodesOptions`` model.
|
|
320
|
-
:type node_options: ``NodesOptions``
|
|
321
|
-
"""
|
|
322
|
-
_u = nodes_options.username or input('UI username: ')
|
|
323
|
-
_p = nodes_options.password or getpass('UI password: ')
|
|
324
|
-
self._auth = (_u, _p)
|
|
325
|
-
|
|
326
|
-
def _initialize_executor(self, job_pool_options: JobPoolOptions) -> None:
|
|
327
|
-
"""
|
|
328
|
-
Initializes the ``Executor``.
|
|
329
|
-
|
|
330
|
-
:param job_pool_options: A ``JobPoolOptions`` model.
|
|
331
|
-
:type job_pool_options: ``JobPoolOptions``.
|
|
332
|
-
"""
|
|
333
|
-
if job_pool_options.get_pool_type() == JobPool.thread_pool:
|
|
334
|
-
self._executor = ThreadPoolExecutor(max_workers=job_pool_options.get_pool_size())
|
|
335
|
-
elif job_pool_options.get_pool_type() == JobPool.process_pool:
|
|
336
|
-
self._executor = ProcessPoolExecutor(max_workers=job_pool_options.get_pool_size())
|
|
337
|
-
else:
|
|
338
|
-
raise InternalError()
|
|
339
|
-
|
|
340
|
-
def _license(self, string_command: StringCommand) -> None:
|
|
341
|
-
self._do_string_command(string_command)
|
|
342
|
-
|
|
343
|
-
def _po(self, auid_command: AuidCommand) -> None:
|
|
344
|
-
self._poll(auid_command)
|
|
345
|
-
|
|
346
|
-
def _poll(self, auid_command: AuidCommand) -> None:
|
|
347
|
-
self._do_auid_command(auid_command, poll)
|
|
348
|
-
|
|
349
|
-
def _rc(self, node_command: NodeCommand):
|
|
350
|
-
self._reload_config(node_command)
|
|
351
|
-
|
|
352
|
-
def _ri(self, auid_command: AuidCommand) -> None:
|
|
353
|
-
self._reindex_metadata(auid_command)
|
|
354
|
-
|
|
355
|
-
def _reindex_metadata(self, auid_command: AuidCommand) -> None:
|
|
356
|
-
self._do_auid_command(auid_command, reindex_metadata)
|
|
357
|
-
|
|
358
|
-
def _reload_config(self, node_command: NodeCommand):
|
|
359
|
-
self._do_node_command(node_command, reload_config)
|
|
360
|
-
|
|
361
|
-
def _validate_files(self, auid_command: AuidCommand) -> None:
|
|
362
|
-
self._do_auid_command(auid_command, validate_files)
|
|
363
|
-
|
|
364
|
-
def _vf(self, auid_command: AuidCommand) -> None:
|
|
365
|
-
self._validate_files(auid_command)
|
|
366
|
-
|
|
367
|
-
def _version(self, string_command: StringCommand) -> None:
|
|
368
|
-
self._do_string_command(string_command)
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
def main() -> None:
|
|
372
|
-
"""
|
|
373
|
-
Entry point for the debugpanel command line tool.
|
|
374
|
-
"""
|
|
375
|
-
DebugPanelCli().run()
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
if __name__ == '__main__':
|
|
379
|
-
main()
|