workpytools 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- workpytools/__init__.py +1 -0
- workpytools/cli.py +76 -0
- workpytools/common/__init__.py +0 -0
- workpytools/common/browser_preview.py +40 -0
- workpytools/common/clipboard.py +318 -0
- workpytools/common/clustering.py +85 -0
- workpytools/common/config.py +44 -0
- workpytools/common/embedding.py +101 -0
- workpytools/common/help_gen.py +542 -0
- workpytools/common/logging.py +12 -0
- workpytools/common/markdown_html.py +15 -0
- workpytools/common/outline_parse.py +81 -0
- workpytools/common/output.py +71 -0
- workpytools/common/powerpoint.py +43 -0
- workpytools/common/shape_cluster.py +75 -0
- workpytools/common/table_shapes.py +157 -0
- workpytools/common/tabular.py +248 -0
- workpytools/common/textfile.py +90 -0
- workpytools/common/walk.py +133 -0
- workpytools/data/__init__.py +0 -0
- workpytools/data/synonym.tsv +3 -0
- workpytools/data/user.dic +2 -0
- workpytools/processing/__init__.py +0 -0
- workpytools/processing/base.py +23 -0
- workpytools/processing/clipfmt.py +143 -0
- workpytools/processing/clipmd.py +237 -0
- workpytools/processing/clipview.py +194 -0
- workpytools/processing/cwc.py +427 -0
- workpytools/processing/denoise.py +81 -0
- workpytools/processing/help.py +47 -0
- workpytools/processing/ikko.py +205 -0
- workpytools/processing/kukiri.py +94 -0
- workpytools/processing/lsdir.py +218 -0
- workpytools/processing/mdtsv.py +177 -0
- workpytools/processing/mokuji.py +91 -0
- workpytools/processing/nagasa.py +94 -0
- workpytools/processing/outline.py +85 -0
- workpytools/processing/profiler.py +277 -0
- workpytools/processing/seiretsu.py +111 -0
- workpytools/processing/tbl.py +358 -0
- workpytools/processing/touka.py +52 -0
- workpytools/processing/vv.py +147 -0
- workpytools-0.1.4.dist-info/METADATA +458 -0
- workpytools-0.1.4.dist-info/RECORD +48 -0
- workpytools-0.1.4.dist-info/WHEEL +5 -0
- workpytools-0.1.4.dist-info/entry_points.txt +20 -0
- workpytools-0.1.4.dist-info/licenses/LICENSE +21 -0
- workpytools-0.1.4.dist-info/top_level.txt +1 -0
workpytools/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Collection of Python processing utilities."""
|
workpytools/cli.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import importlib
|
|
5
|
+
import inspect
|
|
6
|
+
import pkgutil
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import PureWindowsPath
|
|
9
|
+
|
|
10
|
+
from workpytools import processing
|
|
11
|
+
from workpytools.common.logging import setup_logging
|
|
12
|
+
from workpytools.processing.base import Processor
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _discover_processors() -> dict[str, Processor]:
|
|
16
|
+
"""Import every module under `workpytools.processing` and collect Processor subclasses."""
|
|
17
|
+
processors: dict[str, Processor] = {}
|
|
18
|
+
for module_info in pkgutil.iter_modules(processing.__path__, prefix=f"{processing.__name__}."):
|
|
19
|
+
if module_info.name.endswith(".base"):
|
|
20
|
+
continue
|
|
21
|
+
try:
|
|
22
|
+
module = importlib.import_module(module_info.name)
|
|
23
|
+
except ImportError as exc:
|
|
24
|
+
print(f"warning: skipping {module_info.name}: {exc}", file=sys.stderr)
|
|
25
|
+
continue
|
|
26
|
+
for _, obj in inspect.getmembers(module, inspect.isclass):
|
|
27
|
+
if issubclass(obj, Processor) and obj is not Processor:
|
|
28
|
+
instance = obj()
|
|
29
|
+
processors[instance.name] = instance
|
|
30
|
+
return processors
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def build_parser(processors: dict[str, Processor]) -> argparse.ArgumentParser:
|
|
34
|
+
parser = argparse.ArgumentParser(prog="tools", description="Collection of processing commands")
|
|
35
|
+
parser.add_argument("--log-level", default="INFO", help="Logging level (default: INFO)")
|
|
36
|
+
|
|
37
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
38
|
+
for proc in processors.values():
|
|
39
|
+
sub = subparsers.add_parser(proc.name, help=proc.help)
|
|
40
|
+
proc.add_arguments(sub)
|
|
41
|
+
return parser
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def main(argv: list[str] | None = None) -> int:
|
|
45
|
+
processors = _discover_processors()
|
|
46
|
+
parser = build_parser(processors)
|
|
47
|
+
args = parser.parse_args(argv)
|
|
48
|
+
|
|
49
|
+
setup_logging(args.log_level)
|
|
50
|
+
|
|
51
|
+
processor = processors[args.command]
|
|
52
|
+
return processor.run(args)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_ENTRY_POINT_ALIASES = {
|
|
56
|
+
# toolh.exe だけは他の1コマンド=1exe規則に沿わない特別なエントリーポイント名
|
|
57
|
+
# (help.exe は分かりにくいので toolh とした)。
|
|
58
|
+
"toolh": "help",
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def run_as_subcommand() -> int:
|
|
63
|
+
"""Entry point for a per-command executable (e.g. `denoise.exe`).
|
|
64
|
+
|
|
65
|
+
The subcommand name is taken from how this executable was invoked (its
|
|
66
|
+
own file name), so `denoise.exe args...` behaves like `tools denoise
|
|
67
|
+
args...`. Register one `[project.scripts]` entry per Processor name in
|
|
68
|
+
pyproject.toml, all pointing to this same function.
|
|
69
|
+
"""
|
|
70
|
+
command_name = PureWindowsPath(sys.argv[0]).stem
|
|
71
|
+
command_name = _ENTRY_POINT_ALIASES.get(command_name, command_name)
|
|
72
|
+
return main([command_name, *sys.argv[1:]])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
sys.exit(main())
|
|
File without changes
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import itertools
|
|
4
|
+
import logging
|
|
5
|
+
import tempfile
|
|
6
|
+
import time
|
|
7
|
+
import webbrowser
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
_query_counter = itertools.count()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def cache_busting_query() -> str:
|
|
16
|
+
"""Wall-clock time plus a per-process counter, so consecutive calls
|
|
17
|
+
within the same clock tick still differ. Used to defeat the browser's
|
|
18
|
+
tendency to not reload a fixed-name file:// URL it already has open.
|
|
19
|
+
"""
|
|
20
|
+
return f"{time.time_ns()}-{next(_query_counter)}"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def write_and_open(html: str, filename: str, no_open: bool) -> Path:
|
|
24
|
+
"""Write `html` to a fixed-name file under %TEMP% and open it in the
|
|
25
|
+
default browser (unless `no_open`). Reused by every command that
|
|
26
|
+
previews a result in the browser (clipview, profiler --view, ...).
|
|
27
|
+
"""
|
|
28
|
+
preview_path = Path(tempfile.gettempdir()) / filename
|
|
29
|
+
try:
|
|
30
|
+
preview_path.write_text(html, encoding="utf-8")
|
|
31
|
+
except OSError as exc:
|
|
32
|
+
raise RuntimeError(f"一時ファイルの書き込みに失敗しました: {preview_path}") from exc
|
|
33
|
+
|
|
34
|
+
logger.info("プレビューを書き出しました: %s", preview_path)
|
|
35
|
+
|
|
36
|
+
if not no_open:
|
|
37
|
+
url = f"{preview_path.as_uri()}?v={cache_busting_query()}"
|
|
38
|
+
webbrowser.open(url)
|
|
39
|
+
|
|
40
|
+
return preview_path
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import struct
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
import win32clipboard
|
|
10
|
+
import win32con
|
|
11
|
+
from PIL import Image, ImageGrab
|
|
12
|
+
|
|
13
|
+
from workpytools.common.textfile import (
|
|
14
|
+
TextFileError,
|
|
15
|
+
normalize_newlines,
|
|
16
|
+
read_text_with_fallback,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
SourceKind = Literal["path", "clipboard_file", "clipboard_data", "clipboard_text"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ClipboardImageError(RuntimeError):
|
|
23
|
+
"""Raised when no usable image can be obtained from the clipboard."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ClipboardTextError(RuntimeError):
|
|
27
|
+
"""Raised when no usable text can be obtained from the clipboard."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class LoadedImage:
|
|
32
|
+
"""An image plus where it came from.
|
|
33
|
+
|
|
34
|
+
`source_kind` distinguishes the three supported input patterns:
|
|
35
|
+
- "path": an explicit file path argument
|
|
36
|
+
- "clipboard_file": clipboard holds a copied file object (e.g. Ctrl+C on
|
|
37
|
+
a file in Explorer)
|
|
38
|
+
- "clipboard_data": clipboard holds raw image data with no file behind
|
|
39
|
+
it (e.g. "Copy Image" in a viewer)
|
|
40
|
+
|
|
41
|
+
`source_path` is set for "path" and "clipboard_file" (a real file exists),
|
|
42
|
+
and is `None` for "clipboard_data".
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
image: Image.Image
|
|
46
|
+
source_path: Path | None
|
|
47
|
+
source_kind: SourceKind
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def load_image(path: str | Path | None) -> LoadedImage:
|
|
51
|
+
"""Load an image from a file path, or from the Windows clipboard if `path` is None.
|
|
52
|
+
|
|
53
|
+
Clipboard input covers two cases:
|
|
54
|
+
- raw image data (e.g. copied via "Copy Image" in a browser/viewer)
|
|
55
|
+
- a copied file object (e.g. Ctrl+C on a file in Explorer), which Pillow
|
|
56
|
+
reports as a list of file paths on Windows.
|
|
57
|
+
"""
|
|
58
|
+
if path is not None:
|
|
59
|
+
resolved = Path(path)
|
|
60
|
+
return LoadedImage(
|
|
61
|
+
image=Image.open(resolved).convert("RGBA"), source_path=resolved, source_kind="path"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
clip = ImageGrab.grabclipboard()
|
|
65
|
+
if clip is None:
|
|
66
|
+
raise ClipboardImageError("クリップボードに画像データがありません")
|
|
67
|
+
if isinstance(clip, list):
|
|
68
|
+
if not clip:
|
|
69
|
+
raise ClipboardImageError("クリップボードにファイルがありません")
|
|
70
|
+
resolved = Path(clip[0])
|
|
71
|
+
return LoadedImage(
|
|
72
|
+
image=Image.open(resolved).convert("RGBA"),
|
|
73
|
+
source_path=resolved,
|
|
74
|
+
source_kind="clipboard_file",
|
|
75
|
+
)
|
|
76
|
+
return LoadedImage(image=clip.convert("RGBA"), source_path=None, source_kind="clipboard_data")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class LoadedText:
|
|
81
|
+
"""Text plus where it came from. Mirrors `LoadedImage` for text-based commands."""
|
|
82
|
+
|
|
83
|
+
text: str
|
|
84
|
+
source_path: Path | None
|
|
85
|
+
source_kind: SourceKind
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _normalize_newlines(text: str) -> str:
|
|
89
|
+
return normalize_newlines(text)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _read_text_file(path: Path, encoding: str | None) -> str:
|
|
93
|
+
"""Read a text file, re-raising as ClipboardTextError for this module's callers."""
|
|
94
|
+
try:
|
|
95
|
+
return read_text_with_fallback(path, encoding)
|
|
96
|
+
except TextFileError as exc:
|
|
97
|
+
raise ClipboardTextError(str(exc)) from exc
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def load_text(path: str | Path | None, encoding: str | None = None) -> LoadedText:
|
|
101
|
+
"""Load text from a file path, or from the Windows clipboard if `path` is None.
|
|
102
|
+
|
|
103
|
+
Clipboard input covers two cases:
|
|
104
|
+
- plain text (e.g. copied from a text editor)
|
|
105
|
+
- a copied file object (e.g. Ctrl+C on a file in Explorer), read as text
|
|
106
|
+
|
|
107
|
+
`encoding` forces a specific codec (no fallback) when reading a file;
|
|
108
|
+
`None` tries UTF-8 (with BOM support) then falls back to CP932.
|
|
109
|
+
"""
|
|
110
|
+
if path is not None:
|
|
111
|
+
resolved = Path(path)
|
|
112
|
+
text = _read_text_file(resolved, encoding)
|
|
113
|
+
return LoadedText(
|
|
114
|
+
text=_normalize_newlines(text), source_path=resolved, source_kind="path"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
win32clipboard.OpenClipboard()
|
|
118
|
+
try:
|
|
119
|
+
has_files = win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_HDROP)
|
|
120
|
+
has_text = win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_UNICODETEXT)
|
|
121
|
+
if has_files:
|
|
122
|
+
files = win32clipboard.GetClipboardData(win32clipboard.CF_HDROP)
|
|
123
|
+
elif has_text:
|
|
124
|
+
clip_text = win32clipboard.GetClipboardData(win32clipboard.CF_UNICODETEXT)
|
|
125
|
+
else:
|
|
126
|
+
raise ClipboardTextError("クリップボードにテキストがありません")
|
|
127
|
+
finally:
|
|
128
|
+
win32clipboard.CloseClipboard()
|
|
129
|
+
|
|
130
|
+
if has_files:
|
|
131
|
+
if not files:
|
|
132
|
+
raise ClipboardTextError("クリップボードにファイルがありません")
|
|
133
|
+
resolved = Path(files[0])
|
|
134
|
+
text = _read_text_file(resolved, encoding)
|
|
135
|
+
return LoadedText(
|
|
136
|
+
text=_normalize_newlines(text), source_path=resolved, source_kind="clipboard_file"
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
return LoadedText(
|
|
140
|
+
text=_normalize_newlines(clip_text), source_path=None, source_kind="clipboard_text"
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def copy_file_to_clipboard(path: Path) -> None:
|
|
145
|
+
"""Put a file object on the clipboard (CF_HDROP), as if copied in Explorer.
|
|
146
|
+
|
|
147
|
+
Lets the user paste the resulting file with Ctrl+V into Explorer, Outlook,
|
|
148
|
+
Slack, etc.
|
|
149
|
+
"""
|
|
150
|
+
path_str = str(path.resolve())
|
|
151
|
+
# DROPFILES構造体 + ダブルNUL終端のファイルパス列(CF_HDROPの仕様)
|
|
152
|
+
dropfiles = struct.pack("Iiiii", 20, 0, 0, 0, 0)
|
|
153
|
+
data = dropfiles + path_str.encode("utf-16-le") + b"\x00\x00\x00\x00"
|
|
154
|
+
|
|
155
|
+
win32clipboard.OpenClipboard()
|
|
156
|
+
try:
|
|
157
|
+
win32clipboard.EmptyClipboard()
|
|
158
|
+
win32clipboard.SetClipboardData(win32con.CF_HDROP, data)
|
|
159
|
+
finally:
|
|
160
|
+
win32clipboard.CloseClipboard()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def copy_image_to_clipboard(image: Image.Image) -> None:
|
|
164
|
+
"""Put raw image data on the clipboard (CF_DIB), as if via "Copy Image"."""
|
|
165
|
+
with io.BytesIO() as buf:
|
|
166
|
+
image.convert("RGB").save(buf, "BMP")
|
|
167
|
+
# BMPファイルヘッダー(先頭14バイト)を除いたDIB部分のみがCF_DIBの中身
|
|
168
|
+
dib = buf.getvalue()[14:]
|
|
169
|
+
|
|
170
|
+
win32clipboard.OpenClipboard()
|
|
171
|
+
try:
|
|
172
|
+
win32clipboard.EmptyClipboard()
|
|
173
|
+
win32clipboard.SetClipboardData(win32con.CF_DIB, dib)
|
|
174
|
+
finally:
|
|
175
|
+
win32clipboard.CloseClipboard()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _cf_html_format() -> int:
|
|
179
|
+
"""The CF_HTML format ID is not a fixed constant; it must be looked up at runtime."""
|
|
180
|
+
return int(win32clipboard.RegisterClipboardFormat("HTML Format"))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def has_clipboard_html() -> bool:
|
|
184
|
+
win32clipboard.OpenClipboard()
|
|
185
|
+
try:
|
|
186
|
+
return bool(win32clipboard.IsClipboardFormatAvailable(_cf_html_format()))
|
|
187
|
+
finally:
|
|
188
|
+
win32clipboard.CloseClipboard()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def has_clipboard_text() -> bool:
|
|
192
|
+
win32clipboard.OpenClipboard()
|
|
193
|
+
try:
|
|
194
|
+
return bool(win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_UNICODETEXT))
|
|
195
|
+
finally:
|
|
196
|
+
win32clipboard.CloseClipboard()
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def get_clipboard_text() -> str:
|
|
200
|
+
"""Read CF_UNICODETEXT from the clipboard, raising if not available."""
|
|
201
|
+
win32clipboard.OpenClipboard()
|
|
202
|
+
try:
|
|
203
|
+
if not win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_UNICODETEXT):
|
|
204
|
+
raise ClipboardTextError("クリップボードにテキストがありません")
|
|
205
|
+
return str(win32clipboard.GetClipboardData(win32clipboard.CF_UNICODETEXT))
|
|
206
|
+
finally:
|
|
207
|
+
win32clipboard.CloseClipboard()
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def get_clipboard_html_fragment() -> str:
|
|
211
|
+
"""Read CF_HTML from the clipboard and return just the fragment between
|
|
212
|
+
<!--StartFragment--> and <!--EndFragment-->, falling back to the header
|
|
213
|
+
byte offsets if the comment markers are missing or the offsets are stale.
|
|
214
|
+
"""
|
|
215
|
+
win32clipboard.OpenClipboard()
|
|
216
|
+
try:
|
|
217
|
+
fmt = _cf_html_format()
|
|
218
|
+
if not win32clipboard.IsClipboardFormatAvailable(fmt):
|
|
219
|
+
raise ClipboardTextError("クリップボードにHTMLがありません")
|
|
220
|
+
raw = win32clipboard.GetClipboardData(fmt)
|
|
221
|
+
finally:
|
|
222
|
+
win32clipboard.CloseClipboard()
|
|
223
|
+
|
|
224
|
+
raw_bytes = raw if isinstance(raw, bytes) else str(raw).encode("utf-8")
|
|
225
|
+
text = raw_bytes.decode("utf-8", errors="replace")
|
|
226
|
+
|
|
227
|
+
start_marker = "<!--StartFragment-->"
|
|
228
|
+
end_marker = "<!--EndFragment-->"
|
|
229
|
+
start_idx = text.find(start_marker)
|
|
230
|
+
end_idx = text.find(end_marker)
|
|
231
|
+
if start_idx != -1 and end_idx != -1:
|
|
232
|
+
return text[start_idx + len(start_marker) : end_idx]
|
|
233
|
+
|
|
234
|
+
# コメントマーカーが見つからない場合、ヘッダーのバイトオフセットにフォールバックする
|
|
235
|
+
headers: dict[str, int] = {}
|
|
236
|
+
for line in text.split("\r\n"):
|
|
237
|
+
if not line or ":" not in line:
|
|
238
|
+
continue
|
|
239
|
+
key, _, value = line.partition(":")
|
|
240
|
+
if key in ("StartFragment", "EndFragment"):
|
|
241
|
+
try:
|
|
242
|
+
headers[key] = int(value)
|
|
243
|
+
except ValueError:
|
|
244
|
+
continue
|
|
245
|
+
if len(headers) == 2:
|
|
246
|
+
break
|
|
247
|
+
|
|
248
|
+
if "StartFragment" not in headers or "EndFragment" not in headers:
|
|
249
|
+
raise ClipboardTextError("CF_HTMLのヘッダーを解析できません")
|
|
250
|
+
|
|
251
|
+
return raw_bytes[headers["StartFragment"] : headers["EndFragment"]].decode(
|
|
252
|
+
"utf-8", errors="replace"
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def get_clipboard_html_raw() -> bytes:
|
|
257
|
+
"""Read the raw CF_HTML payload (with its header) as bytes, for marker detection
|
|
258
|
+
(e.g. Excel's ProgId marker) that needs to see the whole payload, not just the fragment.
|
|
259
|
+
"""
|
|
260
|
+
win32clipboard.OpenClipboard()
|
|
261
|
+
try:
|
|
262
|
+
fmt = _cf_html_format()
|
|
263
|
+
if not win32clipboard.IsClipboardFormatAvailable(fmt):
|
|
264
|
+
raise ClipboardTextError("クリップボードにHTMLがありません")
|
|
265
|
+
raw = win32clipboard.GetClipboardData(fmt)
|
|
266
|
+
finally:
|
|
267
|
+
win32clipboard.CloseClipboard()
|
|
268
|
+
return raw if isinstance(raw, bytes) else str(raw).encode("utf-8")
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _build_cf_html(html_fragment: str) -> bytes:
|
|
272
|
+
"""Wrap an HTML fragment in the CF_HTML header format (byte offsets, UTF-8)."""
|
|
273
|
+
header_template = (
|
|
274
|
+
"Version:0.9\r\n"
|
|
275
|
+
"StartHTML:{:010d}\r\n"
|
|
276
|
+
"EndHTML:{:010d}\r\n"
|
|
277
|
+
"StartFragment:{:010d}\r\n"
|
|
278
|
+
"EndFragment:{:010d}\r\n"
|
|
279
|
+
)
|
|
280
|
+
header_len = len(header_template.format(0, 0, 0, 0).encode("utf-8"))
|
|
281
|
+
|
|
282
|
+
prefix = "<html><body>\r\n<!--StartFragment-->"
|
|
283
|
+
suffix = "<!--EndFragment-->\r\n</body></html>"
|
|
284
|
+
|
|
285
|
+
start_html = header_len
|
|
286
|
+
start_fragment = start_html + len(prefix.encode("utf-8"))
|
|
287
|
+
end_fragment = start_fragment + len(html_fragment.encode("utf-8"))
|
|
288
|
+
end_html = end_fragment + len(suffix.encode("utf-8"))
|
|
289
|
+
|
|
290
|
+
header = header_template.format(start_html, end_html, start_fragment, end_fragment)
|
|
291
|
+
return (header + prefix + html_fragment + suffix).encode("utf-8")
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def copy_html_and_text_to_clipboard(html_fragment: str, plain_text: str) -> None:
|
|
295
|
+
"""Put both CF_HTML and CF_UNICODETEXT on the clipboard in one session.
|
|
296
|
+
|
|
297
|
+
Rich-text-aware apps (Outlook/Word/Slack) read CF_HTML for styled paste;
|
|
298
|
+
plain-text-only apps fall back to CF_UNICODETEXT.
|
|
299
|
+
"""
|
|
300
|
+
html_data = _build_cf_html(html_fragment)
|
|
301
|
+
|
|
302
|
+
win32clipboard.OpenClipboard()
|
|
303
|
+
try:
|
|
304
|
+
win32clipboard.EmptyClipboard()
|
|
305
|
+
win32clipboard.SetClipboardData(_cf_html_format(), html_data)
|
|
306
|
+
win32clipboard.SetClipboardData(win32con.CF_UNICODETEXT, plain_text)
|
|
307
|
+
finally:
|
|
308
|
+
win32clipboard.CloseClipboard()
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def copy_text_to_clipboard(text: str) -> None:
|
|
312
|
+
"""Put plain text on the clipboard (CF_UNICODETEXT only)."""
|
|
313
|
+
win32clipboard.OpenClipboard()
|
|
314
|
+
try:
|
|
315
|
+
win32clipboard.EmptyClipboard()
|
|
316
|
+
win32clipboard.SetClipboardData(win32con.CF_UNICODETEXT, text)
|
|
317
|
+
finally:
|
|
318
|
+
win32clipboard.CloseClipboard()
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from numpy.typing import NDArray
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def agglomerative_average_linkage(
|
|
8
|
+
distance_matrix: NDArray[np.float64], distance_threshold: float
|
|
9
|
+
) -> NDArray[np.int_]:
|
|
10
|
+
"""Average-linkage agglomerative clustering via the Lance-Williams update.
|
|
11
|
+
|
|
12
|
+
Equivalent to `sklearn.cluster.AgglomerativeClustering(metric="precomputed",
|
|
13
|
+
linkage="average", distance_threshold=distance_threshold, n_clusters=None)`
|
|
14
|
+
(verified against sklearn on random inputs during development).
|
|
15
|
+
|
|
16
|
+
Clusters are merged, closest pair first, until the closest remaining pair
|
|
17
|
+
exceeds `distance_threshold`. Merged-cluster distances are updated in O(1)
|
|
18
|
+
per neighbor via Lance-Williams (`d(k, i∪j) = (|i|·d(k,i) + |j|·d(k,j)) / (|i|+|j|)`)
|
|
19
|
+
rather than recomputed from scratch, keeping updates to O(n^2) overall
|
|
20
|
+
(finding the closest pair each round is still a linear scan).
|
|
21
|
+
|
|
22
|
+
Returns a 0-indexed label array, one entry per input row.
|
|
23
|
+
"""
|
|
24
|
+
n = distance_matrix.shape[0]
|
|
25
|
+
if n == 0:
|
|
26
|
+
return np.array([], dtype=int)
|
|
27
|
+
if n == 1:
|
|
28
|
+
return np.array([0], dtype=int)
|
|
29
|
+
|
|
30
|
+
sizes: dict[int, int] = {i: 1 for i in range(n)}
|
|
31
|
+
members: dict[int, list[int]] = {i: [i] for i in range(n)}
|
|
32
|
+
dist: dict[tuple[int, int], float] = {}
|
|
33
|
+
for i in range(n):
|
|
34
|
+
for j in range(i + 1, n):
|
|
35
|
+
dist[(i, j)] = float(distance_matrix[i, j])
|
|
36
|
+
|
|
37
|
+
def get_dist(a: int, b: int) -> float:
|
|
38
|
+
return dist[(a, b)] if a < b else dist[(b, a)]
|
|
39
|
+
|
|
40
|
+
def set_dist(a: int, b: int, value: float) -> None:
|
|
41
|
+
if a < b:
|
|
42
|
+
dist[(a, b)] = value
|
|
43
|
+
else:
|
|
44
|
+
dist[(b, a)] = value
|
|
45
|
+
|
|
46
|
+
active = list(range(n))
|
|
47
|
+
next_id = n
|
|
48
|
+
|
|
49
|
+
while len(active) > 1:
|
|
50
|
+
best_pair: tuple[int, int] | None = None
|
|
51
|
+
best_distance: float | None = None
|
|
52
|
+
for ai in range(len(active)):
|
|
53
|
+
for bi in range(ai + 1, len(active)):
|
|
54
|
+
a, b = active[ai], active[bi]
|
|
55
|
+
d = get_dist(a, b)
|
|
56
|
+
if best_distance is None or d < best_distance:
|
|
57
|
+
best_distance = d
|
|
58
|
+
best_pair = (a, b)
|
|
59
|
+
|
|
60
|
+
assert best_pair is not None and best_distance is not None
|
|
61
|
+
if best_distance > distance_threshold:
|
|
62
|
+
break
|
|
63
|
+
|
|
64
|
+
a, b = best_pair
|
|
65
|
+
new_id = next_id
|
|
66
|
+
next_id += 1
|
|
67
|
+
size_a, size_b = sizes[a], sizes[b]
|
|
68
|
+
new_size = size_a + size_b
|
|
69
|
+
members[new_id] = members[a] + members[b]
|
|
70
|
+
sizes[new_id] = new_size
|
|
71
|
+
|
|
72
|
+
for k in active:
|
|
73
|
+
if k in (a, b):
|
|
74
|
+
continue
|
|
75
|
+
new_d = (size_a * get_dist(k, a) + size_b * get_dist(k, b)) / new_size
|
|
76
|
+
set_dist(k, new_id, new_d)
|
|
77
|
+
|
|
78
|
+
active = [x for x in active if x not in (a, b)]
|
|
79
|
+
active.append(new_id)
|
|
80
|
+
|
|
81
|
+
labels = np.zeros(n, dtype=int)
|
|
82
|
+
for label, cluster_id in enumerate(active):
|
|
83
|
+
for member in members[cluster_id]:
|
|
84
|
+
labels[member] = label
|
|
85
|
+
return labels
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import tomllib
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def load_config(path: str | Path) -> dict[str, Any]:
|
|
10
|
+
"""Load a TOML config file into a dict."""
|
|
11
|
+
with Path(path).open("rb") as f:
|
|
12
|
+
return tomllib.load(f)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ConfigLocationError(RuntimeError):
|
|
16
|
+
"""Raised when the per-user config location can't be determined."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _appdata_dir() -> Path:
|
|
20
|
+
"""%APPDATA%\\workpytools, with a clear error instead of a bare KeyError."""
|
|
21
|
+
appdata = os.environ.get("APPDATA")
|
|
22
|
+
if not appdata:
|
|
23
|
+
raise ConfigLocationError(
|
|
24
|
+
"環境変数 APPDATA が設定されていないため、設定の保存先を決定できません"
|
|
25
|
+
)
|
|
26
|
+
return Path(appdata) / "workpytools"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def default_config_path() -> Path:
|
|
30
|
+
"""Default location of the shared config file: %APPDATA%\\workpytools\\config.toml."""
|
|
31
|
+
return _appdata_dir() / "config.toml"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def vv_prompts_dir() -> Path:
|
|
35
|
+
"""Folder holding one .txt per saved prompt: %APPDATA%\\workpytools\\vv\\."""
|
|
36
|
+
return _appdata_dir() / "vv"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load_default_config() -> dict[str, Any]:
|
|
40
|
+
"""Load the default config file, or return {} if it doesn't exist."""
|
|
41
|
+
path = default_config_path()
|
|
42
|
+
if not path.exists():
|
|
43
|
+
return {}
|
|
44
|
+
return load_config(path)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import urllib.request
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
from numpy.typing import NDArray
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
# 接続先はここに固定値として持つ(huggingface-hubは使わず urllib で直接取得する)。
|
|
14
|
+
_MODEL_BASE_URL = (
|
|
15
|
+
"https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/resolve/main"
|
|
16
|
+
)
|
|
17
|
+
_MODEL_FILES = {
|
|
18
|
+
"model.onnx": f"{_MODEL_BASE_URL}/onnx/model_quantized.onnx",
|
|
19
|
+
"tokenizer.json": f"{_MODEL_BASE_URL}/tokenizer.json",
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
DEFAULT_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ModelDownloadError(RuntimeError):
|
|
26
|
+
"""Raised when the embedding model can't be downloaded."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _model_cache_dir(model_name: str) -> Path:
|
|
30
|
+
return Path(os.environ["LOCALAPPDATA"]) / "workpytools" / "models" / model_name
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def ensure_model(model_name: str = DEFAULT_MODEL_NAME) -> Path:
|
|
34
|
+
"""Download the embedding model (ONNX + tokenizer.json) if not already cached.
|
|
35
|
+
|
|
36
|
+
Downloads from Hugging Face (`huggingface.co`) via `urllib`, not the
|
|
37
|
+
`huggingface-hub` client library, so the connection target is a plain
|
|
38
|
+
constant in this module rather than resolved by a third-party library.
|
|
39
|
+
"""
|
|
40
|
+
if model_name != DEFAULT_MODEL_NAME:
|
|
41
|
+
raise ModelDownloadError(
|
|
42
|
+
f"未対応のモデル名です: {model_name}(対応済み: {DEFAULT_MODEL_NAME})"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
cache_dir = _model_cache_dir(model_name)
|
|
46
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
|
|
48
|
+
for filename, url in _MODEL_FILES.items():
|
|
49
|
+
dest = cache_dir / filename
|
|
50
|
+
if dest.exists():
|
|
51
|
+
continue
|
|
52
|
+
logger.info("埋め込みモデルをダウンロードします: %s -> %s", url, dest)
|
|
53
|
+
try:
|
|
54
|
+
urllib.request.urlretrieve(url, dest)
|
|
55
|
+
except OSError as exc:
|
|
56
|
+
raise ModelDownloadError(
|
|
57
|
+
f"モデルのダウンロードに失敗しました: {url}"
|
|
58
|
+
) from exc
|
|
59
|
+
size_mb = dest.stat().st_size / (1024 * 1024)
|
|
60
|
+
logger.info("ダウンロード完了: %s (%.1f MB)", dest, size_mb)
|
|
61
|
+
|
|
62
|
+
return cache_dir
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def embed_sentences(
|
|
66
|
+
sentences: list[str], model_name: str = DEFAULT_MODEL_NAME
|
|
67
|
+
) -> NDArray[np.float64]:
|
|
68
|
+
"""Embed sentences with mean pooling + L2 normalization. Returns (n, dim) float64 array."""
|
|
69
|
+
import onnxruntime as ort # 遅延インポート: --similar 指定時のみ読み込む
|
|
70
|
+
from tokenizers import Tokenizer
|
|
71
|
+
|
|
72
|
+
cache_dir = ensure_model(model_name)
|
|
73
|
+
tokenizer = Tokenizer.from_file(str(cache_dir / "tokenizer.json"))
|
|
74
|
+
tokenizer.enable_padding()
|
|
75
|
+
session = ort.InferenceSession(
|
|
76
|
+
str(cache_dir / "model.onnx"), providers=["CPUExecutionProvider"]
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
encodings = tokenizer.encode_batch(sentences)
|
|
80
|
+
input_ids = np.array([e.ids for e in encodings], dtype=np.int64)
|
|
81
|
+
attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64)
|
|
82
|
+
token_type_ids = np.zeros_like(input_ids)
|
|
83
|
+
|
|
84
|
+
outputs = session.run(
|
|
85
|
+
None,
|
|
86
|
+
{
|
|
87
|
+
"input_ids": input_ids,
|
|
88
|
+
"attention_mask": attention_mask,
|
|
89
|
+
"token_type_ids": token_type_ids,
|
|
90
|
+
},
|
|
91
|
+
)
|
|
92
|
+
last_hidden = outputs[0]
|
|
93
|
+
|
|
94
|
+
mask = attention_mask[:, :, None].astype(np.float64)
|
|
95
|
+
summed = (last_hidden * mask).sum(axis=1)
|
|
96
|
+
counts = mask.sum(axis=1)
|
|
97
|
+
pooled = summed / counts
|
|
98
|
+
|
|
99
|
+
norms = np.linalg.norm(pooled, axis=1, keepdims=True)
|
|
100
|
+
normalized: NDArray[np.float64] = pooled / norms
|
|
101
|
+
return normalized
|