workpytools 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. workpytools/__init__.py +1 -0
  2. workpytools/cli.py +76 -0
  3. workpytools/common/__init__.py +0 -0
  4. workpytools/common/browser_preview.py +40 -0
  5. workpytools/common/clipboard.py +318 -0
  6. workpytools/common/clustering.py +85 -0
  7. workpytools/common/config.py +44 -0
  8. workpytools/common/embedding.py +101 -0
  9. workpytools/common/help_gen.py +542 -0
  10. workpytools/common/logging.py +12 -0
  11. workpytools/common/markdown_html.py +15 -0
  12. workpytools/common/outline_parse.py +81 -0
  13. workpytools/common/output.py +71 -0
  14. workpytools/common/powerpoint.py +43 -0
  15. workpytools/common/shape_cluster.py +75 -0
  16. workpytools/common/table_shapes.py +157 -0
  17. workpytools/common/tabular.py +248 -0
  18. workpytools/common/textfile.py +90 -0
  19. workpytools/common/walk.py +133 -0
  20. workpytools/data/__init__.py +0 -0
  21. workpytools/data/synonym.tsv +3 -0
  22. workpytools/data/user.dic +2 -0
  23. workpytools/processing/__init__.py +0 -0
  24. workpytools/processing/base.py +23 -0
  25. workpytools/processing/clipfmt.py +143 -0
  26. workpytools/processing/clipmd.py +237 -0
  27. workpytools/processing/clipview.py +194 -0
  28. workpytools/processing/cwc.py +427 -0
  29. workpytools/processing/denoise.py +81 -0
  30. workpytools/processing/help.py +47 -0
  31. workpytools/processing/ikko.py +205 -0
  32. workpytools/processing/kukiri.py +94 -0
  33. workpytools/processing/lsdir.py +218 -0
  34. workpytools/processing/mdtsv.py +177 -0
  35. workpytools/processing/mokuji.py +91 -0
  36. workpytools/processing/nagasa.py +94 -0
  37. workpytools/processing/outline.py +85 -0
  38. workpytools/processing/profiler.py +277 -0
  39. workpytools/processing/seiretsu.py +111 -0
  40. workpytools/processing/tbl.py +358 -0
  41. workpytools/processing/touka.py +52 -0
  42. workpytools/processing/vv.py +147 -0
  43. workpytools-0.1.4.dist-info/METADATA +458 -0
  44. workpytools-0.1.4.dist-info/RECORD +48 -0
  45. workpytools-0.1.4.dist-info/WHEEL +5 -0
  46. workpytools-0.1.4.dist-info/entry_points.txt +20 -0
  47. workpytools-0.1.4.dist-info/licenses/LICENSE +21 -0
  48. workpytools-0.1.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1 @@
1
+ """Collection of Python processing utilities."""
workpytools/cli.py ADDED
@@ -0,0 +1,76 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import importlib
5
+ import inspect
6
+ import pkgutil
7
+ import sys
8
+ from pathlib import PureWindowsPath
9
+
10
+ from workpytools import processing
11
+ from workpytools.common.logging import setup_logging
12
+ from workpytools.processing.base import Processor
13
+
14
+
15
+ def _discover_processors() -> dict[str, Processor]:
16
+ """Import every module under `workpytools.processing` and collect Processor subclasses."""
17
+ processors: dict[str, Processor] = {}
18
+ for module_info in pkgutil.iter_modules(processing.__path__, prefix=f"{processing.__name__}."):
19
+ if module_info.name.endswith(".base"):
20
+ continue
21
+ try:
22
+ module = importlib.import_module(module_info.name)
23
+ except ImportError as exc:
24
+ print(f"warning: skipping {module_info.name}: {exc}", file=sys.stderr)
25
+ continue
26
+ for _, obj in inspect.getmembers(module, inspect.isclass):
27
+ if issubclass(obj, Processor) and obj is not Processor:
28
+ instance = obj()
29
+ processors[instance.name] = instance
30
+ return processors
31
+
32
+
33
+ def build_parser(processors: dict[str, Processor]) -> argparse.ArgumentParser:
34
+ parser = argparse.ArgumentParser(prog="tools", description="Collection of processing commands")
35
+ parser.add_argument("--log-level", default="INFO", help="Logging level (default: INFO)")
36
+
37
+ subparsers = parser.add_subparsers(dest="command", required=True)
38
+ for proc in processors.values():
39
+ sub = subparsers.add_parser(proc.name, help=proc.help)
40
+ proc.add_arguments(sub)
41
+ return parser
42
+
43
+
44
+ def main(argv: list[str] | None = None) -> int:
45
+ processors = _discover_processors()
46
+ parser = build_parser(processors)
47
+ args = parser.parse_args(argv)
48
+
49
+ setup_logging(args.log_level)
50
+
51
+ processor = processors[args.command]
52
+ return processor.run(args)
53
+
54
+
55
+ _ENTRY_POINT_ALIASES = {
56
+ # toolh.exe だけは他の1コマンド=1exe規則に沿わない特別なエントリーポイント名
57
+ # (help.exe は分かりにくいので toolh とした)。
58
+ "toolh": "help",
59
+ }
60
+
61
+
62
+ def run_as_subcommand() -> int:
63
+ """Entry point for a per-command executable (e.g. `denoise.exe`).
64
+
65
+ The subcommand name is taken from how this executable was invoked (its
66
+ own file name), so `denoise.exe args...` behaves like `tools denoise
67
+ args...`. Register one `[project.scripts]` entry per Processor name in
68
+ pyproject.toml, all pointing to this same function.
69
+ """
70
+ command_name = PureWindowsPath(sys.argv[0]).stem
71
+ command_name = _ENTRY_POINT_ALIASES.get(command_name, command_name)
72
+ return main([command_name, *sys.argv[1:]])
73
+
74
+
75
+ if __name__ == "__main__":
76
+ sys.exit(main())
File without changes
@@ -0,0 +1,40 @@
1
+ from __future__ import annotations
2
+
3
+ import itertools
4
+ import logging
5
+ import tempfile
6
+ import time
7
+ import webbrowser
8
+ from pathlib import Path
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+ _query_counter = itertools.count()
13
+
14
+
15
+ def cache_busting_query() -> str:
16
+ """Wall-clock time plus a per-process counter, so consecutive calls
17
+ within the same clock tick still differ. Used to defeat the browser's
18
+ tendency to not reload a fixed-name file:// URL it already has open.
19
+ """
20
+ return f"{time.time_ns()}-{next(_query_counter)}"
21
+
22
+
23
+ def write_and_open(html: str, filename: str, no_open: bool) -> Path:
24
+ """Write `html` to a fixed-name file under %TEMP% and open it in the
25
+ default browser (unless `no_open`). Reused by every command that
26
+ previews a result in the browser (clipview, profiler --view, ...).
27
+ """
28
+ preview_path = Path(tempfile.gettempdir()) / filename
29
+ try:
30
+ preview_path.write_text(html, encoding="utf-8")
31
+ except OSError as exc:
32
+ raise RuntimeError(f"一時ファイルの書き込みに失敗しました: {preview_path}") from exc
33
+
34
+ logger.info("プレビューを書き出しました: %s", preview_path)
35
+
36
+ if not no_open:
37
+ url = f"{preview_path.as_uri()}?v={cache_busting_query()}"
38
+ webbrowser.open(url)
39
+
40
+ return preview_path
@@ -0,0 +1,318 @@
1
+ from __future__ import annotations
2
+
3
+ import io
4
+ import struct
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Literal
8
+
9
+ import win32clipboard
10
+ import win32con
11
+ from PIL import Image, ImageGrab
12
+
13
+ from workpytools.common.textfile import (
14
+ TextFileError,
15
+ normalize_newlines,
16
+ read_text_with_fallback,
17
+ )
18
+
19
+ SourceKind = Literal["path", "clipboard_file", "clipboard_data", "clipboard_text"]
20
+
21
+
22
+ class ClipboardImageError(RuntimeError):
23
+ """Raised when no usable image can be obtained from the clipboard."""
24
+
25
+
26
+ class ClipboardTextError(RuntimeError):
27
+ """Raised when no usable text can be obtained from the clipboard."""
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class LoadedImage:
32
+ """An image plus where it came from.
33
+
34
+ `source_kind` distinguishes the three supported input patterns:
35
+ - "path": an explicit file path argument
36
+ - "clipboard_file": clipboard holds a copied file object (e.g. Ctrl+C on
37
+ a file in Explorer)
38
+ - "clipboard_data": clipboard holds raw image data with no file behind
39
+ it (e.g. "Copy Image" in a viewer)
40
+
41
+ `source_path` is set for "path" and "clipboard_file" (a real file exists),
42
+ and is `None` for "clipboard_data".
43
+ """
44
+
45
+ image: Image.Image
46
+ source_path: Path | None
47
+ source_kind: SourceKind
48
+
49
+
50
+ def load_image(path: str | Path | None) -> LoadedImage:
51
+ """Load an image from a file path, or from the Windows clipboard if `path` is None.
52
+
53
+ Clipboard input covers two cases:
54
+ - raw image data (e.g. copied via "Copy Image" in a browser/viewer)
55
+ - a copied file object (e.g. Ctrl+C on a file in Explorer), which Pillow
56
+ reports as a list of file paths on Windows.
57
+ """
58
+ if path is not None:
59
+ resolved = Path(path)
60
+ return LoadedImage(
61
+ image=Image.open(resolved).convert("RGBA"), source_path=resolved, source_kind="path"
62
+ )
63
+
64
+ clip = ImageGrab.grabclipboard()
65
+ if clip is None:
66
+ raise ClipboardImageError("クリップボードに画像データがありません")
67
+ if isinstance(clip, list):
68
+ if not clip:
69
+ raise ClipboardImageError("クリップボードにファイルがありません")
70
+ resolved = Path(clip[0])
71
+ return LoadedImage(
72
+ image=Image.open(resolved).convert("RGBA"),
73
+ source_path=resolved,
74
+ source_kind="clipboard_file",
75
+ )
76
+ return LoadedImage(image=clip.convert("RGBA"), source_path=None, source_kind="clipboard_data")
77
+
78
+
79
+ @dataclass(frozen=True)
80
+ class LoadedText:
81
+ """Text plus where it came from. Mirrors `LoadedImage` for text-based commands."""
82
+
83
+ text: str
84
+ source_path: Path | None
85
+ source_kind: SourceKind
86
+
87
+
88
+ def _normalize_newlines(text: str) -> str:
89
+ return normalize_newlines(text)
90
+
91
+
92
+ def _read_text_file(path: Path, encoding: str | None) -> str:
93
+ """Read a text file, re-raising as ClipboardTextError for this module's callers."""
94
+ try:
95
+ return read_text_with_fallback(path, encoding)
96
+ except TextFileError as exc:
97
+ raise ClipboardTextError(str(exc)) from exc
98
+
99
+
100
+ def load_text(path: str | Path | None, encoding: str | None = None) -> LoadedText:
101
+ """Load text from a file path, or from the Windows clipboard if `path` is None.
102
+
103
+ Clipboard input covers two cases:
104
+ - plain text (e.g. copied from a text editor)
105
+ - a copied file object (e.g. Ctrl+C on a file in Explorer), read as text
106
+
107
+ `encoding` forces a specific codec (no fallback) when reading a file;
108
+ `None` tries UTF-8 (with BOM support) then falls back to CP932.
109
+ """
110
+ if path is not None:
111
+ resolved = Path(path)
112
+ text = _read_text_file(resolved, encoding)
113
+ return LoadedText(
114
+ text=_normalize_newlines(text), source_path=resolved, source_kind="path"
115
+ )
116
+
117
+ win32clipboard.OpenClipboard()
118
+ try:
119
+ has_files = win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_HDROP)
120
+ has_text = win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_UNICODETEXT)
121
+ if has_files:
122
+ files = win32clipboard.GetClipboardData(win32clipboard.CF_HDROP)
123
+ elif has_text:
124
+ clip_text = win32clipboard.GetClipboardData(win32clipboard.CF_UNICODETEXT)
125
+ else:
126
+ raise ClipboardTextError("クリップボードにテキストがありません")
127
+ finally:
128
+ win32clipboard.CloseClipboard()
129
+
130
+ if has_files:
131
+ if not files:
132
+ raise ClipboardTextError("クリップボードにファイルがありません")
133
+ resolved = Path(files[0])
134
+ text = _read_text_file(resolved, encoding)
135
+ return LoadedText(
136
+ text=_normalize_newlines(text), source_path=resolved, source_kind="clipboard_file"
137
+ )
138
+
139
+ return LoadedText(
140
+ text=_normalize_newlines(clip_text), source_path=None, source_kind="clipboard_text"
141
+ )
142
+
143
+
144
+ def copy_file_to_clipboard(path: Path) -> None:
145
+ """Put a file object on the clipboard (CF_HDROP), as if copied in Explorer.
146
+
147
+ Lets the user paste the resulting file with Ctrl+V into Explorer, Outlook,
148
+ Slack, etc.
149
+ """
150
+ path_str = str(path.resolve())
151
+ # DROPFILES構造体 + ダブルNUL終端のファイルパス列(CF_HDROPの仕様)
152
+ dropfiles = struct.pack("Iiiii", 20, 0, 0, 0, 0)
153
+ data = dropfiles + path_str.encode("utf-16-le") + b"\x00\x00\x00\x00"
154
+
155
+ win32clipboard.OpenClipboard()
156
+ try:
157
+ win32clipboard.EmptyClipboard()
158
+ win32clipboard.SetClipboardData(win32con.CF_HDROP, data)
159
+ finally:
160
+ win32clipboard.CloseClipboard()
161
+
162
+
163
+ def copy_image_to_clipboard(image: Image.Image) -> None:
164
+ """Put raw image data on the clipboard (CF_DIB), as if via "Copy Image"."""
165
+ with io.BytesIO() as buf:
166
+ image.convert("RGB").save(buf, "BMP")
167
+ # BMPファイルヘッダー(先頭14バイト)を除いたDIB部分のみがCF_DIBの中身
168
+ dib = buf.getvalue()[14:]
169
+
170
+ win32clipboard.OpenClipboard()
171
+ try:
172
+ win32clipboard.EmptyClipboard()
173
+ win32clipboard.SetClipboardData(win32con.CF_DIB, dib)
174
+ finally:
175
+ win32clipboard.CloseClipboard()
176
+
177
+
178
+ def _cf_html_format() -> int:
179
+ """The CF_HTML format ID is not a fixed constant; it must be looked up at runtime."""
180
+ return int(win32clipboard.RegisterClipboardFormat("HTML Format"))
181
+
182
+
183
+ def has_clipboard_html() -> bool:
184
+ win32clipboard.OpenClipboard()
185
+ try:
186
+ return bool(win32clipboard.IsClipboardFormatAvailable(_cf_html_format()))
187
+ finally:
188
+ win32clipboard.CloseClipboard()
189
+
190
+
191
+ def has_clipboard_text() -> bool:
192
+ win32clipboard.OpenClipboard()
193
+ try:
194
+ return bool(win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_UNICODETEXT))
195
+ finally:
196
+ win32clipboard.CloseClipboard()
197
+
198
+
199
+ def get_clipboard_text() -> str:
200
+ """Read CF_UNICODETEXT from the clipboard, raising if not available."""
201
+ win32clipboard.OpenClipboard()
202
+ try:
203
+ if not win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_UNICODETEXT):
204
+ raise ClipboardTextError("クリップボードにテキストがありません")
205
+ return str(win32clipboard.GetClipboardData(win32clipboard.CF_UNICODETEXT))
206
+ finally:
207
+ win32clipboard.CloseClipboard()
208
+
209
+
210
+ def get_clipboard_html_fragment() -> str:
211
+ """Read CF_HTML from the clipboard and return just the fragment between
212
+ <!--StartFragment--> and <!--EndFragment-->, falling back to the header
213
+ byte offsets if the comment markers are missing or the offsets are stale.
214
+ """
215
+ win32clipboard.OpenClipboard()
216
+ try:
217
+ fmt = _cf_html_format()
218
+ if not win32clipboard.IsClipboardFormatAvailable(fmt):
219
+ raise ClipboardTextError("クリップボードにHTMLがありません")
220
+ raw = win32clipboard.GetClipboardData(fmt)
221
+ finally:
222
+ win32clipboard.CloseClipboard()
223
+
224
+ raw_bytes = raw if isinstance(raw, bytes) else str(raw).encode("utf-8")
225
+ text = raw_bytes.decode("utf-8", errors="replace")
226
+
227
+ start_marker = "<!--StartFragment-->"
228
+ end_marker = "<!--EndFragment-->"
229
+ start_idx = text.find(start_marker)
230
+ end_idx = text.find(end_marker)
231
+ if start_idx != -1 and end_idx != -1:
232
+ return text[start_idx + len(start_marker) : end_idx]
233
+
234
+ # コメントマーカーが見つからない場合、ヘッダーのバイトオフセットにフォールバックする
235
+ headers: dict[str, int] = {}
236
+ for line in text.split("\r\n"):
237
+ if not line or ":" not in line:
238
+ continue
239
+ key, _, value = line.partition(":")
240
+ if key in ("StartFragment", "EndFragment"):
241
+ try:
242
+ headers[key] = int(value)
243
+ except ValueError:
244
+ continue
245
+ if len(headers) == 2:
246
+ break
247
+
248
+ if "StartFragment" not in headers or "EndFragment" not in headers:
249
+ raise ClipboardTextError("CF_HTMLのヘッダーを解析できません")
250
+
251
+ return raw_bytes[headers["StartFragment"] : headers["EndFragment"]].decode(
252
+ "utf-8", errors="replace"
253
+ )
254
+
255
+
256
+ def get_clipboard_html_raw() -> bytes:
257
+ """Read the raw CF_HTML payload (with its header) as bytes, for marker detection
258
+ (e.g. Excel's ProgId marker) that needs to see the whole payload, not just the fragment.
259
+ """
260
+ win32clipboard.OpenClipboard()
261
+ try:
262
+ fmt = _cf_html_format()
263
+ if not win32clipboard.IsClipboardFormatAvailable(fmt):
264
+ raise ClipboardTextError("クリップボードにHTMLがありません")
265
+ raw = win32clipboard.GetClipboardData(fmt)
266
+ finally:
267
+ win32clipboard.CloseClipboard()
268
+ return raw if isinstance(raw, bytes) else str(raw).encode("utf-8")
269
+
270
+
271
+ def _build_cf_html(html_fragment: str) -> bytes:
272
+ """Wrap an HTML fragment in the CF_HTML header format (byte offsets, UTF-8)."""
273
+ header_template = (
274
+ "Version:0.9\r\n"
275
+ "StartHTML:{:010d}\r\n"
276
+ "EndHTML:{:010d}\r\n"
277
+ "StartFragment:{:010d}\r\n"
278
+ "EndFragment:{:010d}\r\n"
279
+ )
280
+ header_len = len(header_template.format(0, 0, 0, 0).encode("utf-8"))
281
+
282
+ prefix = "<html><body>\r\n<!--StartFragment-->"
283
+ suffix = "<!--EndFragment-->\r\n</body></html>"
284
+
285
+ start_html = header_len
286
+ start_fragment = start_html + len(prefix.encode("utf-8"))
287
+ end_fragment = start_fragment + len(html_fragment.encode("utf-8"))
288
+ end_html = end_fragment + len(suffix.encode("utf-8"))
289
+
290
+ header = header_template.format(start_html, end_html, start_fragment, end_fragment)
291
+ return (header + prefix + html_fragment + suffix).encode("utf-8")
292
+
293
+
294
+ def copy_html_and_text_to_clipboard(html_fragment: str, plain_text: str) -> None:
295
+ """Put both CF_HTML and CF_UNICODETEXT on the clipboard in one session.
296
+
297
+ Rich-text-aware apps (Outlook/Word/Slack) read CF_HTML for styled paste;
298
+ plain-text-only apps fall back to CF_UNICODETEXT.
299
+ """
300
+ html_data = _build_cf_html(html_fragment)
301
+
302
+ win32clipboard.OpenClipboard()
303
+ try:
304
+ win32clipboard.EmptyClipboard()
305
+ win32clipboard.SetClipboardData(_cf_html_format(), html_data)
306
+ win32clipboard.SetClipboardData(win32con.CF_UNICODETEXT, plain_text)
307
+ finally:
308
+ win32clipboard.CloseClipboard()
309
+
310
+
311
+ def copy_text_to_clipboard(text: str) -> None:
312
+ """Put plain text on the clipboard (CF_UNICODETEXT only)."""
313
+ win32clipboard.OpenClipboard()
314
+ try:
315
+ win32clipboard.EmptyClipboard()
316
+ win32clipboard.SetClipboardData(win32con.CF_UNICODETEXT, text)
317
+ finally:
318
+ win32clipboard.CloseClipboard()
@@ -0,0 +1,85 @@
1
+ from __future__ import annotations
2
+
3
+ import numpy as np
4
+ from numpy.typing import NDArray
5
+
6
+
7
+ def agglomerative_average_linkage(
8
+ distance_matrix: NDArray[np.float64], distance_threshold: float
9
+ ) -> NDArray[np.int_]:
10
+ """Average-linkage agglomerative clustering via the Lance-Williams update.
11
+
12
+ Equivalent to `sklearn.cluster.AgglomerativeClustering(metric="precomputed",
13
+ linkage="average", distance_threshold=distance_threshold, n_clusters=None)`
14
+ (verified against sklearn on random inputs during development).
15
+
16
+ Clusters are merged, closest pair first, until the closest remaining pair
17
+ exceeds `distance_threshold`. Merged-cluster distances are updated in O(1)
18
+ per neighbor via Lance-Williams (`d(k, i∪j) = (|i|·d(k,i) + |j|·d(k,j)) / (|i|+|j|)`)
19
+ rather than recomputed from scratch, keeping updates to O(n^2) overall
20
+ (finding the closest pair each round is still a linear scan).
21
+
22
+ Returns a 0-indexed label array, one entry per input row.
23
+ """
24
+ n = distance_matrix.shape[0]
25
+ if n == 0:
26
+ return np.array([], dtype=int)
27
+ if n == 1:
28
+ return np.array([0], dtype=int)
29
+
30
+ sizes: dict[int, int] = {i: 1 for i in range(n)}
31
+ members: dict[int, list[int]] = {i: [i] for i in range(n)}
32
+ dist: dict[tuple[int, int], float] = {}
33
+ for i in range(n):
34
+ for j in range(i + 1, n):
35
+ dist[(i, j)] = float(distance_matrix[i, j])
36
+
37
+ def get_dist(a: int, b: int) -> float:
38
+ return dist[(a, b)] if a < b else dist[(b, a)]
39
+
40
+ def set_dist(a: int, b: int, value: float) -> None:
41
+ if a < b:
42
+ dist[(a, b)] = value
43
+ else:
44
+ dist[(b, a)] = value
45
+
46
+ active = list(range(n))
47
+ next_id = n
48
+
49
+ while len(active) > 1:
50
+ best_pair: tuple[int, int] | None = None
51
+ best_distance: float | None = None
52
+ for ai in range(len(active)):
53
+ for bi in range(ai + 1, len(active)):
54
+ a, b = active[ai], active[bi]
55
+ d = get_dist(a, b)
56
+ if best_distance is None or d < best_distance:
57
+ best_distance = d
58
+ best_pair = (a, b)
59
+
60
+ assert best_pair is not None and best_distance is not None
61
+ if best_distance > distance_threshold:
62
+ break
63
+
64
+ a, b = best_pair
65
+ new_id = next_id
66
+ next_id += 1
67
+ size_a, size_b = sizes[a], sizes[b]
68
+ new_size = size_a + size_b
69
+ members[new_id] = members[a] + members[b]
70
+ sizes[new_id] = new_size
71
+
72
+ for k in active:
73
+ if k in (a, b):
74
+ continue
75
+ new_d = (size_a * get_dist(k, a) + size_b * get_dist(k, b)) / new_size
76
+ set_dist(k, new_id, new_d)
77
+
78
+ active = [x for x in active if x not in (a, b)]
79
+ active.append(new_id)
80
+
81
+ labels = np.zeros(n, dtype=int)
82
+ for label, cluster_id in enumerate(active):
83
+ for member in members[cluster_id]:
84
+ labels[member] = label
85
+ return labels
@@ -0,0 +1,44 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import tomllib
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+
9
+ def load_config(path: str | Path) -> dict[str, Any]:
10
+ """Load a TOML config file into a dict."""
11
+ with Path(path).open("rb") as f:
12
+ return tomllib.load(f)
13
+
14
+
15
+ class ConfigLocationError(RuntimeError):
16
+ """Raised when the per-user config location can't be determined."""
17
+
18
+
19
+ def _appdata_dir() -> Path:
20
+ """%APPDATA%\\workpytools, with a clear error instead of a bare KeyError."""
21
+ appdata = os.environ.get("APPDATA")
22
+ if not appdata:
23
+ raise ConfigLocationError(
24
+ "環境変数 APPDATA が設定されていないため、設定の保存先を決定できません"
25
+ )
26
+ return Path(appdata) / "workpytools"
27
+
28
+
29
+ def default_config_path() -> Path:
30
+ """Default location of the shared config file: %APPDATA%\\workpytools\\config.toml."""
31
+ return _appdata_dir() / "config.toml"
32
+
33
+
34
+ def vv_prompts_dir() -> Path:
35
+ """Folder holding one .txt per saved prompt: %APPDATA%\\workpytools\\vv\\."""
36
+ return _appdata_dir() / "vv"
37
+
38
+
39
+ def load_default_config() -> dict[str, Any]:
40
+ """Load the default config file, or return {} if it doesn't exist."""
41
+ path = default_config_path()
42
+ if not path.exists():
43
+ return {}
44
+ return load_config(path)
@@ -0,0 +1,101 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import os
5
+ import urllib.request
6
+ from pathlib import Path
7
+
8
+ import numpy as np
9
+ from numpy.typing import NDArray
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+ # 接続先はここに固定値として持つ(huggingface-hubは使わず urllib で直接取得する)。
14
+ _MODEL_BASE_URL = (
15
+ "https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/resolve/main"
16
+ )
17
+ _MODEL_FILES = {
18
+ "model.onnx": f"{_MODEL_BASE_URL}/onnx/model_quantized.onnx",
19
+ "tokenizer.json": f"{_MODEL_BASE_URL}/tokenizer.json",
20
+ }
21
+
22
+ DEFAULT_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
23
+
24
+
25
+ class ModelDownloadError(RuntimeError):
26
+ """Raised when the embedding model can't be downloaded."""
27
+
28
+
29
+ def _model_cache_dir(model_name: str) -> Path:
30
+ return Path(os.environ["LOCALAPPDATA"]) / "workpytools" / "models" / model_name
31
+
32
+
33
+ def ensure_model(model_name: str = DEFAULT_MODEL_NAME) -> Path:
34
+ """Download the embedding model (ONNX + tokenizer.json) if not already cached.
35
+
36
+ Downloads from Hugging Face (`huggingface.co`) via `urllib`, not the
37
+ `huggingface-hub` client library, so the connection target is a plain
38
+ constant in this module rather than resolved by a third-party library.
39
+ """
40
+ if model_name != DEFAULT_MODEL_NAME:
41
+ raise ModelDownloadError(
42
+ f"未対応のモデル名です: {model_name}(対応済み: {DEFAULT_MODEL_NAME})"
43
+ )
44
+
45
+ cache_dir = _model_cache_dir(model_name)
46
+ cache_dir.mkdir(parents=True, exist_ok=True)
47
+
48
+ for filename, url in _MODEL_FILES.items():
49
+ dest = cache_dir / filename
50
+ if dest.exists():
51
+ continue
52
+ logger.info("埋め込みモデルをダウンロードします: %s -> %s", url, dest)
53
+ try:
54
+ urllib.request.urlretrieve(url, dest)
55
+ except OSError as exc:
56
+ raise ModelDownloadError(
57
+ f"モデルのダウンロードに失敗しました: {url}"
58
+ ) from exc
59
+ size_mb = dest.stat().st_size / (1024 * 1024)
60
+ logger.info("ダウンロード完了: %s (%.1f MB)", dest, size_mb)
61
+
62
+ return cache_dir
63
+
64
+
65
+ def embed_sentences(
66
+ sentences: list[str], model_name: str = DEFAULT_MODEL_NAME
67
+ ) -> NDArray[np.float64]:
68
+ """Embed sentences with mean pooling + L2 normalization. Returns (n, dim) float64 array."""
69
+ import onnxruntime as ort # 遅延インポート: --similar 指定時のみ読み込む
70
+ from tokenizers import Tokenizer
71
+
72
+ cache_dir = ensure_model(model_name)
73
+ tokenizer = Tokenizer.from_file(str(cache_dir / "tokenizer.json"))
74
+ tokenizer.enable_padding()
75
+ session = ort.InferenceSession(
76
+ str(cache_dir / "model.onnx"), providers=["CPUExecutionProvider"]
77
+ )
78
+
79
+ encodings = tokenizer.encode_batch(sentences)
80
+ input_ids = np.array([e.ids for e in encodings], dtype=np.int64)
81
+ attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64)
82
+ token_type_ids = np.zeros_like(input_ids)
83
+
84
+ outputs = session.run(
85
+ None,
86
+ {
87
+ "input_ids": input_ids,
88
+ "attention_mask": attention_mask,
89
+ "token_type_ids": token_type_ids,
90
+ },
91
+ )
92
+ last_hidden = outputs[0]
93
+
94
+ mask = attention_mask[:, :, None].astype(np.float64)
95
+ summed = (last_hidden * mask).sum(axis=1)
96
+ counts = mask.sum(axis=1)
97
+ pooled = summed / counts
98
+
99
+ norms = np.linalg.norm(pooled, axis=1, keepdims=True)
100
+ normalized: NDArray[np.float64] = pooled / norms
101
+ return normalized