workpytools 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- workpytools/__init__.py +1 -0
- workpytools/cli.py +76 -0
- workpytools/common/__init__.py +0 -0
- workpytools/common/browser_preview.py +40 -0
- workpytools/common/clipboard.py +318 -0
- workpytools/common/clustering.py +85 -0
- workpytools/common/config.py +44 -0
- workpytools/common/embedding.py +101 -0
- workpytools/common/help_gen.py +542 -0
- workpytools/common/logging.py +12 -0
- workpytools/common/markdown_html.py +15 -0
- workpytools/common/outline_parse.py +81 -0
- workpytools/common/output.py +71 -0
- workpytools/common/powerpoint.py +43 -0
- workpytools/common/shape_cluster.py +75 -0
- workpytools/common/table_shapes.py +157 -0
- workpytools/common/tabular.py +248 -0
- workpytools/common/textfile.py +90 -0
- workpytools/common/walk.py +133 -0
- workpytools/data/__init__.py +0 -0
- workpytools/data/synonym.tsv +3 -0
- workpytools/data/user.dic +2 -0
- workpytools/processing/__init__.py +0 -0
- workpytools/processing/base.py +23 -0
- workpytools/processing/clipfmt.py +143 -0
- workpytools/processing/clipmd.py +237 -0
- workpytools/processing/clipview.py +194 -0
- workpytools/processing/cwc.py +427 -0
- workpytools/processing/denoise.py +81 -0
- workpytools/processing/help.py +47 -0
- workpytools/processing/ikko.py +205 -0
- workpytools/processing/kukiri.py +94 -0
- workpytools/processing/lsdir.py +218 -0
- workpytools/processing/mdtsv.py +177 -0
- workpytools/processing/mokuji.py +91 -0
- workpytools/processing/nagasa.py +94 -0
- workpytools/processing/outline.py +85 -0
- workpytools/processing/profiler.py +277 -0
- workpytools/processing/seiretsu.py +111 -0
- workpytools/processing/tbl.py +358 -0
- workpytools/processing/touka.py +52 -0
- workpytools/processing/vv.py +147 -0
- workpytools-0.1.4.dist-info/METADATA +458 -0
- workpytools-0.1.4.dist-info/RECORD +48 -0
- workpytools-0.1.4.dist-info/WHEEL +5 -0
- workpytools-0.1.4.dist-info/entry_points.txt +20 -0
- workpytools-0.1.4.dist-info/licenses/LICENSE +21 -0
- workpytools-0.1.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
from workpytools.common.browser_preview import write_and_open
|
|
7
|
+
from workpytools.common.clipboard import (
|
|
8
|
+
get_clipboard_html_fragment,
|
|
9
|
+
get_clipboard_text,
|
|
10
|
+
has_clipboard_html,
|
|
11
|
+
has_clipboard_text,
|
|
12
|
+
)
|
|
13
|
+
from workpytools.common.markdown_html import markdown_to_html_fragment
|
|
14
|
+
from workpytools.processing.base import Processor
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
_PREVIEW_FILENAME = "workpytools_clipview_preview.html"
|
|
19
|
+
_SVG_PREVIEW_FILENAME = "workpytools_clipview_preview.svg"
|
|
20
|
+
|
|
21
|
+
# markdown-it-pyが```mermaidフェンスをレンダリングすると
|
|
22
|
+
# <pre><code class="language-mermaid">...</code></pre> になる。
|
|
23
|
+
# この文字列の有無だけを見て、Mermaidブロックの存在を検出する
|
|
24
|
+
# (存在しない通常のMarkdown/HTMLプレビューでは外部通信を一切発生させないため)。
|
|
25
|
+
_MERMAID_CLASS_MARKER = 'class="language-mermaid"'
|
|
26
|
+
|
|
27
|
+
_MERMAID_CDN_URL = "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js"
|
|
28
|
+
|
|
29
|
+
_MERMAID_SCRIPT = f"""\
|
|
30
|
+
<script src="{_MERMAID_CDN_URL}"></script>
|
|
31
|
+
<script>
|
|
32
|
+
document.querySelectorAll("pre code.language-mermaid").forEach((el) => {{
|
|
33
|
+
const pre = el.parentElement;
|
|
34
|
+
const div = document.createElement("div");
|
|
35
|
+
div.className = "mermaid";
|
|
36
|
+
div.textContent = el.textContent;
|
|
37
|
+
pre.replaceWith(div);
|
|
38
|
+
}});
|
|
39
|
+
mermaid.initialize({{ startOnLoad: true }});
|
|
40
|
+
</script>
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
_STYLE = """\
|
|
44
|
+
:root { color-scheme: light dark; }
|
|
45
|
+
body {
|
|
46
|
+
max-width: 46rem;
|
|
47
|
+
margin: 2rem auto;
|
|
48
|
+
padding: 0 1rem;
|
|
49
|
+
font-family: -apple-system, "Segoe UI", sans-serif;
|
|
50
|
+
line-height: 1.6;
|
|
51
|
+
color: #222;
|
|
52
|
+
background: #fff;
|
|
53
|
+
}
|
|
54
|
+
table { border-collapse: collapse; margin: 1rem 0; }
|
|
55
|
+
th, td { border: 1px solid #999; padding: 0.4rem 0.8rem; }
|
|
56
|
+
pre {
|
|
57
|
+
background: #f5f5f5;
|
|
58
|
+
padding: 0.8rem;
|
|
59
|
+
overflow-x: auto;
|
|
60
|
+
font-family: Consolas, "Courier New", monospace;
|
|
61
|
+
}
|
|
62
|
+
code { font-family: Consolas, "Courier New", monospace; }
|
|
63
|
+
blockquote {
|
|
64
|
+
border-left: 4px solid #ccc;
|
|
65
|
+
margin-left: 0;
|
|
66
|
+
padding-left: 1rem;
|
|
67
|
+
color: #555;
|
|
68
|
+
}
|
|
69
|
+
@media (prefers-color-scheme: dark) {
|
|
70
|
+
body { color: #ddd; background: #1e1e1e; }
|
|
71
|
+
th, td { border-color: #555; }
|
|
72
|
+
pre { background: #2a2a2a; }
|
|
73
|
+
blockquote { border-left-color: #555; color: #aaa; }
|
|
74
|
+
}
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def wrap_preview_html(body_fragment: str) -> str:
|
|
79
|
+
mermaid_script = _MERMAID_SCRIPT if _MERMAID_CLASS_MARKER in body_fragment else ""
|
|
80
|
+
return f"""<!doctype html>
|
|
81
|
+
<html>
|
|
82
|
+
<head>
|
|
83
|
+
<meta charset="utf-8">
|
|
84
|
+
<meta http-equiv="Cache-Control" content="no-store">
|
|
85
|
+
<title>clipview preview</title>
|
|
86
|
+
<style>
|
|
87
|
+
{_STYLE}
|
|
88
|
+
</style>
|
|
89
|
+
</head>
|
|
90
|
+
<body>
|
|
91
|
+
{body_fragment}
|
|
92
|
+
{mermaid_script}</body>
|
|
93
|
+
</html>
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class ClipviewProcessor(Processor):
|
|
98
|
+
"""Preview clipboard Markdown or HTML in the default browser.
|
|
99
|
+
|
|
100
|
+
Unlike the other clip* commands, this one never modifies the clipboard;
|
|
101
|
+
it only reads it and writes a temporary preview file.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
name = "clipview"
|
|
105
|
+
help = "クリップボードのMarkdown/HTMLをブラウザでプレビューする"
|
|
106
|
+
|
|
107
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
108
|
+
group = parser.add_mutually_exclusive_group()
|
|
109
|
+
group.add_argument(
|
|
110
|
+
"--markdown", action="store_true", help="入力をMarkdownとして解釈する"
|
|
111
|
+
)
|
|
112
|
+
group.add_argument("--html", action="store_true", help="入力をHTMLとして解釈する")
|
|
113
|
+
group.add_argument("--svg", action="store_true", help="入力をSVGとして解釈する")
|
|
114
|
+
parser.add_argument(
|
|
115
|
+
"--no-open", action="store_true", help="ブラウザを開かず、パスの表示のみ行う"
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def run(self, args: argparse.Namespace) -> int:
|
|
119
|
+
svg_text = self._resolve_svg(args)
|
|
120
|
+
if svg_text is not None:
|
|
121
|
+
if not svg_text.strip():
|
|
122
|
+
raise SystemExit("変換結果が空です")
|
|
123
|
+
try:
|
|
124
|
+
preview_path = write_and_open(svg_text, _SVG_PREVIEW_FILENAME, args.no_open)
|
|
125
|
+
except RuntimeError as exc:
|
|
126
|
+
raise SystemExit(str(exc)) from exc
|
|
127
|
+
print(preview_path)
|
|
128
|
+
return 0
|
|
129
|
+
|
|
130
|
+
body_fragment = self._resolve_body_fragment(args)
|
|
131
|
+
|
|
132
|
+
if not body_fragment.strip():
|
|
133
|
+
raise SystemExit("変換結果が空です")
|
|
134
|
+
|
|
135
|
+
if _MERMAID_CLASS_MARKER in body_fragment:
|
|
136
|
+
logger.info(
|
|
137
|
+
"Mermaidブロックを検出したため、CDN(%s)からmermaid.jsを読み込みます",
|
|
138
|
+
_MERMAID_CDN_URL,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
html = wrap_preview_html(body_fragment)
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
preview_path = write_and_open(html, _PREVIEW_FILENAME, args.no_open)
|
|
145
|
+
except RuntimeError as exc:
|
|
146
|
+
raise SystemExit(str(exc)) from exc
|
|
147
|
+
|
|
148
|
+
print(preview_path)
|
|
149
|
+
return 0
|
|
150
|
+
|
|
151
|
+
def _resolve_svg(self, args: argparse.Namespace) -> str | None:
|
|
152
|
+
has_text = has_clipboard_text()
|
|
153
|
+
|
|
154
|
+
if args.svg:
|
|
155
|
+
if not has_text:
|
|
156
|
+
raise SystemExit("クリップボードにテキストがありません")
|
|
157
|
+
logger.info("--svg指定: SVGとして扱います")
|
|
158
|
+
return get_clipboard_text()
|
|
159
|
+
|
|
160
|
+
if args.markdown or args.html:
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
if has_text:
|
|
164
|
+
text = get_clipboard_text()
|
|
165
|
+
if text.lstrip().startswith("<svg"):
|
|
166
|
+
logger.info("SVGコードを検出したため、そのままファイルにしてプレビューします")
|
|
167
|
+
return text
|
|
168
|
+
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
def _resolve_body_fragment(self, args: argparse.Namespace) -> str:
|
|
172
|
+
has_html = has_clipboard_html()
|
|
173
|
+
has_text = has_clipboard_text()
|
|
174
|
+
|
|
175
|
+
if args.markdown:
|
|
176
|
+
if not has_text:
|
|
177
|
+
raise SystemExit("クリップボードにテキストがありません")
|
|
178
|
+
logger.info("--markdown指定: Markdownとして変換します")
|
|
179
|
+
return markdown_to_html_fragment(get_clipboard_text())
|
|
180
|
+
|
|
181
|
+
if args.html:
|
|
182
|
+
if not has_html and not has_text:
|
|
183
|
+
raise SystemExit("クリップボードにHTMLがありません")
|
|
184
|
+
logger.info("--html指定: HTMLとして解釈します")
|
|
185
|
+
return get_clipboard_html_fragment() if has_html else get_clipboard_text()
|
|
186
|
+
|
|
187
|
+
if has_html:
|
|
188
|
+
logger.info("CF_HTMLを検出したためHTMLとしてプレビューします")
|
|
189
|
+
return get_clipboard_html_fragment()
|
|
190
|
+
if has_text:
|
|
191
|
+
logger.info("CF_UNICODETEXTのみのためMarkdownとして変換します")
|
|
192
|
+
return markdown_to_html_fragment(get_clipboard_text())
|
|
193
|
+
|
|
194
|
+
raise SystemExit("クリップボードにプレビューできる内容がありません")
|
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
from importlib import resources
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from wordcloud import STOPWORDS, WordCloud
|
|
11
|
+
|
|
12
|
+
from workpytools.common.clipboard import load_text
|
|
13
|
+
from workpytools.common.clustering import agglomerative_average_linkage
|
|
14
|
+
from workpytools.common.config import load_default_config
|
|
15
|
+
from workpytools.common.embedding import DEFAULT_MODEL_NAME, embed_sentences
|
|
16
|
+
from workpytools.common.output import describe_output, save_result
|
|
17
|
+
from workpytools.processing.base import Processor
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
_DEFAULT_FONT = Path(r"C:\Windows\Fonts\meiryo.ttc")
|
|
22
|
+
|
|
23
|
+
# 文字クラス内では ] と - をエスケープする(] は先頭以外だとクラスの終端と解釈される)。
|
|
24
|
+
_SPLIT_CHARS = "。()「」『』【】〔〕[\\]{}〈〉《》()\\[\\]{}\\s"
|
|
25
|
+
_SPLIT_PATTERN = re.compile(f"[{_SPLIT_CHARS}]+")
|
|
26
|
+
|
|
27
|
+
# --similar 用の文分割。正規化後は全角!/?が半角になるため半角で指定する。
|
|
28
|
+
_SENTENCE_SPLIT_PATTERN = re.compile(r"[。!?\s]+")
|
|
29
|
+
|
|
30
|
+
# --similar の入力文数の上限。凝集型クラスタリングはO(n^2)の距離行列を使うため無制限にはしない。
|
|
31
|
+
_SIMILAR_MAX_SENTENCES = 2000
|
|
32
|
+
|
|
33
|
+
_HINSHI_CHOICES = (
|
|
34
|
+
"名詞",
|
|
35
|
+
"動詞",
|
|
36
|
+
"形容詞",
|
|
37
|
+
"副詞",
|
|
38
|
+
"助詞",
|
|
39
|
+
"助動詞",
|
|
40
|
+
"連体詞",
|
|
41
|
+
"接続詞",
|
|
42
|
+
"感動詞",
|
|
43
|
+
"接頭詞",
|
|
44
|
+
"記号",
|
|
45
|
+
"フィラー",
|
|
46
|
+
"その他",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# 送り仮名・漢字かな交ぜ書きのゆれをNFKC正規化の後に統一する変換表。
|
|
50
|
+
_KANA_KANJI_REWRITES = {
|
|
51
|
+
"無し": "なし",
|
|
52
|
+
"有る": "ある",
|
|
53
|
+
"無い": "ない",
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
_DEFAULT_STOPWORDS_JA = frozenset(
|
|
57
|
+
{
|
|
58
|
+
# 形式名詞・指示語
|
|
59
|
+
"こと",
|
|
60
|
+
"もの",
|
|
61
|
+
"これ",
|
|
62
|
+
"それ",
|
|
63
|
+
"あれ",
|
|
64
|
+
"ため",
|
|
65
|
+
"よう",
|
|
66
|
+
"とき",
|
|
67
|
+
# 汎用動詞・補助的な語
|
|
68
|
+
"する",
|
|
69
|
+
"ある",
|
|
70
|
+
"いる",
|
|
71
|
+
"なる",
|
|
72
|
+
"できる",
|
|
73
|
+
# 助詞・助動詞
|
|
74
|
+
"の",
|
|
75
|
+
"に",
|
|
76
|
+
"は",
|
|
77
|
+
"を",
|
|
78
|
+
"が",
|
|
79
|
+
"で",
|
|
80
|
+
"と",
|
|
81
|
+
"も",
|
|
82
|
+
"や",
|
|
83
|
+
"から",
|
|
84
|
+
"まで",
|
|
85
|
+
"です",
|
|
86
|
+
"ます",
|
|
87
|
+
}
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _normalize(text: str) -> str:
|
|
92
|
+
normalized = unicodedata.normalize("NFKC", text)
|
|
93
|
+
for old, new in _KANA_KANJI_REWRITES.items():
|
|
94
|
+
normalized = normalized.replace(old, new)
|
|
95
|
+
return normalized
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _split_default(text: str) -> list[str]:
|
|
99
|
+
return [token for token in _SPLIT_PATTERN.split(text) if token]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _split_wakachi(text: str, user_dict: Path | None) -> list[tuple[str, str, str]]:
|
|
103
|
+
"""Tokenize with Janome. Returns (surface, base_form, hinshi) triples."""
|
|
104
|
+
from janome.tokenizer import Tokenizer # 遅延インポート: -w/--hinshi指定時のみ読み込む
|
|
105
|
+
|
|
106
|
+
if user_dict is not None:
|
|
107
|
+
tokenizer = Tokenizer(udic=str(user_dict), udic_type="simpledic")
|
|
108
|
+
else:
|
|
109
|
+
tokenizer = Tokenizer()
|
|
110
|
+
|
|
111
|
+
result = []
|
|
112
|
+
for token in tokenizer.tokenize(text):
|
|
113
|
+
hinshi = token.part_of_speech.split(",")[0]
|
|
114
|
+
result.append((token.surface, token.base_form, hinshi))
|
|
115
|
+
return result
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _resolve_user_dict(args: argparse.Namespace) -> Path | None:
|
|
119
|
+
if getattr(args, "no_user_dict", False):
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
candidate: Path | None = None
|
|
123
|
+
with resources.as_file(resources.files("workpytools.data") / "user.dic") as bundled:
|
|
124
|
+
if bundled.exists():
|
|
125
|
+
candidate = bundled
|
|
126
|
+
|
|
127
|
+
config = load_default_config()
|
|
128
|
+
configured = config.get("cwc", {}).get("user_dict")
|
|
129
|
+
if configured is not None:
|
|
130
|
+
candidate = Path(configured)
|
|
131
|
+
|
|
132
|
+
if args.user_dict is not None:
|
|
133
|
+
candidate = Path(args.user_dict)
|
|
134
|
+
|
|
135
|
+
if candidate is not None and not candidate.exists():
|
|
136
|
+
logger.warning("ユーザー辞書が見つからないためスキップします: %s", candidate)
|
|
137
|
+
return None
|
|
138
|
+
return candidate
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _resolve_synonym_dict(args: argparse.Namespace) -> Path | None:
|
|
142
|
+
if getattr(args, "no_synonym_dict", False):
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
candidate: Path | None = None
|
|
146
|
+
with resources.as_file(resources.files("workpytools.data") / "synonym.tsv") as bundled:
|
|
147
|
+
if bundled.exists():
|
|
148
|
+
candidate = bundled
|
|
149
|
+
|
|
150
|
+
config = load_default_config()
|
|
151
|
+
configured = config.get("cwc", {}).get("synonym_dict")
|
|
152
|
+
if configured is not None:
|
|
153
|
+
candidate = Path(configured)
|
|
154
|
+
|
|
155
|
+
if args.synonym_dict is not None:
|
|
156
|
+
candidate = Path(args.synonym_dict)
|
|
157
|
+
|
|
158
|
+
if candidate is not None and not candidate.exists():
|
|
159
|
+
logger.warning("同義語辞書が見つからないためスキップします: %s", candidate)
|
|
160
|
+
return None
|
|
161
|
+
return candidate
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _load_synonym_map(path: Path) -> dict[str, str]:
|
|
165
|
+
mapping: dict[str, str] = {}
|
|
166
|
+
with path.open("r", encoding="utf-8") as f:
|
|
167
|
+
for line in f:
|
|
168
|
+
line = line.rstrip("\n")
|
|
169
|
+
if not line or line.startswith("#"):
|
|
170
|
+
continue
|
|
171
|
+
columns = line.split("\t")
|
|
172
|
+
representative, *variants = columns
|
|
173
|
+
for variant in variants:
|
|
174
|
+
mapping[variant] = representative
|
|
175
|
+
return mapping
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _resolve_stopwords(args: argparse.Namespace) -> frozenset[str]:
|
|
179
|
+
if args.no_default_stopwords:
|
|
180
|
+
words: set[str] = set()
|
|
181
|
+
else:
|
|
182
|
+
words = set(_DEFAULT_STOPWORDS_JA) | set(STOPWORDS)
|
|
183
|
+
|
|
184
|
+
if args.stopwords:
|
|
185
|
+
for chunk in args.stopwords:
|
|
186
|
+
words.update(w.strip() for w in chunk.split(",") if w.strip())
|
|
187
|
+
|
|
188
|
+
if args.stopwords_file:
|
|
189
|
+
path = Path(args.stopwords_file)
|
|
190
|
+
with path.open("r", encoding="utf-8") as f:
|
|
191
|
+
words.update(line.strip() for line in f if line.strip())
|
|
192
|
+
|
|
193
|
+
return frozenset(words)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class CwcProcessor(Processor):
|
|
197
|
+
"""Generate a word cloud image from input text.
|
|
198
|
+
|
|
199
|
+
Input source is auto-detected (file path or clipboard), following the
|
|
200
|
+
same convention as touka/denoise/kukiri. Word splitting defaults to
|
|
201
|
+
delimiter-based splitting; `-w` switches to Janome tokenization.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
name = "cwc"
|
|
205
|
+
help = "テキストからワードクラウド画像を生成する(ファイルパス/クリップボード入力対応)"
|
|
206
|
+
|
|
207
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
208
|
+
parser.add_argument(
|
|
209
|
+
"path",
|
|
210
|
+
nargs="?",
|
|
211
|
+
default=None,
|
|
212
|
+
help="テキストファイルパス。省略時はクリップボードのテキスト/コピーしたファイルを使用",
|
|
213
|
+
)
|
|
214
|
+
parser.add_argument(
|
|
215
|
+
"-o", "--output", default=None, help="出力先パス(省略時は自動生成、拡張子はpng)"
|
|
216
|
+
)
|
|
217
|
+
parser.add_argument(
|
|
218
|
+
"--encoding",
|
|
219
|
+
default=None,
|
|
220
|
+
help="入力ファイルのエンコーディングを明示指定(省略時はUTF-8→CP932の順に試す)",
|
|
221
|
+
)
|
|
222
|
+
parser.add_argument(
|
|
223
|
+
"-w", "--wakachi", action="store_true", help="Janomeによる分かち書きで分割する"
|
|
224
|
+
)
|
|
225
|
+
parser.add_argument(
|
|
226
|
+
"--hinshi",
|
|
227
|
+
nargs="+",
|
|
228
|
+
default=None,
|
|
229
|
+
help=(
|
|
230
|
+
"集計対象の品詞を指定(複数可、カンマ区切りも可)。"
|
|
231
|
+
f"指定可能: {', '.join(_HINSHI_CHOICES)}"
|
|
232
|
+
),
|
|
233
|
+
)
|
|
234
|
+
parser.add_argument(
|
|
235
|
+
"--semantic", action="store_true", help="同義語辞書で異表記を代表語に寄せる"
|
|
236
|
+
)
|
|
237
|
+
parser.add_argument("--synonym-dict", default=None, help="同義語辞書ファイルのパス")
|
|
238
|
+
parser.add_argument(
|
|
239
|
+
"--no-synonym-dict", action="store_true", help="同義語辞書を一切使用しない"
|
|
240
|
+
)
|
|
241
|
+
parser.add_argument("--user-dict", default=None, help="Janomeユーザー辞書のパス")
|
|
242
|
+
parser.add_argument(
|
|
243
|
+
"--no-user-dict", action="store_true", help="Janomeユーザー辞書を一切使用しない"
|
|
244
|
+
)
|
|
245
|
+
parser.add_argument(
|
|
246
|
+
"--stopwords",
|
|
247
|
+
action="append",
|
|
248
|
+
default=None,
|
|
249
|
+
help="追加するストップワード(カンマ区切り)",
|
|
250
|
+
)
|
|
251
|
+
parser.add_argument(
|
|
252
|
+
"--stopwords-file", default=None, help="追加するストップワードのファイルパス"
|
|
253
|
+
)
|
|
254
|
+
parser.add_argument(
|
|
255
|
+
"--no-default-stopwords",
|
|
256
|
+
action="store_true",
|
|
257
|
+
help="デフォルトのストップワードを無効化する",
|
|
258
|
+
)
|
|
259
|
+
parser.add_argument("--font", default=None, help="使用するフォントファイルのパス")
|
|
260
|
+
parser.add_argument(
|
|
261
|
+
"--similar",
|
|
262
|
+
action="store_true",
|
|
263
|
+
help="集計単位を語ではなく文にし、埋め込みベクトルの類似度でクラスタリングする",
|
|
264
|
+
)
|
|
265
|
+
parser.add_argument(
|
|
266
|
+
"--similar-threshold",
|
|
267
|
+
type=float,
|
|
268
|
+
default=0.2,
|
|
269
|
+
help="--similar のクラスタをまとめる距離の閾値(コサイン距離、既定: 0.2)",
|
|
270
|
+
)
|
|
271
|
+
parser.add_argument(
|
|
272
|
+
"--similar-model",
|
|
273
|
+
default=DEFAULT_MODEL_NAME,
|
|
274
|
+
help="--similar で使用する埋め込みモデル名",
|
|
275
|
+
)
|
|
276
|
+
parser.add_argument(
|
|
277
|
+
"--similar-max-length",
|
|
278
|
+
type=int,
|
|
279
|
+
default=10,
|
|
280
|
+
help="--similar の代表文の表示文字数の上限(0で無制限、既定: 10)",
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
def run(self, args: argparse.Namespace) -> int:
|
|
284
|
+
self._validate_exclusive_options(args)
|
|
285
|
+
|
|
286
|
+
loaded = load_text(args.path, encoding=args.encoding)
|
|
287
|
+
text = _normalize(loaded.text)
|
|
288
|
+
|
|
289
|
+
if args.similar:
|
|
290
|
+
frequencies = self._frequencies_from_similar(text, args)
|
|
291
|
+
else:
|
|
292
|
+
frequencies = self._frequencies_from_words(text, args)
|
|
293
|
+
|
|
294
|
+
if not frequencies:
|
|
295
|
+
raise SystemExit(
|
|
296
|
+
"集計対象が0件です。入力が空か、全ての単語がストップワードで除去された"
|
|
297
|
+
"可能性があります。--no-default-stopwords を試してください。"
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
font_path = Path(args.font) if args.font else _DEFAULT_FONT
|
|
301
|
+
if not font_path.exists():
|
|
302
|
+
raise SystemExit(f"フォントが見つかりません: {font_path}")
|
|
303
|
+
|
|
304
|
+
wc = WordCloud(font_path=str(font_path), width=800, height=600, background_color="white")
|
|
305
|
+
result = wc.generate_from_frequencies(frequencies).to_image()
|
|
306
|
+
|
|
307
|
+
output_path = save_result(loaded, result, "cwc", args.output)
|
|
308
|
+
print(describe_output(output_path))
|
|
309
|
+
return 0
|
|
310
|
+
|
|
311
|
+
def _validate_exclusive_options(self, args: argparse.Namespace) -> None:
|
|
312
|
+
if args.semantic and (args.wakachi or args.hinshi is not None):
|
|
313
|
+
raise SystemExit("--semantic は -w / --hinshi と同時に指定できません")
|
|
314
|
+
if args.similar and (args.semantic or args.wakachi or args.hinshi is not None):
|
|
315
|
+
raise SystemExit("--similar は --semantic / -w / --hinshi と同時に指定できません")
|
|
316
|
+
|
|
317
|
+
def _frequencies_from_words(self, text: str, args: argparse.Namespace) -> dict[str, int]:
|
|
318
|
+
use_wakachi = args.wakachi or args.hinshi is not None
|
|
319
|
+
if args.hinshi is not None and not args.wakachi:
|
|
320
|
+
logger.info("--hinshi 指定のため分かち書き経路を自動的に有効化します")
|
|
321
|
+
|
|
322
|
+
if use_wakachi:
|
|
323
|
+
words = self._tokens_from_wakachi(text, args)
|
|
324
|
+
else:
|
|
325
|
+
words = _split_default(text)
|
|
326
|
+
if args.semantic:
|
|
327
|
+
words = self._apply_synonyms(words, args)
|
|
328
|
+
|
|
329
|
+
stopwords = _resolve_stopwords(args)
|
|
330
|
+
words = [w for w in words if w not in stopwords]
|
|
331
|
+
|
|
332
|
+
frequencies: dict[str, int] = {}
|
|
333
|
+
for word in words:
|
|
334
|
+
frequencies[word] = frequencies.get(word, 0) + 1
|
|
335
|
+
return frequencies
|
|
336
|
+
|
|
337
|
+
def _frequencies_from_similar(self, text: str, args: argparse.Namespace) -> dict[str, int]:
|
|
338
|
+
sentences = [s for s in _SENTENCE_SPLIT_PATTERN.split(text) if s]
|
|
339
|
+
if not sentences:
|
|
340
|
+
return {}
|
|
341
|
+
|
|
342
|
+
if len(sentences) > _SIMILAR_MAX_SENTENCES:
|
|
343
|
+
logger.warning(
|
|
344
|
+
"--similar の入力文数が上限(%d)を超えたため、先頭%d件のみ使用します(全%d件)",
|
|
345
|
+
_SIMILAR_MAX_SENTENCES,
|
|
346
|
+
_SIMILAR_MAX_SENTENCES,
|
|
347
|
+
len(sentences),
|
|
348
|
+
)
|
|
349
|
+
sentences = sentences[:_SIMILAR_MAX_SENTENCES]
|
|
350
|
+
|
|
351
|
+
vectors = embed_sentences(sentences, model_name=args.similar_model)
|
|
352
|
+
distance_matrix = 1.0 - (vectors @ vectors.T)
|
|
353
|
+
labels = agglomerative_average_linkage(distance_matrix, args.similar_threshold)
|
|
354
|
+
|
|
355
|
+
clusters: dict[int, list[str]] = {}
|
|
356
|
+
for sentence, label in zip(sentences, labels, strict=True):
|
|
357
|
+
clusters.setdefault(int(label), []).append(sentence)
|
|
358
|
+
|
|
359
|
+
max_length = args.similar_max_length
|
|
360
|
+
frequencies: dict[str, int] = {}
|
|
361
|
+
for members in clusters.values():
|
|
362
|
+
counts: dict[str, int] = {}
|
|
363
|
+
for s in members:
|
|
364
|
+
counts[s] = counts.get(s, 0) + 1
|
|
365
|
+
best_count = max(counts.values())
|
|
366
|
+
candidates = [s for s, c in counts.items() if c == best_count]
|
|
367
|
+
representative = min(candidates, key=len)
|
|
368
|
+
|
|
369
|
+
display = representative
|
|
370
|
+
if max_length > 0 and len(display) > max_length:
|
|
371
|
+
display = display[:max_length] + "…"
|
|
372
|
+
logger.info(
|
|
373
|
+
"代表文を切り詰めました: %s -> %s(元クラスタ件数=%d)",
|
|
374
|
+
representative,
|
|
375
|
+
display,
|
|
376
|
+
len(members),
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
if display in frequencies:
|
|
380
|
+
logger.info(
|
|
381
|
+
"切り詰め後の表示が別クラスタと衝突したため頻度を合算します: %s", display
|
|
382
|
+
)
|
|
383
|
+
frequencies[display] = frequencies.get(display, 0) + len(members)
|
|
384
|
+
|
|
385
|
+
return frequencies
|
|
386
|
+
|
|
387
|
+
def _tokens_from_wakachi(self, text: str, args: argparse.Namespace) -> list[str]:
|
|
388
|
+
user_dict = _resolve_user_dict(args)
|
|
389
|
+
tokens = _split_wakachi(text, user_dict)
|
|
390
|
+
|
|
391
|
+
hinshi_filter: set[str] | None = None
|
|
392
|
+
if args.hinshi is not None:
|
|
393
|
+
hinshi_filter = set()
|
|
394
|
+
for chunk in args.hinshi:
|
|
395
|
+
hinshi_filter.update(h.strip() for h in chunk.split(",") if h.strip())
|
|
396
|
+
unknown = hinshi_filter - set(_HINSHI_CHOICES)
|
|
397
|
+
if unknown:
|
|
398
|
+
raise SystemExit(
|
|
399
|
+
f"未知の品詞名です: {', '.join(sorted(unknown))}"
|
|
400
|
+
f"(指定可能: {', '.join(_HINSHI_CHOICES)})"
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
words = []
|
|
404
|
+
for surface, base_form, hinshi in tokens:
|
|
405
|
+
if hinshi_filter is not None and hinshi not in hinshi_filter:
|
|
406
|
+
continue
|
|
407
|
+
if hinshi_filter is not None:
|
|
408
|
+
words.append(base_form if base_form != "*" else surface)
|
|
409
|
+
else:
|
|
410
|
+
words.append(surface)
|
|
411
|
+
return words
|
|
412
|
+
|
|
413
|
+
def _apply_synonyms(self, words: list[str], args: argparse.Namespace) -> list[str]:
|
|
414
|
+
synonym_dict = _resolve_synonym_dict(args)
|
|
415
|
+
if synonym_dict is None:
|
|
416
|
+
return words
|
|
417
|
+
|
|
418
|
+
mapping = _load_synonym_map(synonym_dict)
|
|
419
|
+
result = []
|
|
420
|
+
for word in words:
|
|
421
|
+
representative = mapping.get(word)
|
|
422
|
+
if representative is not None and representative != word:
|
|
423
|
+
logger.info("同義語として寄せました: %s -> %s", word, representative)
|
|
424
|
+
result.append(representative)
|
|
425
|
+
else:
|
|
426
|
+
result.append(word)
|
|
427
|
+
return result
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
import cv2
|
|
7
|
+
import numpy as np
|
|
8
|
+
from PIL import Image
|
|
9
|
+
|
|
10
|
+
from workpytools.common.clipboard import load_image
|
|
11
|
+
from workpytools.common.output import describe_output, save_result
|
|
12
|
+
from workpytools.processing.base import Processor
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
_DEFAULT_H_COLOR = 10.0
|
|
17
|
+
_DEFAULT_TEMPLATE_WINDOW_SIZE = 7
|
|
18
|
+
_DEFAULT_SEARCH_WINDOW_SIZE = 21
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DenoiseProcessor(Processor):
|
|
22
|
+
"""Remove photographic grain / sensor noise using OpenCV Non-local Means Denoising.
|
|
23
|
+
|
|
24
|
+
Input source and default output location follow the same convention as
|
|
25
|
+
touka: explicit `path` or a file copied in Explorer saves next to (or,
|
|
26
|
+
for the clipboard-file case, into the OS temp dir and back onto the
|
|
27
|
+
clipboard as) `{stem}_denoised.png`; raw clipboard image data skips the
|
|
28
|
+
file and is placed directly on the clipboard as image data.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
name = "denoise"
|
|
32
|
+
help = "画像のノイズ除去(ファイルパス/クリップボード入力対応、OpenCV Non-local Means)"
|
|
33
|
+
|
|
34
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"path",
|
|
37
|
+
nargs="?",
|
|
38
|
+
default=None,
|
|
39
|
+
help="画像ファイルパス。省略時はクリップボードの画像/コピーしたファイルを使用",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"-o", "--output", default=None, help="出力先パス(省略時は自動生成、拡張子はpng)"
|
|
43
|
+
)
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--strength",
|
|
46
|
+
type=float,
|
|
47
|
+
default=10.0,
|
|
48
|
+
help="ノイズ除去強度 h(デフォルト: 10.0)。"
|
|
49
|
+
"値を大きくするほどノイズは減るがディテールも失われる",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def run(self, args: argparse.Namespace) -> int:
|
|
53
|
+
loaded = load_image(args.path)
|
|
54
|
+
logger.info("denoise starting (size=%s, strength=%s)", loaded.image.size, args.strength)
|
|
55
|
+
|
|
56
|
+
result = self._denoise(loaded.image, h=args.strength)
|
|
57
|
+
|
|
58
|
+
output_path = save_result(loaded, result, "denoised", args.output)
|
|
59
|
+
print(describe_output(output_path))
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def _denoise(image: Image.Image, h: float) -> Image.Image:
|
|
64
|
+
"""Apply fastNlMeansDenoisingColored, preserving the original alpha channel untouched."""
|
|
65
|
+
rgba = np.array(image)
|
|
66
|
+
rgb = rgba[:, :, :3]
|
|
67
|
+
alpha = rgba[:, :, 3]
|
|
68
|
+
|
|
69
|
+
bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
|
|
70
|
+
denoised_bgr = cv2.fastNlMeansDenoisingColored(
|
|
71
|
+
bgr,
|
|
72
|
+
None,
|
|
73
|
+
h=h,
|
|
74
|
+
hColor=_DEFAULT_H_COLOR,
|
|
75
|
+
templateWindowSize=_DEFAULT_TEMPLATE_WINDOW_SIZE,
|
|
76
|
+
searchWindowSize=_DEFAULT_SEARCH_WINDOW_SIZE,
|
|
77
|
+
)
|
|
78
|
+
denoised_rgb = cv2.cvtColor(denoised_bgr, cv2.COLOR_BGR2RGB)
|
|
79
|
+
|
|
80
|
+
out = np.dstack([denoised_rgb, alpha])
|
|
81
|
+
return Image.fromarray(out, mode="RGBA")
|