GameSentenceMiner 2.15.9__py3-none-any.whl → 2.15.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- GameSentenceMiner/ocr/gsm_ocr_config.py +1 -1
- GameSentenceMiner/ocr/owocr_helper.py +21 -12
- GameSentenceMiner/owocr/owocr/run.py +2 -2
- GameSentenceMiner/util/configuration.py +3 -0
- GameSentenceMiner/util/text_log.py +2 -2
- GameSentenceMiner/web/database_api.py +783 -0
- GameSentenceMiner/web/events.py +178 -0
- GameSentenceMiner/web/stats.py +582 -0
- GameSentenceMiner/web/templates/database.html +277 -0
- GameSentenceMiner/web/templates/search.html +103 -0
- GameSentenceMiner/web/templates/stats.html +330 -0
- GameSentenceMiner/web/templates/text_replacements.html +211 -0
- GameSentenceMiner/web/templates/utility.html +2 -2
- GameSentenceMiner/web/texthooking_page.py +58 -316
- GameSentenceMiner/web/websockets.py +120 -0
- {gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.10.dist-info}/METADATA +1 -1
- {gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.10.dist-info}/RECORD +21 -15
- GameSentenceMiner/web/templates/__init__.py +0 -0
- {gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.10.dist-info}/WHEEL +0 -0
- {gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.10.dist-info}/entry_points.txt +0 -0
- {gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.10.dist-info}/licenses/LICENSE +0 -0
- {gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.10.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
|
|
2
|
+
<!DOCTYPE html>
|
|
3
|
+
<html lang="en">
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="UTF-8">
|
|
6
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
7
|
+
<title>GSM Database Management</title>
|
|
8
|
+
|
|
9
|
+
<!-- Include shared theme styles -->
|
|
10
|
+
{% include 'components/theme-styles.html' %}
|
|
11
|
+
|
|
12
|
+
<!-- Include shared CSS -->
|
|
13
|
+
<link rel="stylesheet" href="/static/css/shared.css">
|
|
14
|
+
</head>
|
|
15
|
+
<body>
|
|
16
|
+
|
|
17
|
+
<div class="container">
|
|
18
|
+
<h1>Game Sentence Miner - Database Management</h1>
|
|
19
|
+
|
|
20
|
+
<!-- Include shared navigation -->
|
|
21
|
+
{% include 'components/navigation.html' %}
|
|
22
|
+
|
|
23
|
+
<div class="management-grid">
|
|
24
|
+
<!-- Games Management Card -->
|
|
25
|
+
<div class="management-card games">
|
|
26
|
+
<div class="card-header">
|
|
27
|
+
<div class="card-icon">🗑️</div>
|
|
28
|
+
<h2 class="card-title">Games Management</h2>
|
|
29
|
+
</div>
|
|
30
|
+
<p class="card-description">
|
|
31
|
+
Manage your game data by viewing statistics and selectively deleting games and their associated sentences.
|
|
32
|
+
</p>
|
|
33
|
+
<div class="stats-row">
|
|
34
|
+
<span class="stats-label">Total Games:</span>
|
|
35
|
+
<span class="stats-value" id="totalGamesCount">Loading...</span>
|
|
36
|
+
</div>
|
|
37
|
+
<div class="stats-row">
|
|
38
|
+
<span class="stats-label">Total Sentences:</span>
|
|
39
|
+
<span class="stats-value" id="totalSentencesCount">Loading...</span>
|
|
40
|
+
</div>
|
|
41
|
+
<div class="card-actions">
|
|
42
|
+
<button class="action-btn danger" data-action="openGameDeletionModal">
|
|
43
|
+
Manage Games
|
|
44
|
+
</button>
|
|
45
|
+
</div>
|
|
46
|
+
</div>
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
<!-- Text Lines Management Card -->
|
|
50
|
+
<div class="management-card text-lines">
|
|
51
|
+
<div class="card-header">
|
|
52
|
+
<div class="card-icon">📄</div>
|
|
53
|
+
<h2 class="card-title">Manage Text Lines</h2>
|
|
54
|
+
</div>
|
|
55
|
+
<p class="card-description">
|
|
56
|
+
Manage specific lines of text based on content matching.
|
|
57
|
+
</p>
|
|
58
|
+
<div class="stats-row">
|
|
59
|
+
<span class="stats-label">Last Cleanup:</span>
|
|
60
|
+
<span class="stats-value" id="lastCleanupDate">Never</span>
|
|
61
|
+
</div>
|
|
62
|
+
<div class="card-actions">
|
|
63
|
+
<button class="action-btn warning" data-action="openTextLinesModal">
|
|
64
|
+
Manage Text Lines
|
|
65
|
+
</button>
|
|
66
|
+
</div>
|
|
67
|
+
</div>
|
|
68
|
+
|
|
69
|
+
<!-- Deduplication Card -->
|
|
70
|
+
<div class="management-card deduplication">
|
|
71
|
+
<div class="card-header">
|
|
72
|
+
<div class="card-icon">🔄</div>
|
|
73
|
+
<h2 class="card-title">Data Deduplication</h2>
|
|
74
|
+
</div>
|
|
75
|
+
<p class="card-description">
|
|
76
|
+
Manage duplicate sentences to improve data quality and reduce storage usage.
|
|
77
|
+
</p>
|
|
78
|
+
<div class="stats-row">
|
|
79
|
+
<span class="stats-label">Potential Duplicates:</span>
|
|
80
|
+
<span class="stats-value" id="duplicatesCount">Calculating...</span>
|
|
81
|
+
</div>
|
|
82
|
+
<div class="card-actions">
|
|
83
|
+
<button class="action-btn success" data-action="openDeduplicationModal">
|
|
84
|
+
Manage Duplicates
|
|
85
|
+
</button>
|
|
86
|
+
</div>
|
|
87
|
+
</div>
|
|
88
|
+
</div>
|
|
89
|
+
</div>
|
|
90
|
+
|
|
91
|
+
<!-- Games Deletion Modal (moved from stats.html) -->
|
|
92
|
+
<div id="gamesDeletionModal" class="modal">
|
|
93
|
+
<div class="modal-content" style="max-width: 800px;">
|
|
94
|
+
<div class="modal-header">
|
|
95
|
+
<h3>Delete Games</h3>
|
|
96
|
+
<span class="close-btn" data-action="closeModal" data-modal="gamesDeletionModal">×</span>
|
|
97
|
+
</div>
|
|
98
|
+
<div class="modal-body">
|
|
99
|
+
<p class="warning-text">Select games to delete. This will permanently remove all associated sentences and data.</p>
|
|
100
|
+
|
|
101
|
+
<div id="gamesLoadingIndicator" class="loading-indicator">
|
|
102
|
+
<div class="spinner"></div>
|
|
103
|
+
<span>Loading games...</span>
|
|
104
|
+
</div>
|
|
105
|
+
|
|
106
|
+
<div id="gamesContent" style="display: none;">
|
|
107
|
+
<div style="margin-bottom: 20px;">
|
|
108
|
+
<button class="action-btn primary" data-action="selectAllGames">Select All</button>
|
|
109
|
+
<button class="action-btn primary" data-action="selectNoGames">Select None</button>
|
|
110
|
+
</div>
|
|
111
|
+
<div id="gamesList" style="max-height: 300px; overflow-y: auto; border: 1px solid var(--border-color); border-radius: 5px; padding: 10px;">
|
|
112
|
+
<!-- Games will be populated here -->
|
|
113
|
+
</div>
|
|
114
|
+
</div>
|
|
115
|
+
</div>
|
|
116
|
+
<div class="modal-footer">
|
|
117
|
+
<button class="action-btn" data-action="closeModal" data-modal="gamesDeletionModal">Cancel</button>
|
|
118
|
+
<button class="action-btn danger" id="deleteSelectedGamesBtn" data-action="deleteSelectedGames" disabled>Delete Selected</button>
|
|
119
|
+
</div>
|
|
120
|
+
</div>
|
|
121
|
+
</div>
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
<!-- Text Lines Deletion Modal -->
|
|
125
|
+
<div id="textLinesModal" class="modal">
|
|
126
|
+
<div class="modal-content">
|
|
127
|
+
<div class="modal-header">
|
|
128
|
+
<h3>Delete Text Lines</h3>
|
|
129
|
+
<span class="close-btn" data-action="closeModal" data-modal="textLinesModal">×</span>
|
|
130
|
+
</div>
|
|
131
|
+
<div class="modal-body">
|
|
132
|
+
<p class="warning-text">Remove specific lines of text from all games using preset patterns or custom regex. This operation cannot be undone.</p>
|
|
133
|
+
|
|
134
|
+
<div class="form-group">
|
|
135
|
+
<label class="form-label">Preset Patterns:</label>
|
|
136
|
+
<select id="presetPatterns" class="form-input">
|
|
137
|
+
<option value="">Select a preset pattern...</option>
|
|
138
|
+
<option value="lines_over_50">Lines over 50 characters</option>
|
|
139
|
+
<option value="lines_over_100">Lines over 100 characters</option>
|
|
140
|
+
<option value="non_japanese">Non-Japanese text</option>
|
|
141
|
+
<option value="ascii_only">ASCII-only lines</option>
|
|
142
|
+
<option value="empty_lines">Empty or whitespace-only lines</option>
|
|
143
|
+
<option value="numbers_only">Lines with numbers only</option>
|
|
144
|
+
<option value="single_char">Single character lines</option>
|
|
145
|
+
<option value="repeated_chars">Lines with repeated characters (3+ times)</option>
|
|
146
|
+
</select>
|
|
147
|
+
<small style="color: var(--text-tertiary); font-size: 12px;">
|
|
148
|
+
Select a common pattern or create a custom regex below.
|
|
149
|
+
</small>
|
|
150
|
+
</div>
|
|
151
|
+
|
|
152
|
+
<div class="form-group">
|
|
153
|
+
<label class="form-label">Custom Regex Pattern:</label>
|
|
154
|
+
<input type="text" id="customRegex" class="form-input" placeholder="Enter custom regex pattern (e.g., ^.{0,10}$ for lines 10 chars or less)">
|
|
155
|
+
<small style="color: var(--text-tertiary); font-size: 12px;">
|
|
156
|
+
Use regex syntax. Leave empty to use exact text matching below.
|
|
157
|
+
</small>
|
|
158
|
+
</div>
|
|
159
|
+
|
|
160
|
+
<div class="form-group">
|
|
161
|
+
<label class="form-label">Exact Text to Delete:</label>
|
|
162
|
+
<textarea id="textToDelete" class="form-textarea" rows="4" placeholder="Enter exact text lines to delete, one per line..."></textarea>
|
|
163
|
+
<small style="color: var(--text-tertiary); font-size: 12px;">
|
|
164
|
+
Each line will be treated as an exact match. Only used if no regex pattern is provided.
|
|
165
|
+
</small>
|
|
166
|
+
</div>
|
|
167
|
+
|
|
168
|
+
<div class="form-group">
|
|
169
|
+
<div class="checkbox-container">
|
|
170
|
+
<input type="checkbox" id="caseSensitiveDelete" class="checkbox-input">
|
|
171
|
+
<label for="caseSensitiveDelete" class="checkbox-label">Case sensitive matching</label>
|
|
172
|
+
</div>
|
|
173
|
+
</div>
|
|
174
|
+
|
|
175
|
+
<div class="form-group">
|
|
176
|
+
<div class="checkbox-container">
|
|
177
|
+
<input type="checkbox" id="useRegexDelete" class="checkbox-input">
|
|
178
|
+
<label for="useRegexDelete" class="checkbox-label">Treat custom pattern as regex</label>
|
|
179
|
+
</div>
|
|
180
|
+
</div>
|
|
181
|
+
|
|
182
|
+
<div id="previewDeleteResults" style="display: none; background: var(--bg-tertiary); padding: 15px; border-radius: 5px; margin: 15px 0;">
|
|
183
|
+
<h4 style="margin: 0 0 10px 0; color: var(--text-primary);">Preview Results</h4>
|
|
184
|
+
<div class="stats-row">
|
|
185
|
+
<span class="stats-label">Lines to be deleted:</span>
|
|
186
|
+
<span class="stats-value" id="previewDeleteCount">0</span>
|
|
187
|
+
</div>
|
|
188
|
+
<div id="previewDeleteSamples" style="max-height: 100px; overflow-y: auto; margin-top: 10px;"></div>
|
|
189
|
+
</div>
|
|
190
|
+
|
|
191
|
+
<div id="textLinesError" class="error-text" style="display: none;"></div>
|
|
192
|
+
<div id="textLinesSuccess" class="success-text" style="display: none;"></div>
|
|
193
|
+
</div>
|
|
194
|
+
<div class="modal-footer">
|
|
195
|
+
<button class="action-btn" data-action="closeModal" data-modal="textLinesModal">Cancel</button>
|
|
196
|
+
<button class="action-btn primary" data-action="previewTextDeletion">Preview</button>
|
|
197
|
+
<button class="action-btn warning" id="executeDeleteBtn" data-action="deleteTextLines" disabled>Delete Lines</button>
|
|
198
|
+
</div>
|
|
199
|
+
</div>
|
|
200
|
+
</div>
|
|
201
|
+
|
|
202
|
+
<!-- Deduplication Modal -->
|
|
203
|
+
<div id="deduplicationModal" class="modal">
|
|
204
|
+
<div class="modal-content">
|
|
205
|
+
<div class="modal-header">
|
|
206
|
+
<h3>Remove Duplicate Sentences</h3>
|
|
207
|
+
<span class="close-btn" data-action="closeModal" data-modal="deduplicationModal">×</span>
|
|
208
|
+
</div>
|
|
209
|
+
<div class="modal-body">
|
|
210
|
+
<p class="warning-text">Remove duplicate sentences from selected games, keeping only the first occurrence of each unique sentence within a specified time window.</p>
|
|
211
|
+
|
|
212
|
+
<div class="form-group">
|
|
213
|
+
<label class="form-label">Select Games:</label>
|
|
214
|
+
<select id="gameSelection" class="form-input" multiple style="height: 120px;">
|
|
215
|
+
<option value="all">All Games</option>
|
|
216
|
+
</select>
|
|
217
|
+
<small style="color: var(--text-tertiary); font-size: 12px;">
|
|
218
|
+
Hold Ctrl/Cmd to select multiple games. Select "All Games" to process all games at once.
|
|
219
|
+
</small>
|
|
220
|
+
</div>
|
|
221
|
+
|
|
222
|
+
<div class="form-group">
|
|
223
|
+
<label class="form-label">Time Window (minutes):</label>
|
|
224
|
+
<input type="number" id="timeWindow" class="form-input" min="1" max="1440" value="5" placeholder="5">
|
|
225
|
+
<small style="color: var(--text-tertiary); font-size: 12px;">
|
|
226
|
+
Only remove duplicates that appear within this many minutes of each other (1-1440 minutes).
|
|
227
|
+
</small>
|
|
228
|
+
</div>
|
|
229
|
+
|
|
230
|
+
<div class="form-group">
|
|
231
|
+
<div class="checkbox-container">
|
|
232
|
+
<input type="checkbox" id="caseSensitiveDedup" class="checkbox-input">
|
|
233
|
+
<label for="caseSensitiveDedup" class="checkbox-label">Case sensitive comparison</label>
|
|
234
|
+
</div>
|
|
235
|
+
</div>
|
|
236
|
+
|
|
237
|
+
<div class="form-group">
|
|
238
|
+
<div class="checkbox-container">
|
|
239
|
+
<input type="checkbox" id="preserveNewest" class="checkbox-input">
|
|
240
|
+
<label for="preserveNewest" class="checkbox-label">Keep newest occurrence instead of oldest</label>
|
|
241
|
+
</div>
|
|
242
|
+
</div>
|
|
243
|
+
|
|
244
|
+
<div id="deduplicationStats" style="display: none; background: var(--bg-tertiary); padding: 15px; border-radius: 5px; margin: 15px 0;">
|
|
245
|
+
<h4 style="margin: 0 0 10px 0; color: var(--text-primary);">Scan Results</h4>
|
|
246
|
+
<div class="stats-row">
|
|
247
|
+
<span class="stats-label">Duplicates Found:</span>
|
|
248
|
+
<span class="stats-value" id="duplicatesFoundCount">0</span>
|
|
249
|
+
</div>
|
|
250
|
+
<div class="stats-row">
|
|
251
|
+
<span class="stats-label">Games Affected:</span>
|
|
252
|
+
<span class="stats-value" id="gamesAffectedCount">0</span>
|
|
253
|
+
</div>
|
|
254
|
+
<div class="stats-row">
|
|
255
|
+
<span class="stats-label">Space to be Freed:</span>
|
|
256
|
+
<span class="stats-value" id="spaceToFree">0 sentences</span>
|
|
257
|
+
</div>
|
|
258
|
+
<div id="duplicatesSampleList" style="max-height: 100px; overflow-y: auto; margin-top: 10px;"></div>
|
|
259
|
+
</div>
|
|
260
|
+
|
|
261
|
+
<div id="deduplicationError" class="error-text" style="display: none;"></div>
|
|
262
|
+
<div id="deduplicationSuccess" class="success-text" style="display: none;"></div>
|
|
263
|
+
</div>
|
|
264
|
+
<div class="modal-footer">
|
|
265
|
+
<button class="action-btn" data-action="closeModal" data-modal="deduplicationModal">Cancel</button>
|
|
266
|
+
<button class="action-btn primary" data-action="scanForDuplicates">Scan for Duplicates</button>
|
|
267
|
+
<button class="action-btn success" id="removeDuplicatesBtn" data-action="removeDuplicates" disabled>Remove Duplicates</button>
|
|
268
|
+
</div>
|
|
269
|
+
</div>
|
|
270
|
+
</div>
|
|
271
|
+
|
|
272
|
+
<!-- Include shared JavaScript first (required dependency for database.js) -->
|
|
273
|
+
<script src="/static/js/shared.js"></script>
|
|
274
|
+
<script src="/static/js/database.js"></script>
|
|
275
|
+
|
|
276
|
+
</body>
|
|
277
|
+
</html>
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>GSM Search</title>
|
|
7
|
+
|
|
8
|
+
<!-- Include shared theme styles -->
|
|
9
|
+
{% include 'components/theme-styles.html' %}
|
|
10
|
+
|
|
11
|
+
<!-- Include shared CSS -->
|
|
12
|
+
<link rel="stylesheet" href="/static/css/shared.css">
|
|
13
|
+
|
|
14
|
+
<!-- Include search-specific CSS -->
|
|
15
|
+
<link rel="stylesheet" href="/static/css/search.css">
|
|
16
|
+
</head>
|
|
17
|
+
<body>
|
|
18
|
+
|
|
19
|
+
<div class="container">
|
|
20
|
+
<h1>Game Sentence Miner - Search</h1>
|
|
21
|
+
|
|
22
|
+
<!-- Include shared navigation -->
|
|
23
|
+
{% include 'components/navigation.html' %}
|
|
24
|
+
|
|
25
|
+
<div class="search-container">
|
|
26
|
+
<div class="search-header">
|
|
27
|
+
<h2>Search Your Collected Sentences</h2>
|
|
28
|
+
<p class="search-description">Find sentences from your gaming sessions with real-time search and filtering</p>
|
|
29
|
+
</div>
|
|
30
|
+
|
|
31
|
+
<div class="search-input-container">
|
|
32
|
+
<input
|
|
33
|
+
type="text"
|
|
34
|
+
id="searchInput"
|
|
35
|
+
class="search-input"
|
|
36
|
+
placeholder="Search for sentences, words, or phrases..."
|
|
37
|
+
autocomplete="off"
|
|
38
|
+
/>
|
|
39
|
+
</div>
|
|
40
|
+
|
|
41
|
+
<div class="search-filters">
|
|
42
|
+
<div class="filter-group">
|
|
43
|
+
<label class="filter-label">Game:</label>
|
|
44
|
+
<select id="gameFilter" class="filter-select">
|
|
45
|
+
<option value="">All Games</option>
|
|
46
|
+
</select>
|
|
47
|
+
</div>
|
|
48
|
+
<div class="filter-group">
|
|
49
|
+
<label class="filter-label">Sort by:</label>
|
|
50
|
+
<select id="sortFilter" class="filter-select">
|
|
51
|
+
<option value="relevance">Relevance</option>
|
|
52
|
+
<option value="date_desc">Newest First</option>
|
|
53
|
+
<option value="date_asc">Oldest First</option>
|
|
54
|
+
<option value="game_name">Game Name</option>
|
|
55
|
+
</select>
|
|
56
|
+
</div>
|
|
57
|
+
</div>
|
|
58
|
+
|
|
59
|
+
<div class="search-stats">
|
|
60
|
+
<span id="searchStats">Ready to search</span>
|
|
61
|
+
<span id="searchTime"></span>
|
|
62
|
+
</div>
|
|
63
|
+
</div>
|
|
64
|
+
|
|
65
|
+
<div class="results-container">
|
|
66
|
+
<div id="emptyState" class="empty-state">
|
|
67
|
+
<div class="empty-state-icon">🔍</div>
|
|
68
|
+
<h3>Search Your Gaming Journey</h3>
|
|
69
|
+
<p>Enter a word or phrase above to find sentences from your collected gaming sessions.<br>
|
|
70
|
+
You can search across all games or filter by specific titles.</p>
|
|
71
|
+
</div>
|
|
72
|
+
|
|
73
|
+
<div id="loadingIndicator" class="loading-indicator" style="display: none;">
|
|
74
|
+
<div class="spinner"></div>
|
|
75
|
+
<span>Searching...</span>
|
|
76
|
+
</div>
|
|
77
|
+
|
|
78
|
+
<div id="noResults" class="no-results" style="display: none;">
|
|
79
|
+
<div class="no-results-icon">📭</div>
|
|
80
|
+
<h3>No Results Found</h3>
|
|
81
|
+
<p>Try adjusting your search terms or filters</p>
|
|
82
|
+
</div>
|
|
83
|
+
|
|
84
|
+
<div id="errorMessage" class="error-message" style="display: none;">
|
|
85
|
+
<strong>Search Error:</strong> <span id="errorText"></span>
|
|
86
|
+
</div>
|
|
87
|
+
|
|
88
|
+
<div id="searchResults"></div>
|
|
89
|
+
|
|
90
|
+
<div id="pagination" class="pagination" style="display: none;">
|
|
91
|
+
<button id="prevPage" class="page-btn">« Previous</button>
|
|
92
|
+
<span id="pageInfo"></span>
|
|
93
|
+
<button id="nextPage" class="page-btn">Next »</button>
|
|
94
|
+
</div>
|
|
95
|
+
</div>
|
|
96
|
+
</div>
|
|
97
|
+
|
|
98
|
+
<!-- Include shared JavaScript first (required dependency for search.js) -->
|
|
99
|
+
<script src="/static/js/shared.js"></script>
|
|
100
|
+
<script src="/static/js/search.js"></script>
|
|
101
|
+
|
|
102
|
+
</body>
|
|
103
|
+
</html>
|