photo-stack-finder 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orchestrator/__init__.py +2 -2
- orchestrator/app.py +6 -11
- orchestrator/build_pipeline.py +19 -21
- orchestrator/orchestrator_runner.py +11 -8
- orchestrator/pipeline_builder.py +126 -126
- orchestrator/pipeline_orchestrator.py +604 -604
- orchestrator/review_persistence.py +162 -162
- orchestrator/static/orchestrator.css +76 -76
- orchestrator/static/orchestrator.html +11 -5
- orchestrator/static/orchestrator.js +3 -1
- overlap_metrics/__init__.py +1 -1
- overlap_metrics/config.py +135 -135
- overlap_metrics/core.py +284 -284
- overlap_metrics/estimators.py +292 -292
- overlap_metrics/metrics.py +307 -307
- overlap_metrics/registry.py +99 -99
- overlap_metrics/utils.py +104 -104
- photo_compare/__init__.py +1 -1
- photo_compare/base.py +285 -285
- photo_compare/config.py +225 -225
- photo_compare/distance.py +15 -15
- photo_compare/feature_methods.py +173 -173
- photo_compare/file_hash.py +29 -29
- photo_compare/hash_methods.py +99 -99
- photo_compare/histogram_methods.py +118 -118
- photo_compare/pixel_methods.py +58 -58
- photo_compare/structural_methods.py +104 -104
- photo_compare/types.py +28 -28
- {photo_stack_finder-0.1.7.dist-info → photo_stack_finder-0.1.8.dist-info}/METADATA +21 -22
- photo_stack_finder-0.1.8.dist-info/RECORD +75 -0
- scripts/orchestrate.py +12 -10
- utils/__init__.py +4 -3
- utils/base_pipeline_stage.py +171 -171
- utils/base_ports.py +176 -176
- utils/benchmark_utils.py +823 -823
- utils/channel.py +74 -74
- utils/comparison_gates.py +40 -21
- utils/compute_benchmarks.py +355 -355
- utils/compute_identical.py +94 -24
- utils/compute_indices.py +235 -235
- utils/compute_perceptual_hash.py +127 -127
- utils/compute_perceptual_match.py +240 -240
- utils/compute_sha_bins.py +64 -20
- utils/compute_template_similarity.py +1 -1
- utils/compute_versions.py +483 -483
- utils/config.py +8 -5
- utils/data_io.py +83 -83
- utils/graph_context.py +44 -44
- utils/logger.py +2 -2
- utils/models.py +2 -2
- utils/photo_file.py +90 -91
- utils/pipeline_graph.py +334 -334
- utils/pipeline_stage.py +408 -408
- utils/plot_helpers.py +123 -123
- utils/ports.py +136 -136
- utils/progress.py +415 -415
- utils/report_builder.py +139 -139
- utils/review_types.py +55 -55
- utils/review_utils.py +10 -19
- utils/sequence.py +10 -8
- utils/sequence_clustering.py +1 -1
- utils/template.py +57 -57
- utils/template_parsing.py +71 -0
- photo_stack_finder-0.1.7.dist-info/RECORD +0 -74
- {photo_stack_finder-0.1.7.dist-info → photo_stack_finder-0.1.8.dist-info}/WHEEL +0 -0
- {photo_stack_finder-0.1.7.dist-info → photo_stack_finder-0.1.8.dist-info}/entry_points.txt +0 -0
- {photo_stack_finder-0.1.7.dist-info → photo_stack_finder-0.1.8.dist-info}/licenses/LICENSE +0 -0
- {photo_stack_finder-0.1.7.dist-info → photo_stack_finder-0.1.8.dist-info}/top_level.txt +0 -0
utils/progress.py
CHANGED
|
@@ -1,415 +1,415 @@
|
|
|
1
|
-
"""Progress tracking with in-place updates for cleaner logs."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import sys
|
|
6
|
-
import threading
|
|
7
|
-
import time
|
|
8
|
-
from dataclasses import dataclass
|
|
9
|
-
from datetime import datetime, timedelta
|
|
10
|
-
from typing import Any
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@dataclass
|
|
14
|
-
class ProgressInfo:
|
|
15
|
-
"""Complete progress information, pre-formatted for UI display.
|
|
16
|
-
|
|
17
|
-
All formatting happens in the backend - frontend just displays values.
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
# Raw values (for programmatic use if needed)
|
|
21
|
-
fraction_complete: float # 0.0 to 1.0
|
|
22
|
-
current_count: int # Current item count
|
|
23
|
-
total_count: int | None # Total item count (None if unknown)
|
|
24
|
-
rate: float | None # Items per second
|
|
25
|
-
eta_seconds: float | None # Estimated seconds remaining
|
|
26
|
-
|
|
27
|
-
# Formatted strings (ready for direct display in HTML)
|
|
28
|
-
percentage_display: str # "73%"
|
|
29
|
-
progress_bar_width: str # "73%" (CSS width property)
|
|
30
|
-
status_message: str # "Processing photos"
|
|
31
|
-
items_display: str | None # "11,123 / 15,234" or None
|
|
32
|
-
rate_display: str | None # "1,250 items/sec" or None
|
|
33
|
-
eta_display: str | None # "3 seconds" or "2 minutes" or "1h 23m"
|
|
34
|
-
stage_display: str # "Compute Identical"
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def _format_time(seconds: float) -> str:
|
|
38
|
-
"""Format time in seconds to human-readable format.
|
|
39
|
-
|
|
40
|
-
Args:
|
|
41
|
-
seconds: Time in seconds
|
|
42
|
-
|
|
43
|
-
Returns:
|
|
44
|
-
Formatted string like "2d3h45s" or "1h30s" or "45s"
|
|
45
|
-
"""
|
|
46
|
-
if seconds < 60:
|
|
47
|
-
return f"{seconds:.0f}s"
|
|
48
|
-
|
|
49
|
-
days: int = int(seconds // 86400)
|
|
50
|
-
remaining: float = seconds % 86400
|
|
51
|
-
hours: int = int(remaining // 3600)
|
|
52
|
-
remaining = remaining % 3600
|
|
53
|
-
minutes: int = int(remaining // 60)
|
|
54
|
-
secs: int = int(remaining % 60)
|
|
55
|
-
|
|
56
|
-
parts: list[str] = []
|
|
57
|
-
if days > 0:
|
|
58
|
-
parts.append(f"{days}d")
|
|
59
|
-
if hours > 0:
|
|
60
|
-
parts.append(f"{hours}h")
|
|
61
|
-
if minutes > 0:
|
|
62
|
-
parts.append(f"{minutes}m")
|
|
63
|
-
if secs > 0 or not parts: # Always show seconds if nothing else to show
|
|
64
|
-
parts.append(f"{secs}s")
|
|
65
|
-
|
|
66
|
-
return "".join(parts)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def _format_completion_time(eta_seconds: float) -> str:
|
|
70
|
-
"""Format estimated completion time as a timestamp.
|
|
71
|
-
|
|
72
|
-
Args:
|
|
73
|
-
eta_seconds: Estimated seconds remaining until completion
|
|
74
|
-
|
|
75
|
-
Returns:
|
|
76
|
-
Formatted string like "14:35:20" for same-day or "tomorrow 09:30" for next day
|
|
77
|
-
"""
|
|
78
|
-
completion: datetime = datetime.now() + timedelta(seconds=eta_seconds)
|
|
79
|
-
now: datetime = datetime.now()
|
|
80
|
-
|
|
81
|
-
# Check if completion is tomorrow or later
|
|
82
|
-
if completion.date() > now.date():
|
|
83
|
-
days_diff: int = (completion.date() - now.date()).days
|
|
84
|
-
if days_diff == 1:
|
|
85
|
-
return f"tomorrow {completion.strftime('%H:%M')}"
|
|
86
|
-
return f"in {days_diff} days at {completion.strftime('%H:%M')}"
|
|
87
|
-
|
|
88
|
-
# Same day - just show time
|
|
89
|
-
return completion.strftime("%H:%M:%S")
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def format_seconds_weighted(total_seconds: float) -> str:
|
|
93
|
-
"""Formats a number of seconds into DDDdHHhMMmSS.ffs.
|
|
94
|
-
|
|
95
|
-
Omits leading zero-value fields.
|
|
96
|
-
"""
|
|
97
|
-
if total_seconds < 0:
|
|
98
|
-
return "-" + format_seconds_weighted(-total_seconds)
|
|
99
|
-
|
|
100
|
-
# 1. Separate whole seconds and fractional seconds
|
|
101
|
-
whole_seconds = int(total_seconds)
|
|
102
|
-
fractional_seconds = total_seconds - whole_seconds
|
|
103
|
-
|
|
104
|
-
# 2. Convert whole seconds to Days, Hours, Minutes, Seconds
|
|
105
|
-
days = whole_seconds // (24 * 3600)
|
|
106
|
-
whole_seconds %= 24 * 3600
|
|
107
|
-
|
|
108
|
-
hours = whole_seconds // 3600
|
|
109
|
-
whole_seconds %= 3600
|
|
110
|
-
|
|
111
|
-
minutes = whole_seconds // 60
|
|
112
|
-
seconds = whole_seconds % 60
|
|
113
|
-
|
|
114
|
-
# 3. Build the components list (Days, Hours, Minutes, Seconds)
|
|
115
|
-
components = []
|
|
116
|
-
|
|
117
|
-
if days > 0:
|
|
118
|
-
components.append(f"{days}d")
|
|
119
|
-
|
|
120
|
-
if hours > 0 or (days > 0 and (minutes > 0 or seconds > 0 or fractional_seconds > 0)):
|
|
121
|
-
# Include hours if non-zero, OR if days are present and there's any time after hours.
|
|
122
|
-
components.append(f"{hours}h")
|
|
123
|
-
|
|
124
|
-
if minutes > 0 or (hours > 0 and (seconds > 0 or fractional_seconds > 0)):
|
|
125
|
-
# Include minutes if non-zero, OR if hours are present and there's any time after minutes.
|
|
126
|
-
components.append(f"{minutes}m")
|
|
127
|
-
|
|
128
|
-
# 4. Handle the Seconds and Fractional part
|
|
129
|
-
|
|
130
|
-
# The seconds and fractional part must always be included,
|
|
131
|
-
# unless the entire duration is 0.0.
|
|
132
|
-
if not components and seconds == 0 and fractional_seconds == 0:
|
|
133
|
-
return "0s" # Special case for exact zero
|
|
134
|
-
|
|
135
|
-
# The Seconds part is built using the combined whole and fractional seconds
|
|
136
|
-
# which ensures the correct two decimal places.
|
|
137
|
-
# Note: We must use the 'seconds' variable and add the fractional part back
|
|
138
|
-
# to avoid issues with floating point arithmetic accumulating large errors
|
|
139
|
-
# if we simply calculated total_seconds % 60.
|
|
140
|
-
|
|
141
|
-
formatted_seconds = seconds + fractional_seconds
|
|
142
|
-
|
|
143
|
-
# Check if a higher-order component (d, h, m) was printed.
|
|
144
|
-
# If not, the seconds field is the leading field and should not be zero-padded.
|
|
145
|
-
# The format is S.ffs (no padding) or SS.ffs (padding if it follows another field)
|
|
146
|
-
|
|
147
|
-
if components:
|
|
148
|
-
# Pad with 0 to ensure two digits if it follows an hour or minute field
|
|
149
|
-
# e.g., '1h05.12s', not '1h5.12s'
|
|
150
|
-
components.append(f"{formatted_seconds:05.2f}s")
|
|
151
|
-
else:
|
|
152
|
-
# This is the leading field, so no mandatory padding on the integer part
|
|
153
|
-
# The formatting will be "S.ffs" or "SS.ffs"
|
|
154
|
-
components.append(f"{formatted_seconds:.2f}s")
|
|
155
|
-
|
|
156
|
-
# 5. Combine and return
|
|
157
|
-
return "".join(components).replace(
|
|
158
|
-
":", ""
|
|
159
|
-
) # Remove potential colons from hour/minute formatting if using a different approach
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
class ProgressTracker:
|
|
163
|
-
"""Track progress with in-place console updates.
|
|
164
|
-
|
|
165
|
-
Updates the same line repeatedly to avoid cluttering logs with progress messages.
|
|
166
|
-
Falls back to periodic line updates if terminal doesn't support in-place updates.
|
|
167
|
-
"""
|
|
168
|
-
|
|
169
|
-
def __init__(
|
|
170
|
-
self,
|
|
171
|
-
description: str,
|
|
172
|
-
total: int | None = None,
|
|
173
|
-
update_interval: float = 0.5,
|
|
174
|
-
):
|
|
175
|
-
"""Initialize progress tracker.
|
|
176
|
-
|
|
177
|
-
Args:
|
|
178
|
-
description: Description of what's being tracked (e.g., "Processing photos")
|
|
179
|
-
total: Expected total count (if known). If None, shows count without percentage.
|
|
180
|
-
update_interval: Minimum seconds between display updates (default 0.5s)
|
|
181
|
-
"""
|
|
182
|
-
self.description = description
|
|
183
|
-
self.total = total
|
|
184
|
-
self.update_interval = update_interval
|
|
185
|
-
|
|
186
|
-
self.current = 0
|
|
187
|
-
self.last_update_time = 0.0
|
|
188
|
-
self.start_time = time.time()
|
|
189
|
-
self.last_displayed_message = ""
|
|
190
|
-
|
|
191
|
-
# Final metrics (set when finish() is called)
|
|
192
|
-
self.elapsed_seconds: float | None = None
|
|
193
|
-
self.final_rate: float | None = None # items per second
|
|
194
|
-
|
|
195
|
-
# Thread safety for concurrent access from UI polling
|
|
196
|
-
self._lock = threading.Lock()
|
|
197
|
-
|
|
198
|
-
# Always use carriage return for in-place updates
|
|
199
|
-
self.supports_inplace = True
|
|
200
|
-
|
|
201
|
-
def update(self, increment: int = 1) -> None:
|
|
202
|
-
"""Update progress by incrementing the counter.
|
|
203
|
-
|
|
204
|
-
Args:
|
|
205
|
-
increment: Amount to add to current count (default 1)
|
|
206
|
-
"""
|
|
207
|
-
with self._lock:
|
|
208
|
-
self.current += increment
|
|
209
|
-
now: float = time.time()
|
|
210
|
-
|
|
211
|
-
# Only update display if enough time has passed
|
|
212
|
-
if now - self.last_update_time >= self.update_interval:
|
|
213
|
-
self._display()
|
|
214
|
-
self.last_update_time = now
|
|
215
|
-
|
|
216
|
-
def set(self, value: int) -> None:
|
|
217
|
-
"""Set progress to a specific value.
|
|
218
|
-
|
|
219
|
-
Args:
|
|
220
|
-
value: Absolute count value
|
|
221
|
-
"""
|
|
222
|
-
with self._lock:
|
|
223
|
-
self.current = value
|
|
224
|
-
now: float = time.time()
|
|
225
|
-
|
|
226
|
-
if now - self.last_update_time >= self.update_interval:
|
|
227
|
-
self._display()
|
|
228
|
-
self.last_update_time = now
|
|
229
|
-
|
|
230
|
-
def set_status(self, status: str) -> None:
|
|
231
|
-
"""Update status message without changing count.
|
|
232
|
-
|
|
233
|
-
Useful for showing status during finalize/save operations.
|
|
234
|
-
|
|
235
|
-
Args:
|
|
236
|
-
status: New status message (e.g., "Finalizing results...", "Saving to cache...")
|
|
237
|
-
"""
|
|
238
|
-
with self._lock:
|
|
239
|
-
self.description = status
|
|
240
|
-
self._display()
|
|
241
|
-
self.last_update_time = time.time()
|
|
242
|
-
|
|
243
|
-
def finish(self, message: str | None = None) -> None:
|
|
244
|
-
"""Finish progress tracking and print final message.
|
|
245
|
-
|
|
246
|
-
Stores final elapsed time and throughput for later access.
|
|
247
|
-
|
|
248
|
-
Args:
|
|
249
|
-
message: Optional custom completion message. If None, uses default format.
|
|
250
|
-
"""
|
|
251
|
-
elapsed: float = time.time() - self.start_time
|
|
252
|
-
|
|
253
|
-
# Store final metrics for web UI
|
|
254
|
-
self.elapsed_seconds = elapsed
|
|
255
|
-
self.final_rate = self.current / elapsed if elapsed > 0 and self.current > 0 else None
|
|
256
|
-
|
|
257
|
-
if message is None:
|
|
258
|
-
elapsed_s: str = format_seconds_weighted(elapsed)
|
|
259
|
-
if self.total is not None:
|
|
260
|
-
message = f"{self.description}: {self.current}/{self.total} ({elapsed_s})"
|
|
261
|
-
else:
|
|
262
|
-
message = f"{self.description}: {self.current} ({elapsed_s})"
|
|
263
|
-
|
|
264
|
-
# Clear the line and print final message
|
|
265
|
-
if self.supports_inplace and self.last_displayed_message:
|
|
266
|
-
sys.stderr.write("\r" + " " * len(self.last_displayed_message) + "\r")
|
|
267
|
-
sys.stderr.write(message + "\n")
|
|
268
|
-
sys.stderr.flush()
|
|
269
|
-
self.last_displayed_message = ""
|
|
270
|
-
|
|
271
|
-
def _display(self) -> None:
|
|
272
|
-
"""Display current progress."""
|
|
273
|
-
elapsed: float = time.time() - self.start_time
|
|
274
|
-
|
|
275
|
-
# Build progress message
|
|
276
|
-
message: str
|
|
277
|
-
if self.total is not None:
|
|
278
|
-
percentage: float = (self.current / self.total * 100) if self.total > 0 else 0
|
|
279
|
-
rate: float = self.current / elapsed if elapsed > 0 else 0
|
|
280
|
-
eta: float = (self.total - self.current) / rate if rate > 0 else 0
|
|
281
|
-
|
|
282
|
-
message = (
|
|
283
|
-
f"{self.description}: {self.current}/{self.total} "
|
|
284
|
-
f"({percentage:.1f}%, {rate:.1f}/s, finishes {_format_completion_time(eta)})"
|
|
285
|
-
)
|
|
286
|
-
else:
|
|
287
|
-
rate_simple: float = self.current / elapsed if elapsed > 0 else 0
|
|
288
|
-
message = f"{self.description}: {self.current} ({rate_simple:.1f}/s)"
|
|
289
|
-
|
|
290
|
-
# Display with carriage return for in-place update, or newline if not supported
|
|
291
|
-
if self.supports_inplace:
|
|
292
|
-
# Pad with spaces to clear any previous longer message
|
|
293
|
-
if len(message) < len(self.last_displayed_message):
|
|
294
|
-
message = message + " " * (len(self.last_displayed_message) - len(message))
|
|
295
|
-
sys.stderr.write("\r" + message)
|
|
296
|
-
sys.stderr.flush()
|
|
297
|
-
else:
|
|
298
|
-
# No in-place support, print new line
|
|
299
|
-
sys.stderr.write(message + "\n")
|
|
300
|
-
sys.stderr.flush()
|
|
301
|
-
|
|
302
|
-
self.last_displayed_message = message
|
|
303
|
-
|
|
304
|
-
def get_snapshot(self) -> ProgressInfo:
|
|
305
|
-
"""Get formatted progress snapshot (for UI polling).
|
|
306
|
-
|
|
307
|
-
Thread-safe method that returns complete progress information
|
|
308
|
-
with all values pre-formatted for display.
|
|
309
|
-
|
|
310
|
-
Returns:
|
|
311
|
-
ProgressInfo with all fields formatted for UI display
|
|
312
|
-
"""
|
|
313
|
-
with self._lock:
|
|
314
|
-
# Calculate raw metrics
|
|
315
|
-
elapsed = time.time() - self.start_time
|
|
316
|
-
items_per_second = self.current / elapsed if elapsed > 0 else None
|
|
317
|
-
|
|
318
|
-
# Calculate ETA if we have total and rate
|
|
319
|
-
eta_seconds = None
|
|
320
|
-
if self.total and items_per_second and items_per_second > 0:
|
|
321
|
-
remaining = self.total - self.current
|
|
322
|
-
eta_seconds = remaining / items_per_second
|
|
323
|
-
|
|
324
|
-
# Calculate completion fraction
|
|
325
|
-
if self.total and self.total > 0:
|
|
326
|
-
fraction = min(1.0, self.current / self.total)
|
|
327
|
-
else:
|
|
328
|
-
fraction = 0.0
|
|
329
|
-
|
|
330
|
-
percentage = int(fraction * 100)
|
|
331
|
-
|
|
332
|
-
# Format everything for display (backend does all formatting)
|
|
333
|
-
return ProgressInfo(
|
|
334
|
-
# Raw values
|
|
335
|
-
fraction_complete=fraction,
|
|
336
|
-
current_count=self.current,
|
|
337
|
-
total_count=self.total,
|
|
338
|
-
rate=items_per_second,
|
|
339
|
-
eta_seconds=eta_seconds,
|
|
340
|
-
# Formatted strings
|
|
341
|
-
percentage_display=f"{percentage}%",
|
|
342
|
-
progress_bar_width=f"{percentage}%",
|
|
343
|
-
status_message=self.description,
|
|
344
|
-
items_display=self._format_items(self.current, self.total),
|
|
345
|
-
rate_display=self._format_rate(items_per_second),
|
|
346
|
-
eta_display=self._format_eta(eta_seconds),
|
|
347
|
-
stage_display=self.description.replace("_", " ").title(),
|
|
348
|
-
)
|
|
349
|
-
|
|
350
|
-
@staticmethod
|
|
351
|
-
def _format_items(current: int, total: int | None) -> str | None:
|
|
352
|
-
"""Format item counts: '11,123 / 15,234'.
|
|
353
|
-
|
|
354
|
-
Args:
|
|
355
|
-
current: Current item count
|
|
356
|
-
total: Total item count (or None if unknown)
|
|
357
|
-
|
|
358
|
-
Returns:
|
|
359
|
-
Formatted string or None if current is 0
|
|
360
|
-
"""
|
|
361
|
-
if current == 0:
|
|
362
|
-
return None
|
|
363
|
-
if total is None:
|
|
364
|
-
return f"{current:,} items processed"
|
|
365
|
-
return f"{current:,} / {total:,}"
|
|
366
|
-
|
|
367
|
-
@staticmethod
|
|
368
|
-
def _format_rate(items_per_second: float | None) -> str | None:
|
|
369
|
-
"""Format processing rate: '1,250 items/sec'.
|
|
370
|
-
|
|
371
|
-
Args:
|
|
372
|
-
items_per_second: Processing rate in items per second
|
|
373
|
-
|
|
374
|
-
Returns:
|
|
375
|
-
Formatted string or None if rate is too low/unknown
|
|
376
|
-
"""
|
|
377
|
-
if items_per_second is None or items_per_second < 0.1:
|
|
378
|
-
return None
|
|
379
|
-
return f"{items_per_second:,.0f} items/sec"
|
|
380
|
-
|
|
381
|
-
@staticmethod
|
|
382
|
-
def _format_eta(seconds: float | None) -> str | None:
|
|
383
|
-
"""Format estimated time remaining: '3 seconds', '2 minutes', '1h 23m'.
|
|
384
|
-
|
|
385
|
-
Args:
|
|
386
|
-
seconds: Estimated seconds remaining
|
|
387
|
-
|
|
388
|
-
Returns:
|
|
389
|
-
Formatted string or None if ETA unknown or less than 1 second
|
|
390
|
-
"""
|
|
391
|
-
if seconds is None or seconds < 1:
|
|
392
|
-
return None
|
|
393
|
-
|
|
394
|
-
if seconds < 60:
|
|
395
|
-
return f"{int(seconds)} second{'s' if int(seconds) != 1 else ''}"
|
|
396
|
-
if seconds < 3600:
|
|
397
|
-
minutes = int(seconds / 60)
|
|
398
|
-
return f"{minutes} minute{'s' if minutes != 1 else ''}"
|
|
399
|
-
hours = int(seconds / 3600)
|
|
400
|
-
minutes = int((seconds % 3600) / 60)
|
|
401
|
-
if minutes > 0:
|
|
402
|
-
return f"{hours}h {minutes}m"
|
|
403
|
-
return f"{hours} hour{'s' if hours != 1 else ''}"
|
|
404
|
-
|
|
405
|
-
def __enter__(self) -> ProgressTracker:
|
|
406
|
-
"""Context manager entry."""
|
|
407
|
-
return self
|
|
408
|
-
|
|
409
|
-
def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: Any) -> None:
|
|
410
|
-
"""Context manager exit - finish progress tracking."""
|
|
411
|
-
if exc_type is None:
|
|
412
|
-
self.finish()
|
|
413
|
-
else:
|
|
414
|
-
# Error occurred, still clean up display
|
|
415
|
-
self.finish(f"{self.description}: Failed at {self.current}")
|
|
1
|
+
"""Progress tracking with in-place updates for cleaner logs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import datetime, timedelta
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class ProgressInfo:
|
|
15
|
+
"""Complete progress information, pre-formatted for UI display.
|
|
16
|
+
|
|
17
|
+
All formatting happens in the backend - frontend just displays values.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# Raw values (for programmatic use if needed)
|
|
21
|
+
fraction_complete: float # 0.0 to 1.0
|
|
22
|
+
current_count: int # Current item count
|
|
23
|
+
total_count: int | None # Total item count (None if unknown)
|
|
24
|
+
rate: float | None # Items per second
|
|
25
|
+
eta_seconds: float | None # Estimated seconds remaining
|
|
26
|
+
|
|
27
|
+
# Formatted strings (ready for direct display in HTML)
|
|
28
|
+
percentage_display: str # "73%"
|
|
29
|
+
progress_bar_width: str # "73%" (CSS width property)
|
|
30
|
+
status_message: str # "Processing photos"
|
|
31
|
+
items_display: str | None # "11,123 / 15,234" or None
|
|
32
|
+
rate_display: str | None # "1,250 items/sec" or None
|
|
33
|
+
eta_display: str | None # "3 seconds" or "2 minutes" or "1h 23m"
|
|
34
|
+
stage_display: str # "Compute Identical"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _format_time(seconds: float) -> str:
|
|
38
|
+
"""Format time in seconds to human-readable format.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
seconds: Time in seconds
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
Formatted string like "2d3h45s" or "1h30s" or "45s"
|
|
45
|
+
"""
|
|
46
|
+
if seconds < 60:
|
|
47
|
+
return f"{seconds:.0f}s"
|
|
48
|
+
|
|
49
|
+
days: int = int(seconds // 86400)
|
|
50
|
+
remaining: float = seconds % 86400
|
|
51
|
+
hours: int = int(remaining // 3600)
|
|
52
|
+
remaining = remaining % 3600
|
|
53
|
+
minutes: int = int(remaining // 60)
|
|
54
|
+
secs: int = int(remaining % 60)
|
|
55
|
+
|
|
56
|
+
parts: list[str] = []
|
|
57
|
+
if days > 0:
|
|
58
|
+
parts.append(f"{days}d")
|
|
59
|
+
if hours > 0:
|
|
60
|
+
parts.append(f"{hours}h")
|
|
61
|
+
if minutes > 0:
|
|
62
|
+
parts.append(f"{minutes}m")
|
|
63
|
+
if secs > 0 or not parts: # Always show seconds if nothing else to show
|
|
64
|
+
parts.append(f"{secs}s")
|
|
65
|
+
|
|
66
|
+
return "".join(parts)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _format_completion_time(eta_seconds: float) -> str:
|
|
70
|
+
"""Format estimated completion time as a timestamp.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
eta_seconds: Estimated seconds remaining until completion
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
Formatted string like "14:35:20" for same-day or "tomorrow 09:30" for next day
|
|
77
|
+
"""
|
|
78
|
+
completion: datetime = datetime.now() + timedelta(seconds=eta_seconds)
|
|
79
|
+
now: datetime = datetime.now()
|
|
80
|
+
|
|
81
|
+
# Check if completion is tomorrow or later
|
|
82
|
+
if completion.date() > now.date():
|
|
83
|
+
days_diff: int = (completion.date() - now.date()).days
|
|
84
|
+
if days_diff == 1:
|
|
85
|
+
return f"tomorrow {completion.strftime('%H:%M')}"
|
|
86
|
+
return f"in {days_diff} days at {completion.strftime('%H:%M')}"
|
|
87
|
+
|
|
88
|
+
# Same day - just show time
|
|
89
|
+
return completion.strftime("%H:%M:%S")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def format_seconds_weighted(total_seconds: float) -> str:
|
|
93
|
+
"""Formats a number of seconds into DDDdHHhMMmSS.ffs.
|
|
94
|
+
|
|
95
|
+
Omits leading zero-value fields.
|
|
96
|
+
"""
|
|
97
|
+
if total_seconds < 0:
|
|
98
|
+
return "-" + format_seconds_weighted(-total_seconds)
|
|
99
|
+
|
|
100
|
+
# 1. Separate whole seconds and fractional seconds
|
|
101
|
+
whole_seconds = int(total_seconds)
|
|
102
|
+
fractional_seconds = total_seconds - whole_seconds
|
|
103
|
+
|
|
104
|
+
# 2. Convert whole seconds to Days, Hours, Minutes, Seconds
|
|
105
|
+
days = whole_seconds // (24 * 3600)
|
|
106
|
+
whole_seconds %= 24 * 3600
|
|
107
|
+
|
|
108
|
+
hours = whole_seconds // 3600
|
|
109
|
+
whole_seconds %= 3600
|
|
110
|
+
|
|
111
|
+
minutes = whole_seconds // 60
|
|
112
|
+
seconds = whole_seconds % 60
|
|
113
|
+
|
|
114
|
+
# 3. Build the components list (Days, Hours, Minutes, Seconds)
|
|
115
|
+
components = []
|
|
116
|
+
|
|
117
|
+
if days > 0:
|
|
118
|
+
components.append(f"{days}d")
|
|
119
|
+
|
|
120
|
+
if hours > 0 or (days > 0 and (minutes > 0 or seconds > 0 or fractional_seconds > 0)):
|
|
121
|
+
# Include hours if non-zero, OR if days are present and there's any time after hours.
|
|
122
|
+
components.append(f"{hours}h")
|
|
123
|
+
|
|
124
|
+
if minutes > 0 or (hours > 0 and (seconds > 0 or fractional_seconds > 0)):
|
|
125
|
+
# Include minutes if non-zero, OR if hours are present and there's any time after minutes.
|
|
126
|
+
components.append(f"{minutes}m")
|
|
127
|
+
|
|
128
|
+
# 4. Handle the Seconds and Fractional part
|
|
129
|
+
|
|
130
|
+
# The seconds and fractional part must always be included,
|
|
131
|
+
# unless the entire duration is 0.0.
|
|
132
|
+
if not components and seconds == 0 and fractional_seconds == 0:
|
|
133
|
+
return "0s" # Special case for exact zero
|
|
134
|
+
|
|
135
|
+
# The Seconds part is built using the combined whole and fractional seconds
|
|
136
|
+
# which ensures the correct two decimal places.
|
|
137
|
+
# Note: We must use the 'seconds' variable and add the fractional part back
|
|
138
|
+
# to avoid issues with floating point arithmetic accumulating large errors
|
|
139
|
+
# if we simply calculated total_seconds % 60.
|
|
140
|
+
|
|
141
|
+
formatted_seconds = seconds + fractional_seconds
|
|
142
|
+
|
|
143
|
+
# Check if a higher-order component (d, h, m) was printed.
|
|
144
|
+
# If not, the seconds field is the leading field and should not be zero-padded.
|
|
145
|
+
# The format is S.ffs (no padding) or SS.ffs (padding if it follows another field)
|
|
146
|
+
|
|
147
|
+
if components:
|
|
148
|
+
# Pad with 0 to ensure two digits if it follows an hour or minute field
|
|
149
|
+
# e.g., '1h05.12s', not '1h5.12s'
|
|
150
|
+
components.append(f"{formatted_seconds:05.2f}s")
|
|
151
|
+
else:
|
|
152
|
+
# This is the leading field, so no mandatory padding on the integer part
|
|
153
|
+
# The formatting will be "S.ffs" or "SS.ffs"
|
|
154
|
+
components.append(f"{formatted_seconds:.2f}s")
|
|
155
|
+
|
|
156
|
+
# 5. Combine and return
|
|
157
|
+
return "".join(components).replace(
|
|
158
|
+
":", ""
|
|
159
|
+
) # Remove potential colons from hour/minute formatting if using a different approach
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class ProgressTracker:
|
|
163
|
+
"""Track progress with in-place console updates.
|
|
164
|
+
|
|
165
|
+
Updates the same line repeatedly to avoid cluttering logs with progress messages.
|
|
166
|
+
Falls back to periodic line updates if terminal doesn't support in-place updates.
|
|
167
|
+
"""
|
|
168
|
+
|
|
169
|
+
def __init__(
|
|
170
|
+
self,
|
|
171
|
+
description: str,
|
|
172
|
+
total: int | None = None,
|
|
173
|
+
update_interval: float = 0.5,
|
|
174
|
+
):
|
|
175
|
+
"""Initialize progress tracker.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
description: Description of what's being tracked (e.g., "Processing photos")
|
|
179
|
+
total: Expected total count (if known). If None, shows count without percentage.
|
|
180
|
+
update_interval: Minimum seconds between display updates (default 0.5s)
|
|
181
|
+
"""
|
|
182
|
+
self.description = description
|
|
183
|
+
self.total = total
|
|
184
|
+
self.update_interval = update_interval
|
|
185
|
+
|
|
186
|
+
self.current = 0
|
|
187
|
+
self.last_update_time = 0.0
|
|
188
|
+
self.start_time = time.time()
|
|
189
|
+
self.last_displayed_message = ""
|
|
190
|
+
|
|
191
|
+
# Final metrics (set when finish() is called)
|
|
192
|
+
self.elapsed_seconds: float | None = None
|
|
193
|
+
self.final_rate: float | None = None # items per second
|
|
194
|
+
|
|
195
|
+
# Thread safety for concurrent access from UI polling
|
|
196
|
+
self._lock = threading.Lock()
|
|
197
|
+
|
|
198
|
+
# Always use carriage return for in-place updates
|
|
199
|
+
self.supports_inplace = True
|
|
200
|
+
|
|
201
|
+
def update(self, increment: int = 1) -> None:
|
|
202
|
+
"""Update progress by incrementing the counter.
|
|
203
|
+
|
|
204
|
+
Args:
|
|
205
|
+
increment: Amount to add to current count (default 1)
|
|
206
|
+
"""
|
|
207
|
+
with self._lock:
|
|
208
|
+
self.current += increment
|
|
209
|
+
now: float = time.time()
|
|
210
|
+
|
|
211
|
+
# Only update display if enough time has passed
|
|
212
|
+
if now - self.last_update_time >= self.update_interval:
|
|
213
|
+
self._display()
|
|
214
|
+
self.last_update_time = now
|
|
215
|
+
|
|
216
|
+
def set(self, value: int) -> None:
|
|
217
|
+
"""Set progress to a specific value.
|
|
218
|
+
|
|
219
|
+
Args:
|
|
220
|
+
value: Absolute count value
|
|
221
|
+
"""
|
|
222
|
+
with self._lock:
|
|
223
|
+
self.current = value
|
|
224
|
+
now: float = time.time()
|
|
225
|
+
|
|
226
|
+
if now - self.last_update_time >= self.update_interval:
|
|
227
|
+
self._display()
|
|
228
|
+
self.last_update_time = now
|
|
229
|
+
|
|
230
|
+
def set_status(self, status: str) -> None:
|
|
231
|
+
"""Update status message without changing count.
|
|
232
|
+
|
|
233
|
+
Useful for showing status during finalize/save operations.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
status: New status message (e.g., "Finalizing results...", "Saving to cache...")
|
|
237
|
+
"""
|
|
238
|
+
with self._lock:
|
|
239
|
+
self.description = status
|
|
240
|
+
self._display()
|
|
241
|
+
self.last_update_time = time.time()
|
|
242
|
+
|
|
243
|
+
def finish(self, message: str | None = None) -> None:
|
|
244
|
+
"""Finish progress tracking and print final message.
|
|
245
|
+
|
|
246
|
+
Stores final elapsed time and throughput for later access.
|
|
247
|
+
|
|
248
|
+
Args:
|
|
249
|
+
message: Optional custom completion message. If None, uses default format.
|
|
250
|
+
"""
|
|
251
|
+
elapsed: float = time.time() - self.start_time
|
|
252
|
+
|
|
253
|
+
# Store final metrics for web UI
|
|
254
|
+
self.elapsed_seconds = elapsed
|
|
255
|
+
self.final_rate = self.current / elapsed if elapsed > 0 and self.current > 0 else None
|
|
256
|
+
|
|
257
|
+
if message is None:
|
|
258
|
+
elapsed_s: str = format_seconds_weighted(elapsed)
|
|
259
|
+
if self.total is not None:
|
|
260
|
+
message = f"{self.description}: {self.current}/{self.total} ({elapsed_s})"
|
|
261
|
+
else:
|
|
262
|
+
message = f"{self.description}: {self.current} ({elapsed_s})"
|
|
263
|
+
|
|
264
|
+
# Clear the line and print final message
|
|
265
|
+
if self.supports_inplace and self.last_displayed_message:
|
|
266
|
+
sys.stderr.write("\r" + " " * len(self.last_displayed_message) + "\r")
|
|
267
|
+
sys.stderr.write(message + "\n")
|
|
268
|
+
sys.stderr.flush()
|
|
269
|
+
self.last_displayed_message = ""
|
|
270
|
+
|
|
271
|
+
def _display(self) -> None:
|
|
272
|
+
"""Display current progress."""
|
|
273
|
+
elapsed: float = time.time() - self.start_time
|
|
274
|
+
|
|
275
|
+
# Build progress message
|
|
276
|
+
message: str
|
|
277
|
+
if self.total is not None:
|
|
278
|
+
percentage: float = (self.current / self.total * 100) if self.total > 0 else 0
|
|
279
|
+
rate: float = self.current / elapsed if elapsed > 0 else 0
|
|
280
|
+
eta: float = (self.total - self.current) / rate if rate > 0 else 0
|
|
281
|
+
|
|
282
|
+
message = (
|
|
283
|
+
f"{self.description}: {self.current}/{self.total} "
|
|
284
|
+
f"({percentage:.1f}%, {rate:.1f}/s, finishes {_format_completion_time(eta)})"
|
|
285
|
+
)
|
|
286
|
+
else:
|
|
287
|
+
rate_simple: float = self.current / elapsed if elapsed > 0 else 0
|
|
288
|
+
message = f"{self.description}: {self.current} ({rate_simple:.1f}/s)"
|
|
289
|
+
|
|
290
|
+
# Display with carriage return for in-place update, or newline if not supported
|
|
291
|
+
if self.supports_inplace:
|
|
292
|
+
# Pad with spaces to clear any previous longer message
|
|
293
|
+
if len(message) < len(self.last_displayed_message):
|
|
294
|
+
message = message + " " * (len(self.last_displayed_message) - len(message))
|
|
295
|
+
sys.stderr.write("\r" + message)
|
|
296
|
+
sys.stderr.flush()
|
|
297
|
+
else:
|
|
298
|
+
# No in-place support, print new line
|
|
299
|
+
sys.stderr.write(message + "\n")
|
|
300
|
+
sys.stderr.flush()
|
|
301
|
+
|
|
302
|
+
self.last_displayed_message = message
|
|
303
|
+
|
|
304
|
+
def get_snapshot(self) -> ProgressInfo:
|
|
305
|
+
"""Get formatted progress snapshot (for UI polling).
|
|
306
|
+
|
|
307
|
+
Thread-safe method that returns complete progress information
|
|
308
|
+
with all values pre-formatted for display.
|
|
309
|
+
|
|
310
|
+
Returns:
|
|
311
|
+
ProgressInfo with all fields formatted for UI display
|
|
312
|
+
"""
|
|
313
|
+
with self._lock:
|
|
314
|
+
# Calculate raw metrics
|
|
315
|
+
elapsed = time.time() - self.start_time
|
|
316
|
+
items_per_second = self.current / elapsed if elapsed > 0 else None
|
|
317
|
+
|
|
318
|
+
# Calculate ETA if we have total and rate
|
|
319
|
+
eta_seconds = None
|
|
320
|
+
if self.total and items_per_second and items_per_second > 0:
|
|
321
|
+
remaining = self.total - self.current
|
|
322
|
+
eta_seconds = remaining / items_per_second
|
|
323
|
+
|
|
324
|
+
# Calculate completion fraction
|
|
325
|
+
if self.total and self.total > 0:
|
|
326
|
+
fraction = min(1.0, self.current / self.total)
|
|
327
|
+
else:
|
|
328
|
+
fraction = 0.0
|
|
329
|
+
|
|
330
|
+
percentage = int(fraction * 100)
|
|
331
|
+
|
|
332
|
+
# Format everything for display (backend does all formatting)
|
|
333
|
+
return ProgressInfo(
|
|
334
|
+
# Raw values
|
|
335
|
+
fraction_complete=fraction,
|
|
336
|
+
current_count=self.current,
|
|
337
|
+
total_count=self.total,
|
|
338
|
+
rate=items_per_second,
|
|
339
|
+
eta_seconds=eta_seconds,
|
|
340
|
+
# Formatted strings
|
|
341
|
+
percentage_display=f"{percentage}%",
|
|
342
|
+
progress_bar_width=f"{percentage}%",
|
|
343
|
+
status_message=self.description,
|
|
344
|
+
items_display=self._format_items(self.current, self.total),
|
|
345
|
+
rate_display=self._format_rate(items_per_second),
|
|
346
|
+
eta_display=self._format_eta(eta_seconds),
|
|
347
|
+
stage_display=self.description.replace("_", " ").title(),
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
@staticmethod
|
|
351
|
+
def _format_items(current: int, total: int | None) -> str | None:
|
|
352
|
+
"""Format item counts: '11,123 / 15,234'.
|
|
353
|
+
|
|
354
|
+
Args:
|
|
355
|
+
current: Current item count
|
|
356
|
+
total: Total item count (or None if unknown)
|
|
357
|
+
|
|
358
|
+
Returns:
|
|
359
|
+
Formatted string or None if current is 0
|
|
360
|
+
"""
|
|
361
|
+
if current == 0:
|
|
362
|
+
return None
|
|
363
|
+
if total is None:
|
|
364
|
+
return f"{current:,} items processed"
|
|
365
|
+
return f"{current:,} / {total:,}"
|
|
366
|
+
|
|
367
|
+
@staticmethod
|
|
368
|
+
def _format_rate(items_per_second: float | None) -> str | None:
|
|
369
|
+
"""Format processing rate: '1,250 items/sec'.
|
|
370
|
+
|
|
371
|
+
Args:
|
|
372
|
+
items_per_second: Processing rate in items per second
|
|
373
|
+
|
|
374
|
+
Returns:
|
|
375
|
+
Formatted string or None if rate is too low/unknown
|
|
376
|
+
"""
|
|
377
|
+
if items_per_second is None or items_per_second < 0.1:
|
|
378
|
+
return None
|
|
379
|
+
return f"{items_per_second:,.0f} items/sec"
|
|
380
|
+
|
|
381
|
+
@staticmethod
|
|
382
|
+
def _format_eta(seconds: float | None) -> str | None:
|
|
383
|
+
"""Format estimated time remaining: '3 seconds', '2 minutes', '1h 23m'.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
seconds: Estimated seconds remaining
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
Formatted string or None if ETA unknown or less than 1 second
|
|
390
|
+
"""
|
|
391
|
+
if seconds is None or seconds < 1:
|
|
392
|
+
return None
|
|
393
|
+
|
|
394
|
+
if seconds < 60:
|
|
395
|
+
return f"{int(seconds)} second{'s' if int(seconds) != 1 else ''}"
|
|
396
|
+
if seconds < 3600:
|
|
397
|
+
minutes = int(seconds / 60)
|
|
398
|
+
return f"{minutes} minute{'s' if minutes != 1 else ''}"
|
|
399
|
+
hours = int(seconds / 3600)
|
|
400
|
+
minutes = int((seconds % 3600) / 60)
|
|
401
|
+
if minutes > 0:
|
|
402
|
+
return f"{hours}h {minutes}m"
|
|
403
|
+
return f"{hours} hour{'s' if hours != 1 else ''}"
|
|
404
|
+
|
|
405
|
+
def __enter__(self) -> ProgressTracker:
|
|
406
|
+
"""Context manager entry."""
|
|
407
|
+
return self
|
|
408
|
+
|
|
409
|
+
def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: Any) -> None:
|
|
410
|
+
"""Context manager exit - finish progress tracking."""
|
|
411
|
+
if exc_type is None:
|
|
412
|
+
self.finish()
|
|
413
|
+
else:
|
|
414
|
+
# Error occurred, still clean up display
|
|
415
|
+
self.finish(f"{self.description}: Failed at {self.current}")
|