polytext 0.2.8b7__tar.gz → 0.2.8b8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {polytext-0.2.8b7 → polytext-0.2.8b8}/PKG-INFO +1 -1
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/audio_to_text.py +20 -22
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/document_ocr_to_text.py +15 -15
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/document_ocr_to_text_azure_oai.py +12 -12
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/ocr_to_text.py +13 -16
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/ocr_to_text_azure_oai.py +12 -12
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/pdf.py +3 -3
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/audio.py +6 -6
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/base.py +55 -5
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/document.py +17 -19
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/document_ocr.py +49 -10
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/downloader/downloader.py +4 -4
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/markdown.py +9 -9
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/notebook.py +7 -7
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/ocr.py +6 -6
- polytext-0.2.8b8/polytext/loader/ocr_cache.py +71 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/video.py +8 -8
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/audio_chunker.py +11 -11
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/text_merger.py +5 -5
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/PKG-INFO +1 -1
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/SOURCES.txt +3 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/setup.py +1 -1
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_base_loader_error_mapping.py +13 -0
- polytext-0.2.8b8/tests/test_document_ocr_cache.py +443 -0
- polytext-0.2.8b8/tests/test_ocr_cache.py +125 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/LICENSE +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/README.md +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/base.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/beautiful_text.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/gemini_quality_guards.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/html_to_md.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/md_to_text.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/text_to_md.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/video_to_audio.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/exceptions/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/exceptions/base.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/generator/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/generator/pdf.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/aws_auth.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/downloader/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/html.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/plain_text.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/xml_xbrl.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/youtube.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/youtube_llm.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/transcript_chunker.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/beautiful_text.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/ocr.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/text_merging.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/text_to_md.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/transcription.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/utils/__init__.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/utils/utils.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/dependency_links.txt +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/not-zip-safe +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/requires.txt +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/top_level.txt +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/pyproject.toml +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/setup.cfg +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_chunk_quality.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_chunker.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_comparison_helpers.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_transcription_model_migration.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_aws_auth.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_beautiful_text_manual.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_audio_models.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_document_ocr_to_text_models.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_ocr_to_text_models.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_youtube_models.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_dowload_audio_from_youtube.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_dowload_audio_from_youtube_helpers.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_extracted_text_whitespace.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_gemini_quality_guards.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_audio_transcript_from_gcs.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_customized_pdf_from_markdown.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_ocr.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_ocr_azure_oai.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_text.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_text_from_gcs.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_ocr_from_image.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_text_from_markdown.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_video_transcript_from_gcs.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_library.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_markdown_loader_gzip.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_markitdown_html.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_notebook_loader.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_ocr_fallbacks.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_ocr_image_descriptions.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_pain_text.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_pdf_conversion_error.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_python_version_metadata.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_split_audio_with_llm.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_transcribe_s3_images_from_csv.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_transcribe_s3_images_from_csv_script.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_xml_xbrl_loader.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_youtube_gemini_minimal_check.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_youtube_llm_fallbacks.py +0 -0
- {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_youtube_transcript.py +0 -0
|
@@ -176,7 +176,7 @@ def find_suspicious_fallback_language_indices(
|
|
|
176
176
|
) -> list[int]:
|
|
177
177
|
"""Find internal fallback chunks whose language conflicts with two agreeing neighbours."""
|
|
178
178
|
suspicious_indices = []
|
|
179
|
-
logger.
|
|
179
|
+
logger.debug("Starting fallback language validation for %s transcript chunks", len(transcripts))
|
|
180
180
|
|
|
181
181
|
def has_fallback(result: dict | None) -> bool:
|
|
182
182
|
if not result:
|
|
@@ -402,7 +402,7 @@ def compress_and_convert_audio(input_path: str, bitrate_quality: int = 9) -> str
|
|
|
402
402
|
os.unlink(temp_audio_path)
|
|
403
403
|
raise
|
|
404
404
|
|
|
405
|
-
logger.
|
|
405
|
+
logger.debug("Audio normalization completed")
|
|
406
406
|
return temp_audio_path
|
|
407
407
|
|
|
408
408
|
|
|
@@ -721,7 +721,7 @@ class AudioToTextConverter:
|
|
|
721
721
|
config: types.GenerateContentConfig,
|
|
722
722
|
):
|
|
723
723
|
file_size = os.path.getsize(audio_file)
|
|
724
|
-
logger.
|
|
724
|
+
logger.debug("Audio file size: %.2f MB", file_size / (1024 * 1024))
|
|
725
725
|
marker_nonce = uuid.uuid4().hex[:12]
|
|
726
726
|
marker_start = f"<<<UNTRUSTED_AUDIO_START_{marker_nonce}>>>"
|
|
727
727
|
marker_end = f"<<<UNTRUSTED_AUDIO_END_{marker_nonce}>>>"
|
|
@@ -736,9 +736,9 @@ class AudioToTextConverter:
|
|
|
736
736
|
model=self.transcription_model,
|
|
737
737
|
contents=[my_file]
|
|
738
738
|
)
|
|
739
|
-
logger.
|
|
739
|
+
logger.debug("Audio file token count response received")
|
|
740
740
|
|
|
741
|
-
logger.
|
|
741
|
+
logger.debug("Audio upload completed; starting transcription")
|
|
742
742
|
|
|
743
743
|
return client.models.generate_content(
|
|
744
744
|
model=self.transcription_model,
|
|
@@ -750,7 +750,7 @@ class AudioToTextConverter:
|
|
|
750
750
|
if temp_upload_path and os.path.exists(temp_upload_path):
|
|
751
751
|
os.remove(temp_upload_path)
|
|
752
752
|
|
|
753
|
-
logger.
|
|
753
|
+
logger.debug("Audio can be sent inline")
|
|
754
754
|
with open(audio_file, "rb") as f:
|
|
755
755
|
audio_data = f.read()
|
|
756
756
|
|
|
@@ -853,8 +853,8 @@ class AudioToTextConverter:
|
|
|
853
853
|
completion_tokens = getattr(usage_metadata, "candidates_token_count", 0) or 0
|
|
854
854
|
prompt_tokens = getattr(usage_metadata, "prompt_token_count", 0) or 0
|
|
855
855
|
|
|
856
|
-
logger.
|
|
857
|
-
logger.
|
|
856
|
+
logger.debug("Audio completion tokens: %s", completion_tokens)
|
|
857
|
+
logger.debug("Audio prompt tokens: %s", prompt_tokens)
|
|
858
858
|
|
|
859
859
|
if finish_reason and "RECITATION" in finish_reason:
|
|
860
860
|
raise EmptyDocument(
|
|
@@ -1005,7 +1005,7 @@ class AudioToTextConverter:
|
|
|
1005
1005
|
|
|
1006
1006
|
def process_chunk(self, chunk: dict, index: int) -> tuple[int, dict]:
|
|
1007
1007
|
"""Process a single audio chunk and return its transcript"""
|
|
1008
|
-
logger.
|
|
1008
|
+
logger.debug("Transcribing audio chunk %s", index + 1)
|
|
1009
1009
|
transcript_dict = self.transcribe_audio(chunk["file_path"])
|
|
1010
1010
|
return index, transcript_dict
|
|
1011
1011
|
|
|
@@ -1071,12 +1071,12 @@ class AudioToTextConverter:
|
|
|
1071
1071
|
RuntimeError: If there's an error during audio processing or transcription
|
|
1072
1072
|
"""
|
|
1073
1073
|
processed_audio_path = None
|
|
1074
|
-
logger.
|
|
1074
|
+
logger.debug("Processing audio file")
|
|
1075
1075
|
file_size = os.path.getsize(audio_path)
|
|
1076
|
-
logger.
|
|
1076
|
+
logger.debug("Audio file size: %.2f MB", file_size / (1024 * 1024))
|
|
1077
1077
|
|
|
1078
1078
|
mime_type, _ = mimetypes.guess_type(audio_path)
|
|
1079
|
-
logger.
|
|
1079
|
+
logger.debug("Original audio MIME type: %s", mime_type)
|
|
1080
1080
|
|
|
1081
1081
|
# Check if conversion and/or compression is needed
|
|
1082
1082
|
needs_conversion = mime_type not in SUPPORTED_MIME_TYPES
|
|
@@ -1084,29 +1084,29 @@ class AudioToTextConverter:
|
|
|
1084
1084
|
|
|
1085
1085
|
# If you need at least one of the two, apply compress_and_convert_audio
|
|
1086
1086
|
if needs_conversion: # or needs_compression:
|
|
1087
|
-
logger.
|
|
1087
|
+
logger.debug("Audio requires conversion")
|
|
1088
1088
|
processed_audio_path = compress_and_convert_audio(audio_path)
|
|
1089
1089
|
used_file = processed_audio_path
|
|
1090
|
-
logger.
|
|
1090
|
+
logger.debug("Audio conversion completed")
|
|
1091
1091
|
else:
|
|
1092
1092
|
used_file = audio_path
|
|
1093
|
-
logger.
|
|
1093
|
+
logger.debug("Audio is already in a supported format")
|
|
1094
1094
|
# If you need at least one of the two, apply compress_and_convert_audio
|
|
1095
1095
|
if needs_conversion: # or needs_compression:
|
|
1096
|
-
logger.
|
|
1096
|
+
logger.debug("Audio requires conversion")
|
|
1097
1097
|
processed_audio_path = compress_and_convert_audio(input_path=audio_path,
|
|
1098
1098
|
bitrate_quality=self.bitrate_quality)
|
|
1099
1099
|
used_file = processed_audio_path
|
|
1100
|
-
logger.
|
|
1100
|
+
logger.debug("Audio conversion completed")
|
|
1101
1101
|
else:
|
|
1102
1102
|
used_file = audio_path
|
|
1103
|
-
logger.
|
|
1103
|
+
logger.debug("Audio is already in a supported format")
|
|
1104
1104
|
|
|
1105
1105
|
# Create chunker and extract chunks
|
|
1106
|
-
logger.
|
|
1106
|
+
logger.debug("Creating audio chunker")
|
|
1107
1107
|
chunker = AudioChunker(used_file, max_llm_tokens=self.max_llm_tokens)
|
|
1108
1108
|
self.set_long_audio_protections(chunker.duration_ms)
|
|
1109
|
-
logger.
|
|
1109
|
+
logger.debug(
|
|
1110
1110
|
"Long-audio transcription protections enabled: %s (duration: %sms, threshold: %sms)",
|
|
1111
1111
|
self.long_audio_protections_enabled,
|
|
1112
1112
|
chunker.duration_ms,
|
|
@@ -1114,8 +1114,6 @@ class AudioToTextConverter:
|
|
|
1114
1114
|
)
|
|
1115
1115
|
chunks = chunker.extract_chunks()
|
|
1116
1116
|
|
|
1117
|
-
logger.info(f"chunks: {chunks}")
|
|
1118
|
-
|
|
1119
1117
|
logger.info(f"Split audio into {len(chunks)} chunks")
|
|
1120
1118
|
if len(chunks) > 1 and self.markdown_output:
|
|
1121
1119
|
logger.info("Audio chunking is needed, returning minimal markdown output")
|
|
@@ -97,7 +97,7 @@ def compress_and_convert_image(input_path: str, target_size=1):
|
|
|
97
97
|
loglevel='error'
|
|
98
98
|
).run(quiet=True, overwrite_output=True)
|
|
99
99
|
|
|
100
|
-
logger.
|
|
100
|
+
logger.debug("Successfully processed image")
|
|
101
101
|
return temp_image_path
|
|
102
102
|
|
|
103
103
|
except Exception as e:
|
|
@@ -363,17 +363,17 @@ class DocumentOCRToTextConverter:
|
|
|
363
363
|
start_time = time.time()
|
|
364
364
|
|
|
365
365
|
if self.markdown_output:
|
|
366
|
-
logger.
|
|
366
|
+
logger.debug("Using prompt for markdown format")
|
|
367
367
|
else:
|
|
368
|
-
logger.
|
|
368
|
+
logger.debug("Using prompt for plain text format")
|
|
369
369
|
prompt_template = self._build_prompt_template()
|
|
370
370
|
|
|
371
371
|
try:
|
|
372
372
|
if self.llm_api_key:
|
|
373
|
-
logger.
|
|
373
|
+
logger.debug("Using provided Google API key")
|
|
374
374
|
client = genai.Client(api_key=self.llm_api_key)
|
|
375
375
|
else:
|
|
376
|
-
logger.
|
|
376
|
+
logger.debug("Using Google API key from environment")
|
|
377
377
|
client = genai.Client()
|
|
378
378
|
|
|
379
379
|
config = types.GenerateContentConfig(
|
|
@@ -407,26 +407,26 @@ class DocumentOCRToTextConverter:
|
|
|
407
407
|
)
|
|
408
408
|
|
|
409
409
|
mime_type, _ = mimetypes.guess_type(file_for_ocr)
|
|
410
|
-
logger.
|
|
410
|
+
logger.debug("OCR MIME type: %s", mime_type)
|
|
411
411
|
file_size = os.path.getsize(file_for_ocr)
|
|
412
|
-
logger.
|
|
412
|
+
logger.debug("Initial OCR image size: %s bytes", file_size)
|
|
413
413
|
if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
|
|
414
|
-
logger.
|
|
414
|
+
logger.debug("OCR image requires compression or conversion")
|
|
415
415
|
temp_file_for_ocr = compress_and_convert_image(input_path=file_for_ocr, target_size=self.target_size)
|
|
416
416
|
should_delete_temp_file = True
|
|
417
417
|
file_size = os.path.getsize(temp_file_for_ocr)
|
|
418
418
|
else:
|
|
419
|
-
logger.
|
|
419
|
+
logger.debug("OCR image requires no compression or conversion")
|
|
420
420
|
temp_file_for_ocr = file_for_ocr
|
|
421
421
|
|
|
422
|
-
logger.
|
|
422
|
+
logger.debug("Final OCR image size: %.2f MB", file_size / (1024 * 1024))
|
|
423
423
|
|
|
424
424
|
if file_size > 20 * 1024 * 1024:
|
|
425
|
-
logger.
|
|
425
|
+
logger.debug("Uploading large OCR image")
|
|
426
426
|
|
|
427
427
|
myfile = client.files.upload(file=temp_file_for_ocr)
|
|
428
428
|
|
|
429
|
-
logger.
|
|
429
|
+
logger.debug("OCR image upload completed")
|
|
430
430
|
|
|
431
431
|
contents = [myfile]
|
|
432
432
|
response = client.models.generate_content(
|
|
@@ -438,7 +438,7 @@ class DocumentOCRToTextConverter:
|
|
|
438
438
|
client.files.delete(name=myfile.name)
|
|
439
439
|
|
|
440
440
|
else:
|
|
441
|
-
logger.
|
|
441
|
+
logger.debug("OCR image can be sent inline")
|
|
442
442
|
with open(temp_file_for_ocr, "rb") as f:
|
|
443
443
|
image_data = f.read()
|
|
444
444
|
|
|
@@ -472,8 +472,8 @@ class DocumentOCRToTextConverter:
|
|
|
472
472
|
threshold=OCR_TAIL_REPETITION_THRESHOLD,
|
|
473
473
|
)
|
|
474
474
|
|
|
475
|
-
logger.
|
|
476
|
-
logger.
|
|
475
|
+
logger.debug("OCR completion tokens: %s", response.usage_metadata.candidates_token_count)
|
|
476
|
+
logger.debug("OCR prompt tokens: %s", response.usage_metadata.prompt_token_count)
|
|
477
477
|
|
|
478
478
|
if finish_reason and "RECITATION" in finish_reason:
|
|
479
479
|
raise EmptyDocument(
|
|
@@ -53,13 +53,13 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
|
|
|
53
53
|
os.close(fd)
|
|
54
54
|
|
|
55
55
|
original_size = os.path.getsize(input_path)
|
|
56
|
-
logger.
|
|
56
|
+
logger.debug("Original image size: %.2fMB", original_size / 1024 / 1024)
|
|
57
57
|
|
|
58
58
|
if original_size > target_size_bytes:
|
|
59
59
|
compression_ratio = (target_size_bytes / original_size) ** 0.5
|
|
60
60
|
compression_ratio = max(0.01, min(compression_ratio, 1.0))
|
|
61
61
|
|
|
62
|
-
logger.
|
|
62
|
+
logger.debug("Compressing image with scale ratio ~%.3f", compression_ratio)
|
|
63
63
|
(
|
|
64
64
|
ffmpeg.input(input_path)
|
|
65
65
|
.output(
|
|
@@ -72,7 +72,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
|
|
|
72
72
|
.run(quiet=True, overwrite_output=True)
|
|
73
73
|
)
|
|
74
74
|
else:
|
|
75
|
-
logger.
|
|
75
|
+
logger.debug("Converting image to PNG without compression")
|
|
76
76
|
(
|
|
77
77
|
ffmpeg.input(input_path)
|
|
78
78
|
.output(
|
|
@@ -84,7 +84,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
|
|
|
84
84
|
.run(quiet=True, overwrite_output=True)
|
|
85
85
|
)
|
|
86
86
|
|
|
87
|
-
logger.
|
|
87
|
+
logger.debug("Successfully processed image")
|
|
88
88
|
return temp_image_path
|
|
89
89
|
|
|
90
90
|
except Exception as e:
|
|
@@ -206,19 +206,19 @@ class DocumentOCRToTextConverter:
|
|
|
206
206
|
start_time = time.time()
|
|
207
207
|
|
|
208
208
|
prompt_template = self._build_prompt_template()
|
|
209
|
-
logger.
|
|
209
|
+
logger.debug("Using prompt for %s format", "markdown" if self.markdown_output else "plain text")
|
|
210
210
|
|
|
211
211
|
client = self._build_client()
|
|
212
212
|
|
|
213
213
|
try:
|
|
214
214
|
mime_type, _ = mimetypes.guess_type(file_for_ocr)
|
|
215
|
-
logger.
|
|
215
|
+
logger.debug("OCR mime type: %s", mime_type)
|
|
216
216
|
|
|
217
217
|
file_size = os.path.getsize(file_for_ocr)
|
|
218
|
-
logger.
|
|
218
|
+
logger.debug("Initial image file size: %s", file_size)
|
|
219
219
|
|
|
220
220
|
if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
|
|
221
|
-
logger.
|
|
221
|
+
logger.debug(
|
|
222
222
|
f"Image exceeds {self.target_size}MB or unsupported mime type; compressing/converting"
|
|
223
223
|
)
|
|
224
224
|
temp_file_for_ocr = compress_and_convert_image(
|
|
@@ -226,11 +226,11 @@ class DocumentOCRToTextConverter:
|
|
|
226
226
|
target_size=self.target_size,
|
|
227
227
|
)
|
|
228
228
|
else:
|
|
229
|
-
logger.
|
|
229
|
+
logger.debug("No compression/conversion needed")
|
|
230
230
|
temp_file_for_ocr = file_for_ocr
|
|
231
231
|
|
|
232
232
|
file_size = os.path.getsize(temp_file_for_ocr)
|
|
233
|
-
logger.
|
|
233
|
+
logger.debug("Final image file size: %.2f MB", file_size / (1024 * 1024))
|
|
234
234
|
|
|
235
235
|
mime_type, _ = mimetypes.guess_type(temp_file_for_ocr)
|
|
236
236
|
if mime_type is None:
|
|
@@ -281,8 +281,8 @@ class DocumentOCRToTextConverter:
|
|
|
281
281
|
prompt_tokens = getattr(response.usage, "prompt_tokens", 0) if response.usage else 0
|
|
282
282
|
completion_tokens = getattr(response.usage, "completion_tokens", 0) if response.usage else 0
|
|
283
283
|
|
|
284
|
-
logger.
|
|
285
|
-
logger.
|
|
284
|
+
logger.debug("Completion tokens: %s", completion_tokens)
|
|
285
|
+
logger.debug("Prompt tokens: %s", prompt_tokens)
|
|
286
286
|
logger.info(f"OCR performed using {self.ocr_model} in {time_elapsed:.2f} seconds")
|
|
287
287
|
|
|
288
288
|
return {
|
|
@@ -97,7 +97,7 @@ def compress_and_convert_image(input_path: str, target_size=1):
|
|
|
97
97
|
loglevel='error'
|
|
98
98
|
).run(quiet=True, overwrite_output=True)
|
|
99
99
|
|
|
100
|
-
logger.
|
|
100
|
+
logger.debug("Successfully processed image")
|
|
101
101
|
return temp_image_path
|
|
102
102
|
|
|
103
103
|
except Exception as e:
|
|
@@ -335,18 +335,15 @@ class OCRToTextConverter:
|
|
|
335
335
|
start_time = time.time()
|
|
336
336
|
|
|
337
337
|
if self.markdown_output:
|
|
338
|
-
logger.
|
|
338
|
+
logger.debug("Using prompt for markdown format")
|
|
339
339
|
else:
|
|
340
|
-
logger.
|
|
340
|
+
logger.debug("Using prompt for plain text format")
|
|
341
341
|
prompt_template = self._build_prompt_template()
|
|
342
|
-
logger.info(prompt_template)
|
|
343
342
|
|
|
344
343
|
try:
|
|
345
344
|
if self.llm_api_key:
|
|
346
|
-
logger.info("Using provided Google API key")
|
|
347
345
|
client = genai.Client(api_key=self.llm_api_key)
|
|
348
346
|
else:
|
|
349
|
-
logger.info("Using Google API key from ENV")
|
|
350
347
|
client = genai.Client()
|
|
351
348
|
|
|
352
349
|
config = types.GenerateContentConfig(
|
|
@@ -380,26 +377,26 @@ class OCRToTextConverter:
|
|
|
380
377
|
)
|
|
381
378
|
|
|
382
379
|
mime_type, _ = mimetypes.guess_type(file_for_ocr)
|
|
383
|
-
logger.
|
|
380
|
+
logger.debug("OCR MIME type: %s", mime_type)
|
|
384
381
|
file_size = os.path.getsize(file_for_ocr)
|
|
385
|
-
logger.
|
|
382
|
+
logger.debug("Initial OCR image size: %s bytes", file_size)
|
|
386
383
|
if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
|
|
387
|
-
logger.
|
|
384
|
+
logger.debug("OCR image requires compression or conversion")
|
|
388
385
|
temp_file_for_ocr = compress_and_convert_image(input_path=file_for_ocr, target_size=self.target_size)
|
|
389
386
|
should_delete_temp_file = True
|
|
390
387
|
file_size = os.path.getsize(temp_file_for_ocr)
|
|
391
388
|
else:
|
|
392
|
-
logger.
|
|
389
|
+
logger.debug("OCR image requires no compression or conversion")
|
|
393
390
|
temp_file_for_ocr = file_for_ocr
|
|
394
391
|
|
|
395
|
-
logger.
|
|
392
|
+
logger.debug("Final OCR image size: %.2f MB", file_size / (1024 * 1024))
|
|
396
393
|
|
|
397
394
|
if file_size > 20 * 1024 * 1024:
|
|
398
|
-
logger.
|
|
395
|
+
logger.debug("Uploading large OCR image")
|
|
399
396
|
|
|
400
397
|
myfile = client.files.upload(file=temp_file_for_ocr)
|
|
401
398
|
|
|
402
|
-
logger.
|
|
399
|
+
logger.debug("OCR image upload completed")
|
|
403
400
|
|
|
404
401
|
contents = [myfile]
|
|
405
402
|
response = client.models.generate_content(
|
|
@@ -411,7 +408,7 @@ class OCRToTextConverter:
|
|
|
411
408
|
client.files.delete(name=myfile.name)
|
|
412
409
|
|
|
413
410
|
else:
|
|
414
|
-
logger.
|
|
411
|
+
logger.debug("OCR image can be sent inline")
|
|
415
412
|
with open(temp_file_for_ocr, "rb") as f:
|
|
416
413
|
image_data = f.read()
|
|
417
414
|
|
|
@@ -445,8 +442,8 @@ class OCRToTextConverter:
|
|
|
445
442
|
threshold=OCR_TAIL_REPETITION_THRESHOLD,
|
|
446
443
|
)
|
|
447
444
|
|
|
448
|
-
logger.
|
|
449
|
-
logger.
|
|
445
|
+
logger.debug("OCR completion tokens: %s", response.usage_metadata.candidates_token_count)
|
|
446
|
+
logger.debug("OCR prompt tokens: %s", response.usage_metadata.prompt_token_count)
|
|
450
447
|
|
|
451
448
|
if finish_reason and "RECITATION" in finish_reason:
|
|
452
449
|
raise EmptyDocument(
|
|
@@ -52,13 +52,13 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
|
|
|
52
52
|
os.close(fd)
|
|
53
53
|
|
|
54
54
|
original_size = os.path.getsize(input_path)
|
|
55
|
-
logger.
|
|
55
|
+
logger.debug("Original image size: %.2fMB", original_size / 1024 / 1024)
|
|
56
56
|
|
|
57
57
|
if original_size > target_size_bytes:
|
|
58
58
|
compression_ratio = (target_size_bytes / original_size) ** 0.5
|
|
59
59
|
compression_ratio = max(0.01, min(compression_ratio, 1.0))
|
|
60
60
|
|
|
61
|
-
logger.
|
|
61
|
+
logger.debug("Compressing image with scale ratio ~%.3f", compression_ratio)
|
|
62
62
|
(
|
|
63
63
|
ffmpeg.input(input_path)
|
|
64
64
|
.output(
|
|
@@ -71,7 +71,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
|
|
|
71
71
|
.run(quiet=True, overwrite_output=True)
|
|
72
72
|
)
|
|
73
73
|
else:
|
|
74
|
-
logger.
|
|
74
|
+
logger.debug("Converting image to PNG without compression")
|
|
75
75
|
(
|
|
76
76
|
ffmpeg.input(input_path)
|
|
77
77
|
.output(
|
|
@@ -83,7 +83,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
|
|
|
83
83
|
.run(quiet=True, overwrite_output=True)
|
|
84
84
|
)
|
|
85
85
|
|
|
86
|
-
logger.
|
|
86
|
+
logger.debug("Successfully processed image")
|
|
87
87
|
return temp_image_path
|
|
88
88
|
|
|
89
89
|
except Exception as e:
|
|
@@ -181,7 +181,7 @@ class OCRToTextConverter:
|
|
|
181
181
|
start_time = time.time()
|
|
182
182
|
|
|
183
183
|
prompt_template = self._build_prompt_template()
|
|
184
|
-
logger.
|
|
184
|
+
logger.debug("Using prompt for %s format", "markdown" if self.markdown_output else "plain text")
|
|
185
185
|
|
|
186
186
|
# Build Azure client
|
|
187
187
|
azure_api_key = self.llm_api_key or os.getenv("AZURE_OPENAI_API_KEY")
|
|
@@ -200,14 +200,14 @@ class OCRToTextConverter:
|
|
|
200
200
|
|
|
201
201
|
try:
|
|
202
202
|
mime_type, _ = mimetypes.guess_type(file_for_ocr)
|
|
203
|
-
logger.
|
|
203
|
+
logger.debug("OCR mime type: %s", mime_type)
|
|
204
204
|
|
|
205
205
|
file_size = os.path.getsize(file_for_ocr)
|
|
206
|
-
logger.
|
|
206
|
+
logger.debug("Initial image file size: %s", file_size)
|
|
207
207
|
|
|
208
208
|
# Compress/convert if too large or unsupported type
|
|
209
209
|
if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
|
|
210
|
-
logger.
|
|
210
|
+
logger.debug(
|
|
211
211
|
f"Image exceeds {self.target_size}MB or unsupported mime type; compressing/converting"
|
|
212
212
|
)
|
|
213
213
|
temp_file_for_ocr = compress_and_convert_image(
|
|
@@ -215,11 +215,11 @@ class OCRToTextConverter:
|
|
|
215
215
|
target_size=self.target_size,
|
|
216
216
|
)
|
|
217
217
|
else:
|
|
218
|
-
logger.
|
|
218
|
+
logger.debug("No compression/conversion needed")
|
|
219
219
|
temp_file_for_ocr = file_for_ocr
|
|
220
220
|
|
|
221
221
|
file_size = os.path.getsize(temp_file_for_ocr)
|
|
222
|
-
logger.
|
|
222
|
+
logger.debug("Final image file size: %.2f MB", file_size / (1024 * 1024))
|
|
223
223
|
|
|
224
224
|
# Azure vision chat expects either a public URL or base64 image data.
|
|
225
225
|
# We'll use base64 data-URL.
|
|
@@ -260,8 +260,8 @@ class OCRToTextConverter:
|
|
|
260
260
|
prompt_tokens = getattr(response.usage, "prompt_tokens", 0) if response.usage else 0
|
|
261
261
|
completion_tokens = getattr(response.usage, "completion_tokens", 0) if response.usage else 0
|
|
262
262
|
|
|
263
|
-
logger.
|
|
264
|
-
logger.
|
|
263
|
+
logger.debug("Completion tokens: %s", completion_tokens)
|
|
264
|
+
logger.debug("Prompt tokens: %s", prompt_tokens)
|
|
265
265
|
|
|
266
266
|
final_ocr_dict = {
|
|
267
267
|
"text": "" if "no readable text present" in text_out.lower() else text_out,
|
|
@@ -89,7 +89,7 @@ class DocumentConverter:
|
|
|
89
89
|
|
|
90
90
|
# Check file extension
|
|
91
91
|
_, ext = os.path.splitext(original_file)
|
|
92
|
-
logger.
|
|
92
|
+
logger.debug("Original file extension: %s", os.path.splitext(original_file)[1])
|
|
93
93
|
if ext.lower() not in self.supported_extensions and ext.lower() != '.pdf':
|
|
94
94
|
logger.warning(f"File extension '{ext}' may not be supported.")
|
|
95
95
|
|
|
@@ -105,7 +105,7 @@ class DocumentConverter:
|
|
|
105
105
|
if ext.lower() == '.pdf':
|
|
106
106
|
import shutil
|
|
107
107
|
shutil.copy2(input_file, output_file)
|
|
108
|
-
logger.
|
|
108
|
+
logger.debug("Input is already a PDF; copied to output")
|
|
109
109
|
return output_file
|
|
110
110
|
|
|
111
111
|
# Check if LibreOffice is installed
|
|
@@ -134,7 +134,7 @@ class DocumentConverter:
|
|
|
134
134
|
text=True,
|
|
135
135
|
check=True,
|
|
136
136
|
)
|
|
137
|
-
logger.
|
|
137
|
+
logger.debug("Document conversion successful")
|
|
138
138
|
except subprocess.CalledProcessError as e:
|
|
139
139
|
output_parts = []
|
|
140
140
|
if e.stdout:
|
|
@@ -89,12 +89,12 @@ class AudioLoader:
|
|
|
89
89
|
if self.s3_client is not None:
|
|
90
90
|
downloader = Downloader(s3_client=self.s3_client, document_aws_bucket=self.document_aws_bucket)
|
|
91
91
|
downloader.download_file_from_s3(file_path, temp_file_path)
|
|
92
|
-
logger.
|
|
92
|
+
logger.debug("Downloaded audio to a temporary file")
|
|
93
93
|
return temp_file_path
|
|
94
94
|
elif self.gcs_client is not None:
|
|
95
95
|
downloader = Downloader(gcs_client=self.gcs_client, document_gcs_bucket=self.document_gcs_bucket)
|
|
96
96
|
downloader.download_file_from_gcs(file_path, temp_file_path)
|
|
97
|
-
logger.
|
|
97
|
+
logger.debug("Downloaded audio to a temporary file")
|
|
98
98
|
return temp_file_path
|
|
99
99
|
raise AttributeError('Storage client not provided')
|
|
100
100
|
|
|
@@ -116,20 +116,20 @@ class AudioLoader:
|
|
|
116
116
|
ValueError: If the `audio_source` is not "cloud" or "local".
|
|
117
117
|
"""
|
|
118
118
|
|
|
119
|
-
logger.
|
|
119
|
+
logger.debug("Starting text extraction from audio")
|
|
120
120
|
|
|
121
121
|
# Load or download the video file
|
|
122
122
|
if self.source == "cloud":
|
|
123
123
|
fd, temp_file_path = tempfile.mkstemp()
|
|
124
124
|
try:
|
|
125
125
|
temp_file_path = self.download_audio(file_path, temp_file_path)
|
|
126
|
-
logger.
|
|
126
|
+
logger.debug("Successfully loaded cloud audio")
|
|
127
127
|
# saved_video_path = self.save_file_locally(temp_file_path, os.getcwd(), 'video')
|
|
128
128
|
finally:
|
|
129
129
|
os.close(fd) # Close the file descriptor
|
|
130
130
|
elif self.source == "local":
|
|
131
131
|
temp_file_path = file_path
|
|
132
|
-
logger.
|
|
132
|
+
logger.debug("Successfully loaded local audio")
|
|
133
133
|
else:
|
|
134
134
|
raise ValueError("Invalid audio source. Choose 'cloud', or 'local'.")
|
|
135
135
|
|
|
@@ -148,7 +148,7 @@ class AudioLoader:
|
|
|
148
148
|
# Clean up temporary file if it was downloaded
|
|
149
149
|
if self.source == "cloud" and os.path.exists(temp_file_path):
|
|
150
150
|
os.remove(temp_file_path)
|
|
151
|
-
logger.
|
|
151
|
+
logger.debug("Removed temporary audio")
|
|
152
152
|
|
|
153
153
|
return result_dict
|
|
154
154
|
|