polytext 0.2.8b7__tar.gz → 0.2.8b8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. {polytext-0.2.8b7 → polytext-0.2.8b8}/PKG-INFO +1 -1
  2. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/audio_to_text.py +20 -22
  3. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/document_ocr_to_text.py +15 -15
  4. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/document_ocr_to_text_azure_oai.py +12 -12
  5. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/ocr_to_text.py +13 -16
  6. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/ocr_to_text_azure_oai.py +12 -12
  7. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/pdf.py +3 -3
  8. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/audio.py +6 -6
  9. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/base.py +55 -5
  10. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/document.py +17 -19
  11. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/document_ocr.py +49 -10
  12. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/downloader/downloader.py +4 -4
  13. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/markdown.py +9 -9
  14. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/notebook.py +7 -7
  15. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/ocr.py +6 -6
  16. polytext-0.2.8b8/polytext/loader/ocr_cache.py +71 -0
  17. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/video.py +8 -8
  18. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/audio_chunker.py +11 -11
  19. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/text_merger.py +5 -5
  20. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/PKG-INFO +1 -1
  21. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/SOURCES.txt +3 -0
  22. {polytext-0.2.8b7 → polytext-0.2.8b8}/setup.py +1 -1
  23. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_base_loader_error_mapping.py +13 -0
  24. polytext-0.2.8b8/tests/test_document_ocr_cache.py +443 -0
  25. polytext-0.2.8b8/tests/test_ocr_cache.py +125 -0
  26. {polytext-0.2.8b7 → polytext-0.2.8b8}/LICENSE +0 -0
  27. {polytext-0.2.8b7 → polytext-0.2.8b8}/README.md +0 -0
  28. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/__init__.py +0 -0
  29. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/__init__.py +0 -0
  30. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/base.py +0 -0
  31. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/beautiful_text.py +0 -0
  32. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/gemini_quality_guards.py +0 -0
  33. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/html_to_md.py +0 -0
  34. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/md_to_text.py +0 -0
  35. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/text_to_md.py +0 -0
  36. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/converter/video_to_audio.py +0 -0
  37. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/exceptions/__init__.py +0 -0
  38. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/exceptions/base.py +0 -0
  39. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/generator/__init__.py +0 -0
  40. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/generator/pdf.py +0 -0
  41. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/__init__.py +0 -0
  42. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/aws_auth.py +0 -0
  43. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/downloader/__init__.py +0 -0
  44. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/html.py +0 -0
  45. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/plain_text.py +0 -0
  46. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/xml_xbrl.py +0 -0
  47. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/youtube.py +0 -0
  48. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/loader/youtube_llm.py +0 -0
  49. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/__init__.py +0 -0
  50. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/processor/transcript_chunker.py +0 -0
  51. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/__init__.py +0 -0
  52. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/beautiful_text.py +0 -0
  53. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/ocr.py +0 -0
  54. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/text_merging.py +0 -0
  55. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/text_to_md.py +0 -0
  56. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/prompts/transcription.py +0 -0
  57. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/utils/__init__.py +0 -0
  58. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext/utils/utils.py +0 -0
  59. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/dependency_links.txt +0 -0
  60. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/not-zip-safe +0 -0
  61. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/requires.txt +0 -0
  62. {polytext-0.2.8b7 → polytext-0.2.8b8}/polytext.egg-info/top_level.txt +0 -0
  63. {polytext-0.2.8b7 → polytext-0.2.8b8}/pyproject.toml +0 -0
  64. {polytext-0.2.8b7 → polytext-0.2.8b8}/setup.cfg +0 -0
  65. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_chunk_quality.py +0 -0
  66. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_chunker.py +0 -0
  67. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_comparison_helpers.py +0 -0
  68. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_audio_transcription_model_migration.py +0 -0
  69. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_aws_auth.py +0 -0
  70. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_beautiful_text_manual.py +0 -0
  71. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_audio_models.py +0 -0
  72. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_document_ocr_to_text_models.py +0 -0
  73. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_ocr_to_text_models.py +0 -0
  74. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_compare_youtube_models.py +0 -0
  75. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_dowload_audio_from_youtube.py +0 -0
  76. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_dowload_audio_from_youtube_helpers.py +0 -0
  77. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_extracted_text_whitespace.py +0 -0
  78. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_gemini_quality_guards.py +0 -0
  79. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_audio_transcript_from_gcs.py +0 -0
  80. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_customized_pdf_from_markdown.py +0 -0
  81. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_ocr.py +0 -0
  82. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_ocr_azure_oai.py +0 -0
  83. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_text.py +0 -0
  84. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_document_text_from_gcs.py +0 -0
  85. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_ocr_from_image.py +0 -0
  86. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_text_from_markdown.py +0 -0
  87. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_get_video_transcript_from_gcs.py +0 -0
  88. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_library.py +0 -0
  89. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_markdown_loader_gzip.py +0 -0
  90. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_markitdown_html.py +0 -0
  91. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_notebook_loader.py +0 -0
  92. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_ocr_fallbacks.py +0 -0
  93. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_ocr_image_descriptions.py +0 -0
  94. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_pain_text.py +0 -0
  95. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_pdf_conversion_error.py +0 -0
  96. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_python_version_metadata.py +0 -0
  97. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_split_audio_with_llm.py +0 -0
  98. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_transcribe_s3_images_from_csv.py +0 -0
  99. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_transcribe_s3_images_from_csv_script.py +0 -0
  100. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_xml_xbrl_loader.py +0 -0
  101. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_youtube_gemini_minimal_check.py +0 -0
  102. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_youtube_llm_fallbacks.py +0 -0
  103. {polytext-0.2.8b7 → polytext-0.2.8b8}/tests/test_youtube_transcript.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: polytext
3
- Version: 0.2.8b7
3
+ Version: 0.2.8b8
4
4
  Summary: Python utilities to simplify document files management
5
5
  Home-page: https://github.com/docsity/polytext
6
6
  Author: Matteo Senardi
@@ -176,7 +176,7 @@ def find_suspicious_fallback_language_indices(
176
176
  ) -> list[int]:
177
177
  """Find internal fallback chunks whose language conflicts with two agreeing neighbours."""
178
178
  suspicious_indices = []
179
- logger.info("Starting fallback language validation for %s transcript chunks", len(transcripts))
179
+ logger.debug("Starting fallback language validation for %s transcript chunks", len(transcripts))
180
180
 
181
181
  def has_fallback(result: dict | None) -> bool:
182
182
  if not result:
@@ -402,7 +402,7 @@ def compress_and_convert_audio(input_path: str, bitrate_quality: int = 9) -> str
402
402
  os.unlink(temp_audio_path)
403
403
  raise
404
404
 
405
- logger.info(f"Successfully normalized audio: {temp_audio_path}")
405
+ logger.debug("Audio normalization completed")
406
406
  return temp_audio_path
407
407
 
408
408
 
@@ -721,7 +721,7 @@ class AudioToTextConverter:
721
721
  config: types.GenerateContentConfig,
722
722
  ):
723
723
  file_size = os.path.getsize(audio_file)
724
- logger.info(f"Audio file size: {file_size / (1024 * 1024):.2f} MB")
724
+ logger.debug("Audio file size: %.2f MB", file_size / (1024 * 1024))
725
725
  marker_nonce = uuid.uuid4().hex[:12]
726
726
  marker_start = f"<<<UNTRUSTED_AUDIO_START_{marker_nonce}>>>"
727
727
  marker_end = f"<<<UNTRUSTED_AUDIO_END_{marker_nonce}>>>"
@@ -736,9 +736,9 @@ class AudioToTextConverter:
736
736
  model=self.transcription_model,
737
737
  contents=[my_file]
738
738
  )
739
- logger.info(f"File size in tokens: {response}")
739
+ logger.debug("Audio file token count response received")
740
740
 
741
- logger.info(f"Uploaded file: {my_file.name} - Starting transcription...")
741
+ logger.debug("Audio upload completed; starting transcription")
742
742
 
743
743
  return client.models.generate_content(
744
744
  model=self.transcription_model,
@@ -750,7 +750,7 @@ class AudioToTextConverter:
750
750
  if temp_upload_path and os.path.exists(temp_upload_path):
751
751
  os.remove(temp_upload_path)
752
752
 
753
- logger.info("Audio file size does not exceed 20MB")
753
+ logger.debug("Audio can be sent inline")
754
754
  with open(audio_file, "rb") as f:
755
755
  audio_data = f.read()
756
756
 
@@ -853,8 +853,8 @@ class AudioToTextConverter:
853
853
  completion_tokens = getattr(usage_metadata, "candidates_token_count", 0) or 0
854
854
  prompt_tokens = getattr(usage_metadata, "prompt_token_count", 0) or 0
855
855
 
856
- logger.info(f"Completion tokens: {completion_tokens}")
857
- logger.info(f"Prompt tokens: {prompt_tokens}")
856
+ logger.debug("Audio completion tokens: %s", completion_tokens)
857
+ logger.debug("Audio prompt tokens: %s", prompt_tokens)
858
858
 
859
859
  if finish_reason and "RECITATION" in finish_reason:
860
860
  raise EmptyDocument(
@@ -1005,7 +1005,7 @@ class AudioToTextConverter:
1005
1005
 
1006
1006
  def process_chunk(self, chunk: dict, index: int) -> tuple[int, dict]:
1007
1007
  """Process a single audio chunk and return its transcript"""
1008
- logger.info(f"Transcribing chunk {index + 1}...")
1008
+ logger.debug("Transcribing audio chunk %s", index + 1)
1009
1009
  transcript_dict = self.transcribe_audio(chunk["file_path"])
1010
1010
  return index, transcript_dict
1011
1011
 
@@ -1071,12 +1071,12 @@ class AudioToTextConverter:
1071
1071
  RuntimeError: If there's an error during audio processing or transcription
1072
1072
  """
1073
1073
  processed_audio_path = None
1074
- logger.info(f"Processing audio file {audio_path}...")
1074
+ logger.debug("Processing audio file")
1075
1075
  file_size = os.path.getsize(audio_path)
1076
- logger.info(f"Audio file size: {file_size / (1024 * 1024):.2f} MB")
1076
+ logger.debug("Audio file size: %.2f MB", file_size / (1024 * 1024))
1077
1077
 
1078
1078
  mime_type, _ = mimetypes.guess_type(audio_path)
1079
- logger.info(f"Original MIME type: {mime_type}")
1079
+ logger.debug("Original audio MIME type: %s", mime_type)
1080
1080
 
1081
1081
  # Check if conversion and/or compression is needed
1082
1082
  needs_conversion = mime_type not in SUPPORTED_MIME_TYPES
@@ -1084,29 +1084,29 @@ class AudioToTextConverter:
1084
1084
 
1085
1085
  # If you need at least one of the two, apply compress_and_convert_audio
1086
1086
  if needs_conversion: # or needs_compression:
1087
- logger.info("Audio file needs conversion, processing file...")
1087
+ logger.debug("Audio requires conversion")
1088
1088
  processed_audio_path = compress_and_convert_audio(audio_path)
1089
1089
  used_file = processed_audio_path
1090
- logger.info(f"Audio file processed (conversion): {used_file}")
1090
+ logger.debug("Audio conversion completed")
1091
1091
  else:
1092
1092
  used_file = audio_path
1093
- logger.info("Audio file is already in supported format")
1093
+ logger.debug("Audio is already in a supported format")
1094
1094
  # If you need at least one of the two, apply compress_and_convert_audio
1095
1095
  if needs_conversion: # or needs_compression:
1096
- logger.info("Audio file needs conversion, processing file...")
1096
+ logger.debug("Audio requires conversion")
1097
1097
  processed_audio_path = compress_and_convert_audio(input_path=audio_path,
1098
1098
  bitrate_quality=self.bitrate_quality)
1099
1099
  used_file = processed_audio_path
1100
- logger.info(f"Audio file processed (conversion): {used_file}")
1100
+ logger.debug("Audio conversion completed")
1101
1101
  else:
1102
1102
  used_file = audio_path
1103
- logger.info("Audio file is already in supported format")
1103
+ logger.debug("Audio is already in a supported format")
1104
1104
 
1105
1105
  # Create chunker and extract chunks
1106
- logger.info("Creating AudioChunker instance...")
1106
+ logger.debug("Creating audio chunker")
1107
1107
  chunker = AudioChunker(used_file, max_llm_tokens=self.max_llm_tokens)
1108
1108
  self.set_long_audio_protections(chunker.duration_ms)
1109
- logger.info(
1109
+ logger.debug(
1110
1110
  "Long-audio transcription protections enabled: %s (duration: %sms, threshold: %sms)",
1111
1111
  self.long_audio_protections_enabled,
1112
1112
  chunker.duration_ms,
@@ -1114,8 +1114,6 @@ class AudioToTextConverter:
1114
1114
  )
1115
1115
  chunks = chunker.extract_chunks()
1116
1116
 
1117
- logger.info(f"chunks: {chunks}")
1118
-
1119
1117
  logger.info(f"Split audio into {len(chunks)} chunks")
1120
1118
  if len(chunks) > 1 and self.markdown_output:
1121
1119
  logger.info("Audio chunking is needed, returning minimal markdown output")
@@ -97,7 +97,7 @@ def compress_and_convert_image(input_path: str, target_size=1):
97
97
  loglevel='error'
98
98
  ).run(quiet=True, overwrite_output=True)
99
99
 
100
- logger.info(f"Successfully processed image: {temp_image_path}")
100
+ logger.debug("Successfully processed image")
101
101
  return temp_image_path
102
102
 
103
103
  except Exception as e:
@@ -363,17 +363,17 @@ class DocumentOCRToTextConverter:
363
363
  start_time = time.time()
364
364
 
365
365
  if self.markdown_output:
366
- logger.info("Using prompt for markdown format")
366
+ logger.debug("Using prompt for markdown format")
367
367
  else:
368
- logger.info("Using prompt for plain text format")
368
+ logger.debug("Using prompt for plain text format")
369
369
  prompt_template = self._build_prompt_template()
370
370
 
371
371
  try:
372
372
  if self.llm_api_key:
373
- logger.info("Using provided Google API key")
373
+ logger.debug("Using provided Google API key")
374
374
  client = genai.Client(api_key=self.llm_api_key)
375
375
  else:
376
- logger.info("Using Google API key from ENV")
376
+ logger.debug("Using Google API key from environment")
377
377
  client = genai.Client()
378
378
 
379
379
  config = types.GenerateContentConfig(
@@ -407,26 +407,26 @@ class DocumentOCRToTextConverter:
407
407
  )
408
408
 
409
409
  mime_type, _ = mimetypes.guess_type(file_for_ocr)
410
- logger.info(f"OCR mime type: {mime_type}")
410
+ logger.debug("OCR MIME type: %s", mime_type)
411
411
  file_size = os.path.getsize(file_for_ocr)
412
- logger.info(f"Initial image file size: {file_size}")
412
+ logger.debug("Initial OCR image size: %s bytes", file_size)
413
413
  if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
414
- logger.info(f"Image file size exceeds {self.target_size}MB or unsupported mime type, compressing and converting image")
414
+ logger.debug("OCR image requires compression or conversion")
415
415
  temp_file_for_ocr = compress_and_convert_image(input_path=file_for_ocr, target_size=self.target_size)
416
416
  should_delete_temp_file = True
417
417
  file_size = os.path.getsize(temp_file_for_ocr)
418
418
  else:
419
- logger.info(f"Image file size does not exceed {self.target_size}MB and mime type is supported, no compression or conversion needed")
419
+ logger.debug("OCR image requires no compression or conversion")
420
420
  temp_file_for_ocr = file_for_ocr
421
421
 
422
- logger.info(f"Final image file size: {file_size / (1024 * 1024):.2f} MB")
422
+ logger.debug("Final OCR image size: %.2f MB", file_size / (1024 * 1024))
423
423
 
424
424
  if file_size > 20 * 1024 * 1024:
425
- logger.info("Total image file size exceeds 20MB, uploading file before transcription")
425
+ logger.debug("Uploading large OCR image")
426
426
 
427
427
  myfile = client.files.upload(file=temp_file_for_ocr)
428
428
 
429
- logger.info(f"Uploaded image file - Starting OCR...")
429
+ logger.debug("OCR image upload completed")
430
430
 
431
431
  contents = [myfile]
432
432
  response = client.models.generate_content(
@@ -438,7 +438,7 @@ class DocumentOCRToTextConverter:
438
438
  client.files.delete(name=myfile.name)
439
439
 
440
440
  else:
441
- logger.info("Image file size does not exceed 20MB")
441
+ logger.debug("OCR image can be sent inline")
442
442
  with open(temp_file_for_ocr, "rb") as f:
443
443
  image_data = f.read()
444
444
 
@@ -472,8 +472,8 @@ class DocumentOCRToTextConverter:
472
472
  threshold=OCR_TAIL_REPETITION_THRESHOLD,
473
473
  )
474
474
 
475
- logger.info(f"Completion tokens: {response.usage_metadata.candidates_token_count}")
476
- logger.info(f"Prompt tokens: {response.usage_metadata.prompt_token_count}")
475
+ logger.debug("OCR completion tokens: %s", response.usage_metadata.candidates_token_count)
476
+ logger.debug("OCR prompt tokens: %s", response.usage_metadata.prompt_token_count)
477
477
 
478
478
  if finish_reason and "RECITATION" in finish_reason:
479
479
  raise EmptyDocument(
@@ -53,13 +53,13 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
53
53
  os.close(fd)
54
54
 
55
55
  original_size = os.path.getsize(input_path)
56
- logger.info(f"Original image size: {original_size / 1024 / 1024:.2f}MB")
56
+ logger.debug("Original image size: %.2fMB", original_size / 1024 / 1024)
57
57
 
58
58
  if original_size > target_size_bytes:
59
59
  compression_ratio = (target_size_bytes / original_size) ** 0.5
60
60
  compression_ratio = max(0.01, min(compression_ratio, 1.0))
61
61
 
62
- logger.info(f"Compressing image with scale ratio ~{compression_ratio:.3f}")
62
+ logger.debug("Compressing image with scale ratio ~%.3f", compression_ratio)
63
63
  (
64
64
  ffmpeg.input(input_path)
65
65
  .output(
@@ -72,7 +72,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
72
72
  .run(quiet=True, overwrite_output=True)
73
73
  )
74
74
  else:
75
- logger.info("Converting image to PNG without compression")
75
+ logger.debug("Converting image to PNG without compression")
76
76
  (
77
77
  ffmpeg.input(input_path)
78
78
  .output(
@@ -84,7 +84,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
84
84
  .run(quiet=True, overwrite_output=True)
85
85
  )
86
86
 
87
- logger.info(f"Successfully processed image: {temp_image_path}")
87
+ logger.debug("Successfully processed image")
88
88
  return temp_image_path
89
89
 
90
90
  except Exception as e:
@@ -206,19 +206,19 @@ class DocumentOCRToTextConverter:
206
206
  start_time = time.time()
207
207
 
208
208
  prompt_template = self._build_prompt_template()
209
- logger.info("Using prompt for %s format", "markdown" if self.markdown_output else "plain text")
209
+ logger.debug("Using prompt for %s format", "markdown" if self.markdown_output else "plain text")
210
210
 
211
211
  client = self._build_client()
212
212
 
213
213
  try:
214
214
  mime_type, _ = mimetypes.guess_type(file_for_ocr)
215
- logger.info(f"OCR mime type: {mime_type}")
215
+ logger.debug("OCR mime type: %s", mime_type)
216
216
 
217
217
  file_size = os.path.getsize(file_for_ocr)
218
- logger.info(f"Initial image file size: {file_size}")
218
+ logger.debug("Initial image file size: %s", file_size)
219
219
 
220
220
  if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
221
- logger.info(
221
+ logger.debug(
222
222
  f"Image exceeds {self.target_size}MB or unsupported mime type; compressing/converting"
223
223
  )
224
224
  temp_file_for_ocr = compress_and_convert_image(
@@ -226,11 +226,11 @@ class DocumentOCRToTextConverter:
226
226
  target_size=self.target_size,
227
227
  )
228
228
  else:
229
- logger.info("No compression/conversion needed")
229
+ logger.debug("No compression/conversion needed")
230
230
  temp_file_for_ocr = file_for_ocr
231
231
 
232
232
  file_size = os.path.getsize(temp_file_for_ocr)
233
- logger.info(f"Final image file size: {file_size / (1024 * 1024):.2f} MB")
233
+ logger.debug("Final image file size: %.2f MB", file_size / (1024 * 1024))
234
234
 
235
235
  mime_type, _ = mimetypes.guess_type(temp_file_for_ocr)
236
236
  if mime_type is None:
@@ -281,8 +281,8 @@ class DocumentOCRToTextConverter:
281
281
  prompt_tokens = getattr(response.usage, "prompt_tokens", 0) if response.usage else 0
282
282
  completion_tokens = getattr(response.usage, "completion_tokens", 0) if response.usage else 0
283
283
 
284
- logger.info(f"Completion tokens: {completion_tokens}")
285
- logger.info(f"Prompt tokens: {prompt_tokens}")
284
+ logger.debug("Completion tokens: %s", completion_tokens)
285
+ logger.debug("Prompt tokens: %s", prompt_tokens)
286
286
  logger.info(f"OCR performed using {self.ocr_model} in {time_elapsed:.2f} seconds")
287
287
 
288
288
  return {
@@ -97,7 +97,7 @@ def compress_and_convert_image(input_path: str, target_size=1):
97
97
  loglevel='error'
98
98
  ).run(quiet=True, overwrite_output=True)
99
99
 
100
- logger.info(f"Successfully processed image: {temp_image_path}")
100
+ logger.debug("Successfully processed image")
101
101
  return temp_image_path
102
102
 
103
103
  except Exception as e:
@@ -335,18 +335,15 @@ class OCRToTextConverter:
335
335
  start_time = time.time()
336
336
 
337
337
  if self.markdown_output:
338
- logger.info("Using prompt for markdown format")
338
+ logger.debug("Using prompt for markdown format")
339
339
  else:
340
- logger.info("Using prompt for plain text format")
340
+ logger.debug("Using prompt for plain text format")
341
341
  prompt_template = self._build_prompt_template()
342
- logger.info(prompt_template)
343
342
 
344
343
  try:
345
344
  if self.llm_api_key:
346
- logger.info("Using provided Google API key")
347
345
  client = genai.Client(api_key=self.llm_api_key)
348
346
  else:
349
- logger.info("Using Google API key from ENV")
350
347
  client = genai.Client()
351
348
 
352
349
  config = types.GenerateContentConfig(
@@ -380,26 +377,26 @@ class OCRToTextConverter:
380
377
  )
381
378
 
382
379
  mime_type, _ = mimetypes.guess_type(file_for_ocr)
383
- logger.info(f"OCR mime type: {mime_type}")
380
+ logger.debug("OCR MIME type: %s", mime_type)
384
381
  file_size = os.path.getsize(file_for_ocr)
385
- logger.info(f"Initial image file size: {file_size}")
382
+ logger.debug("Initial OCR image size: %s bytes", file_size)
386
383
  if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
387
- logger.info(f"Image file size exceeds {self.target_size}MB or unsupported mime type, compressing and converting image")
384
+ logger.debug("OCR image requires compression or conversion")
388
385
  temp_file_for_ocr = compress_and_convert_image(input_path=file_for_ocr, target_size=self.target_size)
389
386
  should_delete_temp_file = True
390
387
  file_size = os.path.getsize(temp_file_for_ocr)
391
388
  else:
392
- logger.info(f"Image file size does not exceed {self.target_size}MB and mime type is supported, no compression or conversion needed")
389
+ logger.debug("OCR image requires no compression or conversion")
393
390
  temp_file_for_ocr = file_for_ocr
394
391
 
395
- logger.info(f"Final image file size: {file_size / (1024 * 1024):.2f} MB")
392
+ logger.debug("Final OCR image size: %.2f MB", file_size / (1024 * 1024))
396
393
 
397
394
  if file_size > 20 * 1024 * 1024:
398
- logger.info("Total image file size exceeds 20MB, uploading file before transcription")
395
+ logger.debug("Uploading large OCR image")
399
396
 
400
397
  myfile = client.files.upload(file=temp_file_for_ocr)
401
398
 
402
- logger.info(f"Uploaded image file - Starting OCR...")
399
+ logger.debug("OCR image upload completed")
403
400
 
404
401
  contents = [myfile]
405
402
  response = client.models.generate_content(
@@ -411,7 +408,7 @@ class OCRToTextConverter:
411
408
  client.files.delete(name=myfile.name)
412
409
 
413
410
  else:
414
- logger.info("Image file size does not exceed 20MB")
411
+ logger.debug("OCR image can be sent inline")
415
412
  with open(temp_file_for_ocr, "rb") as f:
416
413
  image_data = f.read()
417
414
 
@@ -445,8 +442,8 @@ class OCRToTextConverter:
445
442
  threshold=OCR_TAIL_REPETITION_THRESHOLD,
446
443
  )
447
444
 
448
- logger.info(f"Completion tokens: {response.usage_metadata.candidates_token_count}")
449
- logger.info(f"Prompt tokens: {response.usage_metadata.prompt_token_count}")
445
+ logger.debug("OCR completion tokens: %s", response.usage_metadata.candidates_token_count)
446
+ logger.debug("OCR prompt tokens: %s", response.usage_metadata.prompt_token_count)
450
447
 
451
448
  if finish_reason and "RECITATION" in finish_reason:
452
449
  raise EmptyDocument(
@@ -52,13 +52,13 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
52
52
  os.close(fd)
53
53
 
54
54
  original_size = os.path.getsize(input_path)
55
- logger.info(f"Original image size: {original_size / 1024 / 1024:.2f}MB")
55
+ logger.debug("Original image size: %.2fMB", original_size / 1024 / 1024)
56
56
 
57
57
  if original_size > target_size_bytes:
58
58
  compression_ratio = (target_size_bytes / original_size) ** 0.5
59
59
  compression_ratio = max(0.01, min(compression_ratio, 1.0))
60
60
 
61
- logger.info(f"Compressing image with scale ratio ~{compression_ratio:.3f}")
61
+ logger.debug("Compressing image with scale ratio ~%.3f", compression_ratio)
62
62
  (
63
63
  ffmpeg.input(input_path)
64
64
  .output(
@@ -71,7 +71,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
71
71
  .run(quiet=True, overwrite_output=True)
72
72
  )
73
73
  else:
74
- logger.info("Converting image to PNG without compression")
74
+ logger.debug("Converting image to PNG without compression")
75
75
  (
76
76
  ffmpeg.input(input_path)
77
77
  .output(
@@ -83,7 +83,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
83
83
  .run(quiet=True, overwrite_output=True)
84
84
  )
85
85
 
86
- logger.info(f"Successfully processed image: {temp_image_path}")
86
+ logger.debug("Successfully processed image")
87
87
  return temp_image_path
88
88
 
89
89
  except Exception as e:
@@ -181,7 +181,7 @@ class OCRToTextConverter:
181
181
  start_time = time.time()
182
182
 
183
183
  prompt_template = self._build_prompt_template()
184
- logger.info("Using prompt for %s format", "markdown" if self.markdown_output else "plain text")
184
+ logger.debug("Using prompt for %s format", "markdown" if self.markdown_output else "plain text")
185
185
 
186
186
  # Build Azure client
187
187
  azure_api_key = self.llm_api_key or os.getenv("AZURE_OPENAI_API_KEY")
@@ -200,14 +200,14 @@ class OCRToTextConverter:
200
200
 
201
201
  try:
202
202
  mime_type, _ = mimetypes.guess_type(file_for_ocr)
203
- logger.info(f"OCR mime type: {mime_type}")
203
+ logger.debug("OCR mime type: %s", mime_type)
204
204
 
205
205
  file_size = os.path.getsize(file_for_ocr)
206
- logger.info(f"Initial image file size: {file_size}")
206
+ logger.debug("Initial image file size: %s", file_size)
207
207
 
208
208
  # Compress/convert if too large or unsupported type
209
209
  if file_size > self.target_size * 1024 * 1024 or mime_type not in SUPPORTED_MIME_TYPES:
210
- logger.info(
210
+ logger.debug(
211
211
  f"Image exceeds {self.target_size}MB or unsupported mime type; compressing/converting"
212
212
  )
213
213
  temp_file_for_ocr = compress_and_convert_image(
@@ -215,11 +215,11 @@ class OCRToTextConverter:
215
215
  target_size=self.target_size,
216
216
  )
217
217
  else:
218
- logger.info("No compression/conversion needed")
218
+ logger.debug("No compression/conversion needed")
219
219
  temp_file_for_ocr = file_for_ocr
220
220
 
221
221
  file_size = os.path.getsize(temp_file_for_ocr)
222
- logger.info(f"Final image file size: {file_size / (1024 * 1024):.2f} MB")
222
+ logger.debug("Final image file size: %.2f MB", file_size / (1024 * 1024))
223
223
 
224
224
  # Azure vision chat expects either a public URL or base64 image data.
225
225
  # We'll use base64 data-URL.
@@ -260,8 +260,8 @@ class OCRToTextConverter:
260
260
  prompt_tokens = getattr(response.usage, "prompt_tokens", 0) if response.usage else 0
261
261
  completion_tokens = getattr(response.usage, "completion_tokens", 0) if response.usage else 0
262
262
 
263
- logger.info(f"Completion tokens: {completion_tokens}")
264
- logger.info(f"Prompt tokens: {prompt_tokens}")
263
+ logger.debug("Completion tokens: %s", completion_tokens)
264
+ logger.debug("Prompt tokens: %s", prompt_tokens)
265
265
 
266
266
  final_ocr_dict = {
267
267
  "text": "" if "no readable text present" in text_out.lower() else text_out,
@@ -89,7 +89,7 @@ class DocumentConverter:
89
89
 
90
90
  # Check file extension
91
91
  _, ext = os.path.splitext(original_file)
92
- logger.info(os.path.splitext(original_file))
92
+ logger.debug("Original file extension: %s", os.path.splitext(original_file)[1])
93
93
  if ext.lower() not in self.supported_extensions and ext.lower() != '.pdf':
94
94
  logger.warning(f"File extension '{ext}' may not be supported.")
95
95
 
@@ -105,7 +105,7 @@ class DocumentConverter:
105
105
  if ext.lower() == '.pdf':
106
106
  import shutil
107
107
  shutil.copy2(input_file, output_file)
108
- logger.info(f"File is already a PDF. Copied to '{output_file}'")
108
+ logger.debug("Input is already a PDF; copied to output")
109
109
  return output_file
110
110
 
111
111
  # Check if LibreOffice is installed
@@ -134,7 +134,7 @@ class DocumentConverter:
134
134
  text=True,
135
135
  check=True,
136
136
  )
137
- logger.info(f"Conversion successful: '{output_file}'")
137
+ logger.debug("Document conversion successful")
138
138
  except subprocess.CalledProcessError as e:
139
139
  output_parts = []
140
140
  if e.stdout:
@@ -89,12 +89,12 @@ class AudioLoader:
89
89
  if self.s3_client is not None:
90
90
  downloader = Downloader(s3_client=self.s3_client, document_aws_bucket=self.document_aws_bucket)
91
91
  downloader.download_file_from_s3(file_path, temp_file_path)
92
- logger.info(f'Downloaded {file_path} to {temp_file_path}')
92
+ logger.debug("Downloaded audio to a temporary file")
93
93
  return temp_file_path
94
94
  elif self.gcs_client is not None:
95
95
  downloader = Downloader(gcs_client=self.gcs_client, document_gcs_bucket=self.document_gcs_bucket)
96
96
  downloader.download_file_from_gcs(file_path, temp_file_path)
97
- logger.info(f'Downloaded {file_path} to {temp_file_path}')
97
+ logger.debug("Downloaded audio to a temporary file")
98
98
  return temp_file_path
99
99
  raise AttributeError('Storage client not provided')
100
100
 
@@ -116,20 +116,20 @@ class AudioLoader:
116
116
  ValueError: If the `audio_source` is not "cloud" or "local".
117
117
  """
118
118
 
119
- logger.info("Starting text extraction from audio...")
119
+ logger.debug("Starting text extraction from audio")
120
120
 
121
121
  # Load or download the video file
122
122
  if self.source == "cloud":
123
123
  fd, temp_file_path = tempfile.mkstemp()
124
124
  try:
125
125
  temp_file_path = self.download_audio(file_path, temp_file_path)
126
- logger.info(f"Successfully loaded audio file from {file_path}")
126
+ logger.debug("Successfully loaded cloud audio")
127
127
  # saved_video_path = self.save_file_locally(temp_file_path, os.getcwd(), 'video')
128
128
  finally:
129
129
  os.close(fd) # Close the file descriptor
130
130
  elif self.source == "local":
131
131
  temp_file_path = file_path
132
- logger.info(f"Successfully loaded audio file from local path {file_path}")
132
+ logger.debug("Successfully loaded local audio")
133
133
  else:
134
134
  raise ValueError("Invalid audio source. Choose 'cloud', or 'local'.")
135
135
 
@@ -148,7 +148,7 @@ class AudioLoader:
148
148
  # Clean up temporary file if it was downloaded
149
149
  if self.source == "cloud" and os.path.exists(temp_file_path):
150
150
  os.remove(temp_file_path)
151
- logger.info(f"Removed temporary file {temp_file_path}")
151
+ logger.debug("Removed temporary audio")
152
152
 
153
153
  return result_dict
154
154