markitdown-pro 1.0.2__tar.gz → 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/PKG-INFO +1 -1
- markitdown_pro-1.0.4/markitdown_pro/common/logger.py +24 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/conversion_pipeline.py +1 -3
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/pdf_handler.py +6 -2
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/services/openai_services.py +24 -31
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro.egg-info/PKG-INFO +1 -1
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/setup.py +1 -1
- markitdown_pro-1.0.2/markitdown_pro/common/logger.py +0 -20
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/README.md +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/__init__.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/common/__init__.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/common/utils.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/__init__.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/azure_doc_intel_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/azure_speech_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/base.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/gpt_vision_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/markitdown_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/pymupdf_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/unstructured_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/youtube_wrapper.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/__init__.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/audio_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/base_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/email_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/epub_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/image_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/ipynb_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/markitdown_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/markup_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/office_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/pst_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/tabular_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/handlers/text_handler.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/services/__init__.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/services/azure_doc_intelligence.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/services/azure_speech.py +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro.egg-info/SOURCES.txt +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro.egg-info/dependency_links.txt +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro.egg-info/requires.txt +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro.egg-info/top_level.txt +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/pyproject.toml +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/setup.cfg +0 -0
- {markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/tests/test.py +0 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
log_level = int(os.getenv("LOG_LEVEL", "20"))
|
|
5
|
+
logger = logging.getLogger(__name__)
|
|
6
|
+
logging.basicConfig(level=log_level, format="[%(asctime)s] [%(levelname)s] %(message)s")
|
|
7
|
+
|
|
8
|
+
NOISY_LIST = [
|
|
9
|
+
"azure.core.pipeline.policies.http_logging_policy",
|
|
10
|
+
"azure.core.pipeline.policies._universal",
|
|
11
|
+
"Transmission succeeded...",
|
|
12
|
+
"azure.monitor.opentelemetry.exporter",
|
|
13
|
+
"azure.monitor.opentelemetry.exporter._transmission",
|
|
14
|
+
"azure",
|
|
15
|
+
"opentelemetry",
|
|
16
|
+
"urllib3",
|
|
17
|
+
"urllib3.connectionpool",
|
|
18
|
+
"httpx",
|
|
19
|
+
"httpcore",
|
|
20
|
+
"openai",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
for name in NOISY_LIST:
|
|
24
|
+
logging.getLogger(name).setLevel(logging.WARNING)
|
|
@@ -186,14 +186,12 @@ class ConversionPipeline:
|
|
|
186
186
|
raise RuntimeError(f"No handler for extension '{extension}'.")
|
|
187
187
|
|
|
188
188
|
try:
|
|
189
|
-
logger.debug(f"convert_document_to_md: Using handler for extension '{extension}'")
|
|
190
189
|
md_content = await handler.handle(str(file_path))
|
|
191
190
|
if not md_content or not ensure_minimum_content(md_content):
|
|
192
191
|
return None
|
|
193
192
|
|
|
194
193
|
logger.debug(
|
|
195
|
-
"convert_document_to_md:
|
|
196
|
-
len(md_content),
|
|
194
|
+
f"convert_document_to_md: {file_path} returned {len(md_content)} characters"
|
|
197
195
|
)
|
|
198
196
|
return _write_md(output_md, md_content) if output_md else md_content
|
|
199
197
|
|
|
@@ -164,8 +164,12 @@ class PDFHandler(BaseHandler):
|
|
|
164
164
|
if page.get_images(full=True):
|
|
165
165
|
pages_with_images += 1
|
|
166
166
|
|
|
167
|
-
logger.debug(
|
|
168
|
-
|
|
167
|
+
logger.debug(
|
|
168
|
+
f"PDFHandler: Pages with text for {file_path}: {pages_with_text}/{total_pages}"
|
|
169
|
+
)
|
|
170
|
+
logger.debug(
|
|
171
|
+
f"PDFHandler: Pages with images for {file_path}: {pages_with_images}/{total_pages}"
|
|
172
|
+
)
|
|
169
173
|
|
|
170
174
|
is_text_only = pages_with_text == total_pages and pages_with_images == 0
|
|
171
175
|
is_all_images = pages_with_images == total_pages and pages_with_text == 0
|
|
@@ -224,44 +224,37 @@ class GPTVision:
|
|
|
224
224
|
if not self.lang_client:
|
|
225
225
|
logger.error("GPT-vision client is not initialized.")
|
|
226
226
|
raise ValueError("GPT-vision client is not initialized.")
|
|
227
|
-
|
|
228
227
|
attempt = 0
|
|
229
|
-
payload = list(messages)
|
|
230
|
-
|
|
228
|
+
payload = list(messages)
|
|
231
229
|
while True:
|
|
232
230
|
try:
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
response = await asyncio.wait_for(task, timeout=self.request_timeout_s)
|
|
236
|
-
except asyncio.TimeoutError as te:
|
|
237
|
-
# Cancel in-flight task cleanly
|
|
238
|
-
task.cancel()
|
|
239
|
-
with suppress(asyncio.CancelledError):
|
|
240
|
-
await task
|
|
241
|
-
raise te
|
|
242
|
-
|
|
231
|
+
async with asyncio.timeout(self.request_timeout_s):
|
|
232
|
+
response = await self.lang_client.ainvoke(payload)
|
|
243
233
|
if not response:
|
|
244
234
|
logger.error("No response from GPT-vision client.")
|
|
245
235
|
return None
|
|
246
236
|
return response
|
|
247
|
-
|
|
237
|
+
except asyncio.CancelledError:
|
|
238
|
+
# If caller cancels (shutdown), propagate immediately
|
|
239
|
+
logger.error(f"ainvoke cancelled for {file_or_url}")
|
|
240
|
+
raise
|
|
241
|
+
except asyncio.TimeoutError as e:
|
|
242
|
+
# Treat like transient below
|
|
243
|
+
last_err = e
|
|
248
244
|
except Exception as e:
|
|
249
|
-
|
|
250
|
-
is_transient = self._is_transient_error(e, retry_http_statuses)
|
|
251
|
-
if not is_transient:
|
|
245
|
+
if not self._is_transient_error(e, retry_http_statuses):
|
|
252
246
|
raise
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
await asyncio.sleep(sleep_for)
|
|
247
|
+
last_err = e
|
|
248
|
+
attempt += 1
|
|
249
|
+
if attempt > max_retries:
|
|
250
|
+
raise last_err
|
|
251
|
+
cap = min(max_delay, base_delay * (2 ** (attempt - 1)))
|
|
252
|
+
sleep_for = random.uniform(0, cap)
|
|
253
|
+
logger.warning(
|
|
254
|
+
f"ainvoke transient error {file_or_url}: attempt {attempt}/{max_retries}: "
|
|
255
|
+
f"{type(last_err).__name__}: {last_err}. Retrying in {sleep_for:.2f}s"
|
|
256
|
+
)
|
|
257
|
+
await asyncio.sleep(sleep_for)
|
|
265
258
|
|
|
266
259
|
@staticmethod
|
|
267
260
|
def _is_transient_error(e: Exception, retry_http_statuses: Tuple[int, ...]) -> bool:
|
|
@@ -443,7 +436,7 @@ class GPTVision:
|
|
|
443
436
|
)
|
|
444
437
|
except asyncio.TimeoutError:
|
|
445
438
|
logger.error(
|
|
446
|
-
f"ocr_page:
|
|
439
|
+
f"ocr_page: {file_stem} - page {page_index} timed out after {self.page_timeout_s}s"
|
|
447
440
|
)
|
|
448
441
|
partial_md = "[Timed out]"
|
|
449
442
|
|
|
@@ -455,7 +448,7 @@ class GPTVision:
|
|
|
455
448
|
logger.warning(
|
|
456
449
|
f"ocr_page: {file_stem} - page {page_index}: no OCR result"
|
|
457
450
|
)
|
|
458
|
-
partial_md = "
|
|
451
|
+
partial_md = ""
|
|
459
452
|
return (page_index, f"## Page {page_index + 1}\n\n{partial_md}")
|
|
460
453
|
finally:
|
|
461
454
|
await self._safe_unlink(png_path)
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import os
|
|
3
|
-
|
|
4
|
-
log_level = int(os.getenv("LOG_LEVEL", "20"))
|
|
5
|
-
logger = logging.getLogger(__name__)
|
|
6
|
-
logging.basicConfig(level=log_level, format="[%(asctime)s] [%(levelname)s] %(message)s")
|
|
7
|
-
|
|
8
|
-
# Quiet the Azure SDK HTTP logging
|
|
9
|
-
logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(logging.WARNING)
|
|
10
|
-
logging.getLogger("azure.core.pipeline.policies._universal").setLevel(logging.WARNING)
|
|
11
|
-
|
|
12
|
-
# 2) Azure Monitor exporter "Transmission succeeded..." spam
|
|
13
|
-
logging.getLogger("azure.monitor.opentelemetry.exporter").setLevel(logging.WARNING)
|
|
14
|
-
logging.getLogger("azure.monitor.opentelemetry.exporter._transmission").setLevel(logging.WARNING)
|
|
15
|
-
|
|
16
|
-
# 3) Quiet the rest of the Azure SDK + OTel + urllib3 connection noise
|
|
17
|
-
logging.getLogger("azure").setLevel(logging.WARNING)
|
|
18
|
-
logging.getLogger("opentelemetry").setLevel(logging.WARNING)
|
|
19
|
-
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
|
20
|
-
logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/azure_doc_intel_wrapper.py
RENAMED
|
File without changes
|
{markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/azure_speech_wrapper.py
RENAMED
|
File without changes
|
|
File without changes
|
{markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/gpt_vision_wrapper.py
RENAMED
|
File without changes
|
{markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/markitdown_wrapper.py
RENAMED
|
File without changes
|
|
File without changes
|
{markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/converters/unstructured_wrapper.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{markitdown_pro-1.0.2 → markitdown_pro-1.0.4}/markitdown_pro/services/azure_doc_intelligence.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|