content-extraction 0.4.3__tar.gz → 0.4.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {content_extraction-0.4.3 → content_extraction-0.4.4}/PKG-INFO +1 -1
  2. {content_extraction-0.4.3 → content_extraction-0.4.4}/pyproject.toml +1 -1
  3. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/split_and_create_digest.py +6 -1
  4. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction.egg-info/PKG-INFO +1 -1
  5. {content_extraction-0.4.3 → content_extraction-0.4.4}/README.md +0 -0
  6. {content_extraction-0.4.3 → content_extraction-0.4.4}/setup.cfg +0 -0
  7. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/__init__.py +0 -0
  8. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/common_std_io.py +0 -0
  9. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/do_ocr.py +0 -0
  10. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/extract_from_pptx.py +0 -0
  11. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/file_handlers.py +0 -0
  12. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/fix_ocr.py +0 -0
  13. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/logging_config.py +0 -0
  14. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/parse_html.py +0 -0
  15. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/process.py +0 -0
  16. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/process_document.sh +0 -0
  17. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/semantic_chunk_html.py +0 -0
  18. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction.egg-info/SOURCES.txt +0 -0
  19. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction.egg-info/dependency_links.txt +0 -0
  20. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction.egg-info/requires.txt +0 -0
  21. {content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction.egg-info/top_level.txt +0 -0
  22. {content_extraction-0.4.3 → content_extraction-0.4.4}/tests/test_section_parser.py +0 -0
  23. {content_extraction-0.4.3 → content_extraction-0.4.4}/tests/test_semantic_chunk_html.py +0 -0
{content_extraction-0.4.3 → content_extraction-0.4.4}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: content_extraction
3
- Version: 0.4.3
3
+ Version: 0.4.4
4
4
  Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown
{content_extraction-0.4.3 → content_extraction-0.4.4}/pyproject.toml RENAMED
@@ -10,7 +10,7 @@ where = ["src"]
10
10
 
11
11
  [project]
12
12
  name = "content_extraction"
13
- version = "0.4.3"
13
+ version = "0.4.4"
14
14
  description = "Project dedicated to content extraction from unstructured files that contain some useful information."
15
15
  readme = "README.md"
16
16
  requires-python = ">=3.12"
{content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction/split_and_create_digest.py RENAMED
@@ -1,4 +1,5 @@
1
1
  from langdetect import detect
2
+ from langdetect.lang_detect_exception import LangDetectException
2
3
  import sys
3
4
  import argparse
4
5
  import hashlib
@@ -90,7 +91,11 @@ def process_node(node: dict, parent_digest_hash: str | None = None) -> list[dict
90
91
  Recursively process a node and its subsections, returning a flat list of nodes.
91
92
  """
92
93
  text = node.get('text', '')
93
- language = detect(text)
94
+ try:
95
+ language = detect(text)
96
+ except LangDetectException:
97
+ logger.warning(f'Failed to detect language for {text[:128]=}')
98
+ language = None
94
99
  section_digest = generate_section_digest(node)
95
100
  digest_hash = compute_digest_hash(section_digest)
96
101
  result = ProcessResultNode(
{content_extraction-0.4.3 → content_extraction-0.4.4}/src/content_extraction.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: content_extraction
3
- Version: 0.4.3
3
+ Version: 0.4.4
4
4
  Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown