content-extraction 0.4.2__tar.gz → 0.4.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {content_extraction-0.4.2 → content_extraction-0.4.4}/PKG-INFO +1 -1
  2. {content_extraction-0.4.2 → content_extraction-0.4.4}/pyproject.toml +1 -1
  3. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/split_and_create_digest.py +7 -1
  4. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction.egg-info/PKG-INFO +1 -1
  5. {content_extraction-0.4.2 → content_extraction-0.4.4}/README.md +0 -0
  6. {content_extraction-0.4.2 → content_extraction-0.4.4}/setup.cfg +0 -0
  7. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/__init__.py +0 -0
  8. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/common_std_io.py +0 -0
  9. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/do_ocr.py +0 -0
  10. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/extract_from_pptx.py +0 -0
  11. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/file_handlers.py +0 -0
  12. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/fix_ocr.py +0 -0
  13. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/logging_config.py +0 -0
  14. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/parse_html.py +0 -0
  15. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/process.py +0 -0
  16. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/process_document.sh +0 -0
  17. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/semantic_chunk_html.py +0 -0
  18. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction.egg-info/SOURCES.txt +0 -0
  19. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction.egg-info/dependency_links.txt +0 -0
  20. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction.egg-info/requires.txt +0 -0
  21. {content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction.egg-info/top_level.txt +0 -0
  22. {content_extraction-0.4.2 → content_extraction-0.4.4}/tests/test_section_parser.py +0 -0
  23. {content_extraction-0.4.2 → content_extraction-0.4.4}/tests/test_semantic_chunk_html.py +0 -0
{content_extraction-0.4.2 → content_extraction-0.4.4}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: content_extraction
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown
{content_extraction-0.4.2 → content_extraction-0.4.4}/pyproject.toml RENAMED
@@ -10,7 +10,7 @@ where = ["src"]
10
10
 
11
11
  [project]
12
12
  name = "content_extraction"
13
- version = "0.4.2"
13
+ version = "0.4.4"
14
14
  description = "Project dedicated to content extraction from unstructured files that contain some useful information."
15
15
  readme = "README.md"
16
16
  requires-python = ">=3.12"
{content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction/split_and_create_digest.py RENAMED
@@ -1,4 +1,5 @@
1
1
  from langdetect import detect
2
+ from langdetect.lang_detect_exception import LangDetectException
2
3
  import sys
3
4
  import argparse
4
5
  import hashlib
@@ -35,6 +36,7 @@ class ProcessResultNode:
35
36
  title: str
36
37
  text: str
37
38
  section_digest: SectionDigestNode
39
+ language: str
38
40
 
39
41
 
40
42
  def shorten_text(text: str, max_elements: int = 2, subsections: list[dict] | None = None) -> str:
@@ -89,7 +91,11 @@ def process_node(node: dict, parent_digest_hash: str | None = None) -> list[dict
89
91
  Recursively process a node and its subsections, returning a flat list of nodes.
90
92
  """
91
93
  text = node.get('text', '')
92
- language = detect(text)
94
+ try:
95
+ language = detect(text)
96
+ except LangDetectException:
97
+ logger.warning(f'Failed to detect language for {text[:128]=}')
98
+ language = None
93
99
  section_digest = generate_section_digest(node)
94
100
  digest_hash = compute_digest_hash(section_digest)
95
101
  result = ProcessResultNode(
{content_extraction-0.4.2 → content_extraction-0.4.4}/src/content_extraction.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: content_extraction
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown