content-extraction 0.4.3__py3-none-any.whl → 0.4.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. content_extraction/split_and_create_digest.py +6 -1
  2. {content_extraction-0.4.3.dist-info → content_extraction-0.4.4.dist-info}/METADATA +1 -1
  3. {content_extraction-0.4.3.dist-info → content_extraction-0.4.4.dist-info}/RECORD +5 -5
  4. {content_extraction-0.4.3.dist-info → content_extraction-0.4.4.dist-info}/WHEEL +0 -0
  5. {content_extraction-0.4.3.dist-info → content_extraction-0.4.4.dist-info}/top_level.txt +0 -0
content_extraction/split_and_create_digest.py CHANGED
@@ -1,4 +1,5 @@
1
1
  from langdetect import detect
2
+ from langdetect.lang_detect_exception import LangDetectException
2
3
  import sys
3
4
  import argparse
4
5
  import hashlib
@@ -90,7 +91,11 @@ def process_node(node: dict, parent_digest_hash: str | None = None) -> list[dict
90
91
  Recursively process a node and its subsections, returning a flat list of nodes.
91
92
  """
92
93
  text = node.get('text', '')
93
- language = detect(text)
94
+ try:
95
+ language = detect(text)
96
+ except LangDetectException:
97
+ logger.warning(f'Failed to detect language for {text[:128]=}')
98
+ language = None
94
99
  section_digest = generate_section_digest(node)
95
100
  digest_hash = compute_digest_hash(section_digest)
96
101
  result = ProcessResultNode(
{content_extraction-0.4.3.dist-info → content_extraction-0.4.4.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: content_extraction
3
- Version: 0.4.3
3
+ Version: 0.4.4
4
4
  Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown
{content_extraction-0.4.3.dist-info → content_extraction-0.4.4.dist-info}/RECORD RENAMED
@@ -9,8 +9,8 @@ content_extraction/parse_html.py,sha256=mOrZKXX59YcdWWhmbnoTnfXpwrg0znk38x0DMJIV
9
9
  content_extraction/process.py,sha256=iLcmSjWhEg_DbgnftnVIfybIeLCuTEI57gasot0MtDk,1809
10
10
  content_extraction/process_document.sh,sha256=QbQOrV7isiEyxin1PBNGYmCbfVQ_eW-JgsbuQV4VB2o,1106
11
11
  content_extraction/semantic_chunk_html.py,sha256=PpK2W2Fse5-SU7hBqE-JWEW_sT3cEPaKNRRD2orEs-k,5696
12
- content_extraction/split_and_create_digest.py,sha256=FzCtlCbdGoqe0KAyQ2oiIUVjAGmkf5LTRPaZj7v94vY,4360
13
- content_extraction-0.4.3.dist-info/METADATA,sha256=GJNyTCnhnWjrkmLd-xpHlTOWw0s6ra3TLIak3fCPQ6w,6266
14
- content_extraction-0.4.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
15
- content_extraction-0.4.3.dist-info/top_level.txt,sha256=a0I0EwSzsyd3p_aAENozn9i4I3aBn12XtrbqIvfzZec,19
16
- content_extraction-0.4.3.dist-info/RECORD,,
12
+ content_extraction/split_and_create_digest.py,sha256=NF6S01HLyN4rg1X9kjWb_xRd9ZelWE_LP8Bv1UOP8qU,4565
13
+ content_extraction-0.4.4.dist-info/METADATA,sha256=CbGUuNYfxpo2FnrcEoMJRPwZyFUZyiKlkiXNT3LG2w0,6266
14
+ content_extraction-0.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
15
+ content_extraction-0.4.4.dist-info/top_level.txt,sha256=a0I0EwSzsyd3p_aAENozn9i4I3aBn12XtrbqIvfzZec,19
16
+ content_extraction-0.4.4.dist-info/RECORD,,