content-extraction 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. content_extraction/split_and_create_digest.py +7 -1
  2. {content_extraction-0.4.2.dist-info → content_extraction-0.4.4.dist-info}/METADATA +1 -1
  3. {content_extraction-0.4.2.dist-info → content_extraction-0.4.4.dist-info}/RECORD +5 -5
  4. {content_extraction-0.4.2.dist-info → content_extraction-0.4.4.dist-info}/WHEEL +0 -0
  5. {content_extraction-0.4.2.dist-info → content_extraction-0.4.4.dist-info}/top_level.txt +0 -0
content_extraction/split_and_create_digest.py CHANGED
@@ -1,4 +1,5 @@
1
1
  from langdetect import detect
2
+ from langdetect.lang_detect_exception import LangDetectException
2
3
  import sys
3
4
  import argparse
4
5
  import hashlib
@@ -35,6 +36,7 @@ class ProcessResultNode:
35
36
  title: str
36
37
  text: str
37
38
  section_digest: SectionDigestNode
39
+ language: str
38
40
 
39
41
 
40
42
  def shorten_text(text: str, max_elements: int = 2, subsections: list[dict] | None = None) -> str:
@@ -89,7 +91,11 @@ def process_node(node: dict, parent_digest_hash: str | None = None) -> list[dict
89
91
  Recursively process a node and its subsections, returning a flat list of nodes.
90
92
  """
91
93
  text = node.get('text', '')
92
- language = detect(text)
94
+ try:
95
+ language = detect(text)
96
+ except LangDetectException:
97
+ logger.warning(f'Failed to detect language for {text[:128]=}')
98
+ language = None
93
99
  section_digest = generate_section_digest(node)
94
100
  digest_hash = compute_digest_hash(section_digest)
95
101
  result = ProcessResultNode(
{content_extraction-0.4.2.dist-info → content_extraction-0.4.4.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: content_extraction
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown
{content_extraction-0.4.2.dist-info → content_extraction-0.4.4.dist-info}/RECORD RENAMED
@@ -9,8 +9,8 @@ content_extraction/parse_html.py,sha256=mOrZKXX59YcdWWhmbnoTnfXpwrg0znk38x0DMJIV
9
9
  content_extraction/process.py,sha256=iLcmSjWhEg_DbgnftnVIfybIeLCuTEI57gasot0MtDk,1809
10
10
  content_extraction/process_document.sh,sha256=QbQOrV7isiEyxin1PBNGYmCbfVQ_eW-JgsbuQV4VB2o,1106
11
11
  content_extraction/semantic_chunk_html.py,sha256=PpK2W2Fse5-SU7hBqE-JWEW_sT3cEPaKNRRD2orEs-k,5696
12
- content_extraction/split_and_create_digest.py,sha256=vW4lyeTlRzZcqJS15g8Xqq5IZB06unrUBnQV7RrFDmA,4342
13
- content_extraction-0.4.2.dist-info/METADATA,sha256=glN_ZSgjFvvwhIp0X49yKSxJ1Av1EqjTZONDu_FMHOo,6266
14
- content_extraction-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
15
- content_extraction-0.4.2.dist-info/top_level.txt,sha256=a0I0EwSzsyd3p_aAENozn9i4I3aBn12XtrbqIvfzZec,19
16
- content_extraction-0.4.2.dist-info/RECORD,,
12
+ content_extraction/split_and_create_digest.py,sha256=NF6S01HLyN4rg1X9kjWb_xRd9ZelWE_LP8Bv1UOP8qU,4565
13
+ content_extraction-0.4.4.dist-info/METADATA,sha256=CbGUuNYfxpo2FnrcEoMJRPwZyFUZyiKlkiXNT3LG2w0,6266
14
+ content_extraction-0.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
15
+ content_extraction-0.4.4.dist-info/top_level.txt,sha256=a0I0EwSzsyd3p_aAENozn9i4I3aBn12XtrbqIvfzZec,19
16
+ content_extraction-0.4.4.dist-info/RECORD,,