pdftext 0.3.2__tar.gz → 0.3.4__tar.gz

{pdftext-0.3.2 → pdftext-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pdftext
-Version: 0.3.2
+Version: 0.3.4
 Summary: Extract structured text from pdfs quickly
 Home-page: https://github.com/VikParuchuri/pdftext
 License: Apache-2.0
@@ -64,6 +64,7 @@ pdftext PDF_PATH --out_path output.txt --json
 - `--json` specifies json output
 - `--sort` will attempt to sort in reading order if specified.
 - `--pages` will specify pages (comma separated) to extract
+- `--keep_chars` will keep individual characters in the json output
 
 The output will be a json list, with each item in the list corresponding to a single page in the input pdf (in order).  Each page will include the following keys:
 
@@ -107,7 +108,7 @@ import pypdfium2 as pdfium
 from pdftext.extraction import dictionary_output
 
 pdf = pdfium.PdfDocument(PDF_PATH)
-text = dictionary_output(pdf, sort=False, page_range=[1,2,3]) # Optional arguments explained above
+text = dictionary_output(pdf, sort=False, page_range=[1,2,3], keep_chars=False) # Optional arguments explained above
 ```
 
 If you want more customization, check out the `pdftext.extraction._get_pages` function for a starting point to dig deeper.  pdftext is a pretty thin wrapper around [pypdfium2](https://pypdfium2.readthedocs.io/en/stable/), so you might want to look at the documentation for that as well.

{pdftext-0.3.2 → pdftext-0.3.4}/README.md

@@ -41,6 +41,7 @@ pdftext PDF_PATH --out_path output.txt --json
 - `--json` specifies json output
 - `--sort` will attempt to sort in reading order if specified.
 - `--pages` will specify pages (comma separated) to extract
+- `--keep_chars` will keep individual characters in the json output
 
 The output will be a json list, with each item in the list corresponding to a single page in the input pdf (in order).  Each page will include the following keys:
 
@@ -84,7 +85,7 @@ import pypdfium2 as pdfium
 from pdftext.extraction import dictionary_output
 
 pdf = pdfium.PdfDocument(PDF_PATH)
-text = dictionary_output(pdf, sort=False, page_range=[1,2,3]) # Optional arguments explained above
+text = dictionary_output(pdf, sort=False, page_range=[1,2,3], keep_chars=False) # Optional arguments explained above
 ```
 
 If you want more customization, check out the `pdftext.extraction._get_pages` function for a starting point to dig deeper.  pdftext is a pretty thin wrapper around [pypdfium2](https://pypdfium2.readthedocs.io/en/stable/), so you might want to look at the documentation for that as well.

{pdftext-0.3.2 → pdftext-0.3.4}/extract_text.py

@@ -13,6 +13,7 @@ def main():
     parser.add_argument("--sort", action="store_true", help="Attempt to sort the text by reading order", default=False)
     parser.add_argument("--keep_hyphens", action="store_true", help="Keep hyphens in words", default=False)
     parser.add_argument("--pages", type=str, help="Comma separated pages to extract, like 1,2,3", default=None)
+    parser.add_argument("--keep_chars", action="store_true", help="Keep character level information", default=False)
     args = parser.parse_args()
 
     pdf_doc = pdfium.PdfDocument(args.pdf_path)
@@ -22,7 +23,7 @@ def main():
         assert all(p <= len(pdf_doc) for p in pages), "Invalid page number(s) provided"
 
     if args.json:
-        text = dictionary_output(pdf_doc, sort=args.sort, page_range=pages)
+        text = dictionary_output(pdf_doc, sort=args.sort, page_range=pages, keep_chars=args.keep_chars)
         text = json.dumps(text)
     else:
         text = plain_text_output(pdf_doc, sort=args.sort, hyphens=args.keep_hyphens, page_range=pages)

{pdftext-0.3.2 → pdftext-0.3.4}/pdftext/extraction.py

@@ -28,7 +28,7 @@ def paginated_plain_text_output(pdf_doc, sort=False, model=None, hyphens=False,
     return text
 
 
-def dictionary_output(pdf_doc, sort=False, model=None, page_range=None):
+def dictionary_output(pdf_doc, sort=False, model=None, page_range=None, keep_chars=False):
     pages = _get_pages(pdf_doc, model, page_range)
     for page in pages:
         for block in page["blocks"]:
@@ -44,6 +44,12 @@ def dictionary_output(pdf_doc, sort=False, model=None, page_range=None):
                     span["text"] = postprocess_text(span["text"])
                     span["text"] = handle_hyphens(span["text"], keep_hyphens=True)
 
+                    if not keep_chars:
+                        del span["chars"]
+                    else:
+                        for char in span["chars"]:
+                            char["bbox"] = unnormalize_bbox(char["bbox"], page["width"], page["height"])
+
                 line["bbox"] = unnormalize_bbox(line["bbox"], page["width"], page["height"])
             block["bbox"] = unnormalize_bbox(block["bbox"], page["width"], page["height"])
         if sort:

{pdftext-0.3.2 → pdftext-0.3.4}/pdftext/inference.py

@@ -81,7 +81,11 @@ def update_span(line, span):
         span["char_start_idx"] = span["chars"][0]["char_idx"]
         span["char_end_idx"] = span["chars"][-1]["char_idx"]
 
-    del span["chars"]
+    # Remove unneeded keys from the characters
+    for char in span["chars"]:
+        del_keys = [k for k in list(char.keys()) if k not in ["char", "bbox"]]
+        for key in del_keys:
+            del char[key]
     line["spans"].append(span)
     span = {"chars": []}
     return span
@@ -129,7 +133,7 @@ def infer_single_page(text_chars, block_threshold=settings.BLOCK_THRESHOLD):
                 span = update_span(line, span)
                 line = update_line(block, line)
                 block = update_block(blocks, block)
-            elif prev_char["char"] in LINE_BREAKS and prediction_probs[1] >= .5: # Look for newline character as a forcing signal for a new line
+            elif prev_char["char"] in LINE_BREAKS: # Look for newline character as a forcing signal for a new line
                 span = update_span(line, span)
                 line = update_line(block, line)
             elif prev_font_info != font_info:

{pdftext-0.3.2 → pdftext-0.3.4}/pdftext/pdf/utils.py

@@ -37,13 +37,12 @@ def unnormalize_bbox(bbox, page_width, page_height):
 
 
 def get_fontname(textpage, char_index):
-    n_bytes = settings.FONT_BUFFER_SIZE
+    n_bytes = pdfium_c.FPDFText_GetFontInfo(textpage, char_index, None, 0, None)
     buffer = ctypes.create_string_buffer(n_bytes)
     # Re-interpret the type from char to unsigned short as required by the function
     buffer_ptr = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_ushort))
     flag_buffer = ctypes.c_int()
-    flag_ptr = ctypes.pointer(flag_buffer)
-    font_info = pdfium_c.FPDFText_GetFontInfo(textpage, char_index, buffer_ptr, n_bytes, flag_ptr)
+    font_info = pdfium_c.FPDFText_GetFontInfo(textpage, char_index, buffer_ptr, n_bytes, flag_buffer)
     if font_info == 0:
         return None, None
     try:
@@ -62,13 +61,11 @@ def page_to_device(page, x, y, page_width, page_height, page_rotation: int):
         page_rotation = 3
     else:
         page_rotation = 0
-    device_x = ctypes.c_int()
-    device_y = ctypes.c_int()
-    device_x_ptr = ctypes.pointer(device_x)
-    device_y_ptr = ctypes.pointer(device_y)
     width = math.ceil(page_width)
     height = math.ceil(page_height)
-    pdfium_c.FPDF_PageToDevice(page, 0, 0, width, height, page_rotation, x, y, device_x_ptr, device_y_ptr)
+    device_x = ctypes.c_int()
+    device_y = ctypes.c_int()
+    pdfium_c.FPDF_PageToDevice(page, 0, 0, width, height, page_rotation, x, y, device_x, device_y)
     x = device_x.value
     y = device_y.value
     return x, y
@@ -135,4 +132,4 @@ def rotate_page_bbox(bbox, angle_deg, width, height):
         bbox = rotate_page_bbox(bbox, 90, width, height)
         bbox = rotate_page_bbox(bbox, 180, width, height)
 
-    return bbox
+    return bbox

{pdftext-0.3.2 → pdftext-0.3.4}/pdftext/settings.py

@@ -8,7 +8,6 @@ class Settings(BaseSettings):
     MODEL_PATH: str = os.path.join(BASE_PATH, "models", "dt.joblib")
 
     # Fonts
-    FONT_BUFFER_SIZE: int = 1024 # How many characters to buffer when reading a font name
     FONTNAME_SAMPLE_FREQ: int = 10
 
     # Inference

{pdftext-0.3.2 → pdftext-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pdftext"
-version = "0.3.2"
+version = "0.3.4"
 description = "Extract structured text from pdfs quickly"
 authors = ["Vik Paruchuri <vik.paruchuri@gmail.com>"]
 license = "Apache-2.0"