natural-pdf 0.1.27__py3-none-any.whl → 0.1.30__py3-none-any.whl

{natural_pdf-0.1.27.dist-info → natural_pdf-0.1.30.dist-info}/RECORD

@@ -1,7 +1,11 @@
+bad_pdf_analysis/analyze_10_more.py,sha256=UjsTuHE1GUMoVjkX3afy3x6DfpXyfZXHgS2W1GQqUmw,11906
+bad_pdf_analysis/analyze_final_10.py,sha256=xYkIId0nF9LpWHRLDP1_nlJfJfC0b0Tu4mLu-3mim-0,25170
+bad_pdf_analysis/analyze_specific_pages.py,sha256=wzq3_ZWR28hFdT7GEkayHPYgsk20OpD476LYmy2rAEk,13725
+bad_pdf_analysis/analyze_specific_pages_direct.py,sha256=307gSNplwOtNTR9a0lEQWxlAKGeoZIcDe5z1pROKUXY,14846
 natural_pdf/__init__.py,sha256=qDFJNF8sbEDO-2WSFAxoWEM8updOUP6dB-ckya0kxfs,3275
 natural_pdf/cli.py,sha256=IXrP2lCHihr-ed-CFiDbMTnSsutQa1j1PYALOLGbpsc,4019
 natural_pdf/analyzers/__init__.py,sha256=dIXjsMqoxKmd9OOnSBzn12wvdIz7D7YNQRAnXslpJSM,142
-natural_pdf/analyzers/shape_detection_mixin.py,sha256=aHn4EMdbwOe8VWECPceGs5wN7gJP_kIxyAbmbNlNPSs,83634
+natural_pdf/analyzers/shape_detection_mixin.py,sha256=0a4uuKQ4Z1Ta_UVuUtX7mVhlwmXdAkoHTyC5wZyp5do,94455
 natural_pdf/analyzers/text_options.py,sha256=qEkDaYWla0rIM_gszEOsu52q7C_dAfV81P2HLJZM2sw,3333
 natural_pdf/analyzers/text_structure.py,sha256=VfKTsTFrK877sC0grsis9jK3rrgp0Mbp13VWEbukTcs,28437
 natural_pdf/analyzers/utils.py,sha256=PYbzJzSAHZ7JsMes84WIrSbA0zkjJGs0CLvIeINsf_k,2100
@@ -9,37 +13,38 @@ natural_pdf/analyzers/layout/__init__.py,sha256=oq1uJ5UkGGMbBKGirV1aRKK3hxAUyjTL
 natural_pdf/analyzers/layout/base.py,sha256=bYawhmc_0xqKG-xbxUSiazIU1om-aBox5Jh8qDqv-eM,6451
 natural_pdf/analyzers/layout/docling.py,sha256=4BJYyNVR6VegZGxyisvNIBBRvVk6YKPyDVs7ZdVfzEU,12676
 natural_pdf/analyzers/layout/gemini.py,sha256=ldECVCQ5HNQA3Omjg2NOsTrJXslyYb0vErDncmLIiuE,10510
-natural_pdf/analyzers/layout/layout_analyzer.py,sha256=n327Zjuf7aSzKQKChPHeiCVHinzeDGaWNyKiwQ-DkJk,15571
-natural_pdf/analyzers/layout/layout_manager.py,sha256=sDnh7XE-Wx2EBmgjipbvaLZQ7VSG6MfjEKfNsNXPNHs,8583
+natural_pdf/analyzers/layout/layout_analyzer.py,sha256=1v23FVCIGzkoiyRqiLZBwGZssBFKphtMossMENMuMxE,15519
+natural_pdf/analyzers/layout/layout_manager.py,sha256=vDXBAaNwvp68CRcEPH58MGLxx01OdVgzOh7Uv53L6fs,10319
 natural_pdf/analyzers/layout/layout_options.py,sha256=-Nv6bcu4_pqSCN6uNhCZ9mvoCBtRDZIUkO6kjkuLXsg,7703
 natural_pdf/analyzers/layout/paddle.py,sha256=tX2bI1yayAdmRhvsfZ_Ygs7zAG5e9eW-pLJkw4NUpBQ,21325
 natural_pdf/analyzers/layout/pdfplumber_table_finder.py,sha256=Tk0Q7wv7nGYPo69lh6RoezjdepTnMl90SaNIrP29Pwc,5902
-natural_pdf/analyzers/layout/surya.py,sha256=4RdnhRxSS3i3Ns5mFhOA9-P0xd7Ms19uZuKvUGQfEBI,9789
+natural_pdf/analyzers/layout/surya.py,sha256=ugRXPIHiLoh65lfbbiXO317TbgdtQ-5kVN1nonEf4ws,9778
 natural_pdf/analyzers/layout/table_structure_utils.py,sha256=nISZDBd46RPYkFHxbQyIHwg9WweG4DslpoYJ31OMJYA,2768
 natural_pdf/analyzers/layout/tatr.py,sha256=cVr0ZyhY2mNLAKZ4DGMm-b7XNJpILKh8x8ZpyDeUhLk,15032
-natural_pdf/analyzers/layout/yolo.py,sha256=Iw8qsIOHg2lUP7z9GsmkOm3c9kJ-Ywk01Oej50kZgDw,8360
-natural_pdf/classification/manager.py,sha256=7HOyHdjMJtC9DfzI8OXAREnGDpXaAgSfTFVC42n3tVQ,18889
+natural_pdf/analyzers/layout/yolo.py,sha256=ruchj28sxar0DWDALwUz1j30z0CLIEp2QAs0gLVvC4E,8346
+natural_pdf/classification/manager.py,sha256=pOP2LvJpTBGItvdIODnk735DXq7F2qqxN4AKmBORM3c,21775
 natural_pdf/classification/mixin.py,sha256=nYpmHQ4BlrealdPtIJt-_idME5o-xKLKNuAdIHzWL6c,7580
 natural_pdf/classification/results.py,sha256=Mcay-xLBHbYoZ8U7f4gMj2IhhH_yORNEkZHWdWJzmsU,3239
 natural_pdf/collections/mixins.py,sha256=sj76Cn6EdBtb5f-bdAV-1qpdixX8tI4BzPccPiYLI1w,5117
 natural_pdf/collections/pdf_collection.py,sha256=HLlyakM--23ZOeHDPucoM6Tw3yUyMXm0SSoqJwxRc2E,30744
 natural_pdf/core/__init__.py,sha256=QC8H4M3KbXwMFiQORZ0pdPlzx1Ix6oKKQSS7Ib2KEaA,38
-natural_pdf/core/element_manager.py,sha256=_UdXu51sLi6STzc8Pj4k8R721G3yJixXDLuRHn3hmr8,25731
-natural_pdf/core/highlighting_service.py,sha256=DKoaxiiuQsWgtf6wSroMAIcFiqJOOF7dXhciYdQKdCw,38223
-natural_pdf/core/page.py,sha256=GqYfYiVkuL1M_GoPTcLL0yWFXISN38BUCdQIKyF6vJ0,122721
-natural_pdf/core/pdf.py,sha256=qsSW4RxOJRmCnweLPMs0NhzkRfiAVdghTgnh4D_wuO4,74295
+natural_pdf/core/element_manager.py,sha256=96v_w3kXhSUqRsJlX5Bl6O6hJzpYRqDn4xoyRsdqZ7o,49260
+natural_pdf/core/highlighting_service.py,sha256=WKDqRpex1yS8CWhkNitWtKhxbyRRCLu3Xsct_HTPsD4,40774
+natural_pdf/core/page.py,sha256=kQKKqsbOaNeLhW3ark6mueDS-4tsopJcGcoMmKPK6B8,125624
+natural_pdf/core/pdf.py,sha256=YfniZp54AyptzMyr7ZP8n617n4wlV28SPrajt32nNBk,74233
 natural_pdf/describe/__init__.py,sha256=B3zjuHjFI_dFuBLgXR1Q4v7c72fVDyk84d2hs0H4KV8,561
-natural_pdf/describe/base.py,sha256=mUvEydumXXPJ2FkWAYm1BbWrRWY81I0dMyQrEU32rmc,17256
-natural_pdf/describe/elements.py,sha256=xD8wwR1z5IKat7RIwoAwQRUEL6zJTEwcOKorF4F-xPg,12717
+natural_pdf/describe/base.py,sha256=HaWlHltb-dw6ug4mfR_iBLHWxr1OdPwLaUshXRxO7gg,18462
+natural_pdf/describe/elements.py,sha256=COvKF3B_RbAxXl5ORJDubV4C5PsiuSfuzD0ufPIJTFM,12983
 natural_pdf/describe/mixin.py,sha256=U0x6v8r57KQb8qC3VVo64hvhfXQWsti8vdKBM7AXnMo,3116
 natural_pdf/describe/summary.py,sha256=7FIF3zF6bzNx-gx4pCJr2XQFKiVzOEDnWsAYQ_mr9L0,7982
 natural_pdf/elements/__init__.py,sha256=S8XeiNWJ1WcgnyYKdYV1yxQlAxCCO3FfITT8MQwNbyk,41
-natural_pdf/elements/base.py,sha256=iw-Ab0o7eI69npt0gAxQvA14GPWHAAhkLrJ_JeKvIos,43309
-natural_pdf/elements/collections.py,sha256=JrM42VPRtDOJ9Q9KIR3SrcbamiiCHXI4nzTq2BBkeEk,124223
+natural_pdf/elements/base.py,sha256=VshU4RstdzONJFq_8UVIjT_lVOai0MwMFsSFrCN-IO8,47299
+natural_pdf/elements/collections.py,sha256=52Oac96svzm_QMJcVaItnCG9P7d6JMNiGEx9lHgDEQg,125915
+natural_pdf/elements/image.py,sha256=UjHNzCgDzOseQmLpkKshcxg51DPmWNIAVYxZ0TAMyUI,1423
 natural_pdf/elements/line.py,sha256=aQm4pDdlQSDAAXqrdg4AU-oTl9JCXgYuaJN0EYls6E0,4920
 natural_pdf/elements/rect.py,sha256=kiVa3e377ZnqIOXc89d9ZSY4EcmDxtccdtUw-HOQzpw,3796
-natural_pdf/elements/region.py,sha256=63rdyjOnbmsgTN1WMSOyQWQnvJRUYco9qTWLqBi3TBk,125498
-natural_pdf/elements/text.py,sha256=x163dnr2ZDEIE_WZXWH5hXJtoO-6cvTdA2BABcZd69U,14575
+natural_pdf/elements/region.py,sha256=v1PzWvQoGHGdn7SQiPf4Oq3hIGueIfYGwcZ05ZU6XPE,127692
+natural_pdf/elements/text.py,sha256=2neapKplef0FsAMYWr4OdICt-TmrZ3z9z0YBrX8FrSk,17738
 natural_pdf/export/mixin.py,sha256=L1q3MIEFWuvie4j4_EmW7GT3NerbZ1as0XMUoqTS7gM,5083
 natural_pdf/exporters/__init__.py,sha256=g1WRPCDVzceaUUsm8dchPhzdHFSjYM0NfRyc8iN0mtE,644
 natural_pdf/exporters/base.py,sha256=XhR1xlkHOh7suOuX7mWbsj1h2o1pZNet-OAS5YCJyeI,2115
@@ -55,18 +60,18 @@ natural_pdf/extraction/manager.py,sha256=sASPJZ5cWFsl8A4PyTjg2yqkyC00tRl6glfoFA6
 natural_pdf/extraction/mixin.py,sha256=_5wGnzOCEuRWhqdSUV1Lqo9HIi56YC4MWzbBxOkOEKU,23160
 natural_pdf/extraction/result.py,sha256=D5DhjxLW7IvhEkvsAP7Zs2YA8K4hyuoTg681CSn5qA0,1825
 natural_pdf/flows/__init__.py,sha256=cUN4A8hTDLZSRr4PO2W_lR4z6hWpbNG8Seox-IIcrLU,277
-natural_pdf/flows/collections.py,sha256=qGuSPFSPQF-wiYquG6STiSzg_o951MSsFEq_B44Jef8,28441
+natural_pdf/flows/collections.py,sha256=iF8SsfKKb-YVIGi3m-yMRnfKgo_0n_EGhojnYK24h-Q,28493
 natural_pdf/flows/element.py,sha256=mKzk3B7A7sWNvu4CDvAjLr3_ZFLt--ktrSNoLfLpFxU,23940
 natural_pdf/flows/flow.py,sha256=I61BpFVDQyo6ORsmoqoYiOEP1DBRp0vgDJjm_V8frhc,10562
-natural_pdf/flows/region.py,sha256=4U3S7pLEa3oCyPfS-hpD0lSXf8MWT-MdF9AsVvMJbWU,26670
+natural_pdf/flows/region.py,sha256=s_YAT_0KsrwUs73hhU9xr_35Ufr__XNhRjHSQkxcfYU,27647
 natural_pdf/ocr/__init__.py,sha256=VY8hhvDPf7Gh2lB-d2QRmghLLyTy6ydxlgo1cS4dOSk,2482
 natural_pdf/ocr/engine.py,sha256=ZBC1tZNM5EDbGDJJmZI9mNHr4nCMLEZvUFhiJq8GdF4,8741
 natural_pdf/ocr/engine_doctr.py,sha256=ptKrupMWoulZb-R93zr9btoe94JPWU7vlJuN7OBJEIM,17740
 natural_pdf/ocr/engine_easyocr.py,sha256=bWz6kHUgAJfe3rqdnZBAF-IPvw3B35DlvX5KDdFUtzo,9888
-natural_pdf/ocr/engine_paddle.py,sha256=0vobobjnsM1G3zihYL7f1roLlKKZWRwioxkGkgIxEUA,16159
+natural_pdf/ocr/engine_paddle.py,sha256=9tQZl1VqN6d_KEWUY_S9tfrDLiR4FCHMjgSRNwPlsu8,16152
 natural_pdf/ocr/engine_surya.py,sha256=lOvSbZk53VKFVxRmqcQzM_0dHVdwTkRGiDZ9AWCgL1Q,5951
-natural_pdf/ocr/ocr_factory.py,sha256=GkODuBmqNVECg4u1-KW6ZMfBgVndLkK1W5GM15faf8M,5318
-natural_pdf/ocr/ocr_manager.py,sha256=K2gpFo3e6RB1ouXOstlEAAYd14DbjBNt5RH6J7ZdDQY,14263
+natural_pdf/ocr/ocr_factory.py,sha256=qjGL3hm_nTzxjwYWP0JE7dCFXZjKN8Z7f9c0oqasb9M,5262
+natural_pdf/ocr/ocr_manager.py,sha256=jFJI8v3coapKpERoUlP-ptwguZG_Dl4VlclD0xQ6Us8,16192
 natural_pdf/ocr/ocr_options.py,sha256=l33QKu_93r-uwi3t_v8UH8pEgHo6HTVzP4tfmQFRF1w,5488
 natural_pdf/ocr/utils.py,sha256=OxuHwDbHWj6setvnC0QYwMHrAjxGkhmLzWHpMqqGupA,4397
 natural_pdf/qa/__init__.py,sha256=2u2KJcA71g1I0HnLD-j6yvDw1moAjo9kkLhhfoYRURM,166
@@ -79,9 +84,12 @@ natural_pdf/search/search_options.py,sha256=sq_e8_jSROicD94b_xtDtLnjEr_Zsy4icjzP
 natural_pdf/search/search_service_protocol.py,sha256=Dl-Q-CrutkhZwI69scbW9EWPeYM63qxB60_EA7YqIYo,6699
 natural_pdf/search/searchable_mixin.py,sha256=hqQ_AuID5eTGRCtKYdFLZ1zF35y73uk3x1M1VW9Il8U,23514
 natural_pdf/selectors/__init__.py,sha256=oZGeqSv53EqmIZOhcnawuaGGlRg1h79vArXuZCWKm4A,123
-natural_pdf/selectors/parser.py,sha256=nJJPACzFKBMomAZMXzI_L3EMz9ji1agkmxJ5eaij310,32880
+natural_pdf/selectors/parser.py,sha256=T9r7XZhM1cGSYQrc9amUHbFtX-zBqd9_YPK0scwCjAQ,34231
+natural_pdf/tables/__init__.py,sha256=y65LM2wnu81yzvOX-J_5NXiIK4vEUtHa3EM1xv-0ttQ,105
+natural_pdf/tables/result.py,sha256=OYc-MjnP-VRTVaY-pBt84E-d8N3AaqzwAud0hHt5sVY,3979
 natural_pdf/templates/__init__.py,sha256=jYBxzfi73vew0f6yhIh1MlRxw4F_TVN2hKQR0YXOFe0,20
 natural_pdf/utils/__init__.py,sha256=s3M8FggaK1P3EBYn6R_-HgSDjNc9C73gyKe1hihtNWg,43
+natural_pdf/utils/bidi_mirror.py,sha256=SAe5SnL-xG5Wyo3LtkMttLdsnQqZhzAebLc7BAe6LhQ,1150
 natural_pdf/utils/debug.py,sha256=RN7H3E6ph-GtxubCW6psW7TO8o2BxcNLiEzByTVR9fk,995
 natural_pdf/utils/highlighting.py,sha256=EIY6ihVGtUTS_DjWyxpnr_UXpcR4btC1KhSGQ9VUfKg,698
 natural_pdf/utils/identifiers.py,sha256=P7n6owcubnF8oAMa_UfYtENmIaJQdH_AMC9Jbs2bWXo,1117
@@ -89,12 +97,26 @@ natural_pdf/utils/locks.py,sha256=7HJqV0VsNcOfISnbw8goCKWP5ck11uSJo6T_x9XIPKI,21
 natural_pdf/utils/packaging.py,sha256=e7U2wWvpunlAWpPFexNkD_c4dYbPp5LcKo7og4bNGvk,22411
 natural_pdf/utils/reading_order.py,sha256=s3DsYq_3g_1YA07qhd4BGEjeIRTeyGtnwc_hNtSzwBY,7290
 natural_pdf/utils/text_extraction.py,sha256=mDeN1_VevNi3RwvFe48PM5vBh-A5WeBlYgP6lSjBaOk,10854
-natural_pdf/utils/visualization.py,sha256=30pRWQdsRJh2pSObh-brKVsFgC1n8tHmSrta_UDnVPw,8989
+natural_pdf/utils/visualization.py,sha256=n3IZpbY5cf9LItzGavBcNyVZZrrUVxjYnmqZHYPa7NU,9386
 natural_pdf/widgets/__init__.py,sha256=QTVaUmsw__FCweFYZebwPssQxxUFUMd0wpm_cUbGZJY,181
 natural_pdf/widgets/viewer.py,sha256=2VUY1TzWMDe9I-IVNOosKZ2LaqpjLB62ftMAdk-s6_8,24952
-natural_pdf-0.1.27.dist-info/licenses/LICENSE,sha256=9zfwINwJlarbDmdh6iJV4QUG54QSJlSAUcnC1YiC_Ns,1074
-natural_pdf-0.1.27.dist-info/METADATA,sha256=TpH1LQXwoM6iK9l9q0HcftgezJzePczCtmYfQnAMh5w,6684
-natural_pdf-0.1.27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-natural_pdf-0.1.27.dist-info/entry_points.txt,sha256=1R_KMv7g60UBBpRqGfw7bppsMNGdayR-iJlb9ohEk_8,81
-natural_pdf-0.1.27.dist-info/top_level.txt,sha256=Cyw1zmNDlUZfb5moU-WUWGprrwH7ln_8LDGdmMHF1xI,17
-natural_pdf-0.1.27.dist-info/RECORD,,
+natural_pdf-0.1.30.dist-info/licenses/LICENSE,sha256=9zfwINwJlarbDmdh6iJV4QUG54QSJlSAUcnC1YiC_Ns,1074
+optimization/memory_comparison.py,sha256=XEHtjduSmzXzxnsJMvemTcq-OAlvGUBAm5wwnOXq8TY,6524
+optimization/pdf_analyzer.py,sha256=G3XWhsEqIYbohEgTqz6wzxkAnOx4MkbvbSspx577-8w,19145
+optimization/performance_analysis.py,sha256=vVlFDywEXxhJLd9n2KVVqqQnS6rwWoHV_jlogboGF2k,13784
+optimization/test_cleanup_methods.py,sha256=B_zHiJr1hI8q-tdfBoFi0Jf5lj2PURjA_6teRBGoz8o,6277
+optimization/test_memory_fix.py,sha256=CWc0OSvFfKE0-nxqJOi_HAQc0GXUPKzkQbTeJp5UqxU,6364
+tools/rtl_smoke_test.py,sha256=-ogcbvNzumJasICP0NNQHk4Zb4M1VRx0TnGkJUQC7SM,3043
+tools/bad_pdf_eval/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
+tools/bad_pdf_eval/analyser.py,sha256=sR31aVVmTXRHS8uwLZXlPefTH2_lskxtAzuZwlhsyOo,13391
+tools/bad_pdf_eval/collate_summaries.py,sha256=Mcmf1OvVn0S0efj5ypk0syXKSrfUf6L5dowoGvOTgjU,5047
+tools/bad_pdf_eval/eval_suite.py,sha256=-MK-XLqBo1025sccwYL6tnf7mZ1ZEpxu6EsTYv2ppmU,4294
+tools/bad_pdf_eval/export_enrichment_csv.py,sha256=SMEm9WxFUN_RIf8AGfZfjGEmvBvrOw1q6ARMl-EazIU,1906
+tools/bad_pdf_eval/llm_enrich.py,sha256=PsFMymPc8BNck21T3vupTN18pLdum-A_OLoJEKr6f80,12234
+tools/bad_pdf_eval/reporter.py,sha256=LIhcguDZ5XKgb0WeJsyA7m0kcliebOohzveShvt_KmY,400
+tools/bad_pdf_eval/utils.py,sha256=FuxaPX6f26IjQXu1vP0a2i9h1jgJNbASb8mRyj5-elE,4849
+natural_pdf-0.1.30.dist-info/METADATA,sha256=4Jg-iXXt6zGNE4gSYE_nMF395JDzv1Dierh93x1Lklo,6711
+natural_pdf-0.1.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+natural_pdf-0.1.30.dist-info/entry_points.txt,sha256=1R_KMv7g60UBBpRqGfw7bppsMNGdayR-iJlb9ohEk_8,81
+natural_pdf-0.1.30.dist-info/top_level.txt,sha256=oZlRzSc3nZ9sV3L6kD_Di734Pp62ANrm46imFVa51qQ,58
+natural_pdf-0.1.30.dist-info/RECORD,,

natural_pdf-0.1.30.dist-info/top_level.txt

@@ -0,0 +1,6 @@
+bad_pdf_analysis
+natural_pdf
+optimization
+pdfs
+todo
+tools

optimization/memory_comparison.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+Memory comparison script to measure the effectiveness of the character duplication fix.
+
+This script compares memory usage before and after the optimization by:
+1. Testing with a text-heavy PDF
+2. Measuring detailed memory usage patterns  
+3. Calculating memory savings
+"""
+
+import gc
+import os
+import psutil
+import sys
+from pathlib import Path
+
+import natural_pdf as npdf
+
+
+def get_detailed_memory_info():
+    """Get detailed memory information"""
+    process = psutil.Process()
+    memory_info = process.memory_info()
+    return {
+        'rss_mb': memory_info.rss / 1024 / 1024,
+        'vms_mb': memory_info.vms / 1024 / 1024, 
+        'python_objects': len(gc.get_objects())
+    }
+
+
+def analyze_character_storage(page):
+    """Analyze how characters are stored in the page"""
+    # Force element loading
+    text_elements = page.find_all("text")
+    
+    total_char_indices = 0
+    total_char_dicts = 0
+    total_chars_in_words = 0
+    memory_efficient_words = 0
+    legacy_words = 0
+    
+    for element in text_elements:
+        if hasattr(element, '_char_indices') and element._char_indices:
+            memory_efficient_words += 1
+            total_char_indices += len(element._char_indices)
+            total_chars_in_words += len(element._char_indices)
+            
+        if hasattr(element, '_char_dicts') and element._char_dicts:
+            total_char_dicts += len(element._char_dicts)
+            if not (hasattr(element, '_char_indices') and element._char_indices):
+                legacy_words += 1
+                total_chars_in_words += len(element._char_dicts)
+    
+    # Get individual character elements
+    char_elements = []
+    if hasattr(page, '_element_mgr'):
+        char_elements = page._element_mgr.get_elements('chars')
+    
+    return {
+        'total_words': len(text_elements),
+        'memory_efficient_words': memory_efficient_words,
+        'legacy_words': legacy_words,
+        'total_char_elements': len(char_elements),
+        'total_char_indices': total_char_indices,
+        'total_char_dicts': total_char_dicts,
+        'total_chars_in_words': total_chars_in_words,
+        'estimated_duplication_ratio': total_char_dicts / max(len(char_elements), 1)
+    }
+
+
+def test_memory_optimization():
+    """Test the memory optimization with a real PDF"""
+    
+    # Test with the practice PDF
+    test_pdf = Path("pdfs/01-practice.pdf")
+    if not test_pdf.exists():
+        print(f"Test PDF not found: {test_pdf}")
+        return
+    
+    print("=" * 60)
+    print("MEMORY OPTIMIZATION ANALYSIS")
+    print("=" * 60)
+    
+    # Baseline memory
+    gc.collect()
+    baseline_memory = get_detailed_memory_info()
+    print(f"Baseline memory: {baseline_memory['rss_mb']:.2f} MB RSS, {baseline_memory['python_objects']:,} objects")
+    
+    # Load PDF
+    pdf = npdf.PDF(str(test_pdf))
+    page = pdf.pages[0]
+    
+    post_load_memory = get_detailed_memory_info()
+    print(f"After PDF load: {post_load_memory['rss_mb']:.2f} MB RSS, {post_load_memory['python_objects']:,} objects")
+    
+    # Analyze character storage
+    storage_analysis = analyze_character_storage(page)
+    
+    final_memory = get_detailed_memory_info()
+    print(f"After element load: {final_memory['rss_mb']:.2f} MB RSS, {final_memory['python_objects']:,} objects")
+    
+    print("\n" + "=" * 40)
+    print("CHARACTER STORAGE ANALYSIS")
+    print("=" * 40)
+    
+    print(f"Total words: {storage_analysis['total_words']}")
+    print(f"Memory-efficient words: {storage_analysis['memory_efficient_words']}")
+    print(f"Legacy words: {storage_analysis['legacy_words']}")
+    print(f"Total character elements: {storage_analysis['total_char_elements']}")
+    print(f"Character indices used: {storage_analysis['total_char_indices']}")
+    print(f"Character dicts stored: {storage_analysis['total_char_dicts']}")
+    print(f"Characters referenced by words: {storage_analysis['total_chars_in_words']}")
+    
+    # Calculate optimization metrics
+    duplication_ratio = storage_analysis['estimated_duplication_ratio']
+    optimization_percentage = storage_analysis['memory_efficient_words'] / max(storage_analysis['total_words'], 1) * 100
+    
+    print(f"\nOptimization metrics:")
+    print(f"- Duplication ratio: {duplication_ratio:.2f}x")
+    print(f"- Words using optimization: {optimization_percentage:.1f}%")
+    
+    # Memory savings estimation
+    memory_used = final_memory['rss_mb'] - baseline_memory['rss_mb']
+    chars_total = storage_analysis['total_char_elements']
+    
+    if chars_total > 0:
+        memory_per_char = memory_used / chars_total * 1024  # KB per char
+        print(f"- Memory per character: {memory_per_char:.2f} KB")
+        
+        # Estimate savings from eliminating _char_dicts duplication
+        duplicated_chars = storage_analysis['total_char_dicts']
+        if duplicated_chars > 0:
+            estimated_wasted_memory = duplicated_chars * memory_per_char / 1024  # MB
+            print(f"- Estimated memory saved by optimization: {estimated_wasted_memory:.2f} MB")
+            print(f"- Memory efficiency improvement: {estimated_wasted_memory / memory_used * 100:.1f}%")
+    
+    print(f"\nTotal memory used for page processing: {memory_used:.2f} MB")
+    
+    # Test functionality
+    print("\n" + "=" * 40)
+    print("FUNCTIONALITY VERIFICATION")
+    print("=" * 40)
+    
+    # Test character access
+    test_elements = page.find_all("text")[:3]
+    for i, element in enumerate(test_elements):
+        print(f"\nWord {i+1}: '{element.text[:30]}{'...' if len(element.text) > 30 else ''}'")
+        
+        if hasattr(element, '_char_indices') and element._char_indices:
+            chars = element.chars
+            print(f"  - Uses character indices: {len(element._char_indices)} indices -> {len(chars)} chars")
+            print(f"  - Memory optimization: ACTIVE")
+            
+            # Verify character access works
+            if chars:
+                first_char = chars[0]
+                print(f"  - First char: '{first_char.text}' at ({first_char.x0:.1f}, {first_char.top:.1f})")
+        
+        elif hasattr(element, '_char_dicts') and element._char_dicts:
+            print(f"  - Uses character dicts: {len(element._char_dicts)} dicts")
+            print(f"  - Memory optimization: LEGACY MODE")
+        
+        else:
+            print(f"  - No character data available")
+    
+    print("\n" + "=" * 60)
+    print("✅ MEMORY OPTIMIZATION ANALYSIS COMPLETE")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    test_memory_optimization()