PyPI - kreuzberg - Versions diffs - 3.16.0__py3-none-any.whl → 3.17.1__py3-none-any.whl - Mend

kreuzberg 3.16.0py3-none-any.whl → 3.17.1py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (91) hide show

kreuzberg/__init__.py +2 -0
kreuzberg/_config.py +8 -9
kreuzberg/_extractors/_base.py +0 -46
kreuzberg/_extractors/_html.py +1 -1
kreuzberg/_extractors/_pandoc.py +2 -2
kreuzberg/_extractors/_pdf.py +4 -4
kreuzberg/_gmft.py +2 -2
kreuzberg/_language_detection.py +16 -39
kreuzberg/_mcp/server.py +1 -1
kreuzberg/_mime_types.py +1 -1
kreuzberg/_ocr/_easyocr.py +4 -9
kreuzberg/_ocr/_paddleocr.py +1 -1
kreuzberg/_ocr/_tesseract.py +15 -25
kreuzberg/_token_reduction/__init__.py +11 -0
kreuzberg/_token_reduction/_reducer.py +439 -0
kreuzberg/_token_reduction/_stopwords.py +116 -0
kreuzberg/_token_reduction/stopwords/af_stopwords.json +53 -0
kreuzberg/_token_reduction/stopwords/ar_stopwords.json +482 -0
kreuzberg/_token_reduction/stopwords/bg_stopwords.json +261 -0
kreuzberg/_token_reduction/stopwords/bn_stopwords.json +400 -0
kreuzberg/_token_reduction/stopwords/br_stopwords.json +1205 -0
kreuzberg/_token_reduction/stopwords/ca_stopwords.json +280 -0
kreuzberg/_token_reduction/stopwords/cs_stopwords.json +425 -0
kreuzberg/_token_reduction/stopwords/da_stopwords.json +172 -0
kreuzberg/_token_reduction/stopwords/de_stopwords.json +622 -0
kreuzberg/_token_reduction/stopwords/el_stopwords.json +849 -0
kreuzberg/_token_reduction/stopwords/en_stopwords.json +1300 -0
kreuzberg/_token_reduction/stopwords/eo_stopwords.json +175 -0
kreuzberg/_token_reduction/stopwords/es_stopwords.json +734 -0
kreuzberg/_token_reduction/stopwords/et_stopwords.json +37 -0
kreuzberg/_token_reduction/stopwords/eu_stopwords.json +100 -0
kreuzberg/_token_reduction/stopwords/fa_stopwords.json +801 -0
kreuzberg/_token_reduction/stopwords/fi_stopwords.json +849 -0
kreuzberg/_token_reduction/stopwords/fr_stopwords.json +693 -0
kreuzberg/_token_reduction/stopwords/ga_stopwords.json +111 -0
kreuzberg/_token_reduction/stopwords/gl_stopwords.json +162 -0
kreuzberg/_token_reduction/stopwords/gu_stopwords.json +226 -0
kreuzberg/_token_reduction/stopwords/ha_stopwords.json +41 -0
kreuzberg/_token_reduction/stopwords/he_stopwords.json +196 -0
kreuzberg/_token_reduction/stopwords/hi_stopwords.json +227 -0
kreuzberg/_token_reduction/stopwords/hr_stopwords.json +181 -0
kreuzberg/_token_reduction/stopwords/hu_stopwords.json +791 -0
kreuzberg/_token_reduction/stopwords/hy_stopwords.json +47 -0
kreuzberg/_token_reduction/stopwords/id_stopwords.json +760 -0
kreuzberg/_token_reduction/stopwords/it_stopwords.json +634 -0
kreuzberg/_token_reduction/stopwords/ja_stopwords.json +136 -0
kreuzberg/_token_reduction/stopwords/kn_stopwords.json +84 -0
kreuzberg/_token_reduction/stopwords/ko_stopwords.json +681 -0
kreuzberg/_token_reduction/stopwords/ku_stopwords.json +64 -0
kreuzberg/_token_reduction/stopwords/la_stopwords.json +51 -0
kreuzberg/_token_reduction/stopwords/lt_stopwords.json +476 -0
kreuzberg/_token_reduction/stopwords/lv_stopwords.json +163 -0
kreuzberg/_token_reduction/stopwords/ml_stopwords.json +11 -0
kreuzberg/_token_reduction/stopwords/mr_stopwords.json +101 -0
kreuzberg/_token_reduction/stopwords/ms_stopwords.json +477 -0
kreuzberg/_token_reduction/stopwords/ne_stopwords.json +490 -0
kreuzberg/_token_reduction/stopwords/nl_stopwords.json +415 -0
kreuzberg/_token_reduction/stopwords/no_stopwords.json +223 -0
kreuzberg/_token_reduction/stopwords/pl_stopwords.json +331 -0
kreuzberg/_token_reduction/stopwords/pt_stopwords.json +562 -0
kreuzberg/_token_reduction/stopwords/ro_stopwords.json +436 -0
kreuzberg/_token_reduction/stopwords/ru_stopwords.json +561 -0
kreuzberg/_token_reduction/stopwords/si_stopwords.json +193 -0
kreuzberg/_token_reduction/stopwords/sk_stopwords.json +420 -0
kreuzberg/_token_reduction/stopwords/sl_stopwords.json +448 -0
kreuzberg/_token_reduction/stopwords/so_stopwords.json +32 -0
kreuzberg/_token_reduction/stopwords/st_stopwords.json +33 -0
kreuzberg/_token_reduction/stopwords/sv_stopwords.json +420 -0
kreuzberg/_token_reduction/stopwords/sw_stopwords.json +76 -0
kreuzberg/_token_reduction/stopwords/ta_stopwords.json +129 -0
kreuzberg/_token_reduction/stopwords/te_stopwords.json +54 -0
kreuzberg/_token_reduction/stopwords/th_stopwords.json +118 -0
kreuzberg/_token_reduction/stopwords/tl_stopwords.json +149 -0
kreuzberg/_token_reduction/stopwords/tr_stopwords.json +506 -0
kreuzberg/_token_reduction/stopwords/uk_stopwords.json +75 -0
kreuzberg/_token_reduction/stopwords/ur_stopwords.json +519 -0
kreuzberg/_token_reduction/stopwords/vi_stopwords.json +647 -0
kreuzberg/_token_reduction/stopwords/yo_stopwords.json +62 -0
kreuzberg/_token_reduction/stopwords/zh_stopwords.json +796 -0
kreuzberg/_token_reduction/stopwords/zu_stopwords.json +31 -0
kreuzberg/_types.py +50 -9
kreuzberg/_utils/_image_preprocessing.py +1 -1
kreuzberg/_utils/_ref.py +14 -6
kreuzberg/exceptions.py +0 -1
kreuzberg/extraction.py +33 -10
{kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/METADATA +6 -5
kreuzberg-3.17.1.dist-info/RECORD +128 -0
kreuzberg-3.16.0.dist-info/RECORD +0 -61
{kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/WHEEL +0 -0
{kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/entry_points.txt +0 -0
{kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/licenses/LICENSE +0 -0

kreuzberg/_token_reduction/stopwords/th_stopwords.json ADDED Viewed

@@ -0,0 +1,118 @@
+[
+  "กล่าว",
+  "กว่า",
+  "กัน",
+  "กับ",
+  "การ",
+  "ก็",
+  "ก่อน",
+  "ขณะ",
+  "ขอ",
+  "ของ",
+  "ขึ้น",
+  "คง",
+  "ครั้ง",
+  "ความ",
+  "คือ",
+  "จะ",
+  "จัด",
+  "จาก",
+  "จึง",
+  "ช่วง",
+  "ซึ่ง",
+  "ดัง",
+  "ด้วย",
+  "ด้าน",
+  "ตั้ง",
+  "ตั้งแต่",
+  "ตาม",
+  "ต่อ",
+  "ต่าง",
+  "ต่างๆ",
+  "ต้อง",
+  "ถึง",
+  "ถูก",
+  "ถ้า",
+  "ทั้ง",
+  "ทั้งนี้",
+  "ทาง",
+  "ที่",
+  "ที่สุด",
+  "ทุก",
+  "ทํา",
+  "ทําให้",
+  "นอกจาก",
+  "นัก",
+  "นั้น",
+  "นี้",
+  "น่า",
+  "นํา",
+  "บาง",
+  "ผล",
+  "ผ่าน",
+  "พบ",
+  "พร้อม",
+  "มา",
+  "มาก",
+  "มี",
+  "ยัง",
+  "รวม",
+  "ระหว่าง",
+  "รับ",
+  "ราย",
+  "ร่วม",
+  "ลง",
+  "วัน",
+  "ว่า",
+  "สุด",
+  "ส่ง",
+  "ส่วน",
+  "สําหรับ",
+  "หนึ่ง",
+  "หรือ",
+  "หลัง",
+  "หลังจาก",
+  "หลาย",
+  "หาก",
+  "อยาก",
+  "อยู่",
+  "อย่าง",
+  "ออก",
+  "อะไร",
+  "อาจ",
+  "อีก",
+  "เขา",
+  "เข้า",
+  "เคย",
+  "เฉพาะ",
+  "เช่น",
+  "เดียว",
+  "เดียวกัน",
+  "เนื่องจาก",
+  "เปิด",
+  "เปิดเผย",
+  "เป็น",
+  "เป็นการ",
+  "เพราะ",
+  "เพื่อ",
+  "เมื่อ",
+  "เรา",
+  "เริ่ม",
+  "เลย",
+  "เห็น",
+  "เอง",
+  "แต่",
+  "แบบ",
+  "แรก",
+  "และ",
+  "แล้ว",
+  "แห่ง",
+  "โดย",
+  "ใน",
+  "ให้",
+  "ได้",
+  "ไป",
+  "ไม่",
+  "ไว้",
+  "้ง"
+]

kreuzberg/_token_reduction/stopwords/tl_stopwords.json ADDED Viewed

@@ -0,0 +1,149 @@
+[
+  "akin",
+  "aking",
+  "ako",
+  "alin",
+  "am",
+  "amin",
+  "aming",
+  "ang",
+  "ano",
+  "anumang",
+  "apat",
+  "at",
+  "atin",
+  "ating",
+  "ay",
+  "bababa",
+  "bago",
+  "bakit",
+  "bawat",
+  "bilang",
+  "dahil",
+  "dalawa",
+  "dapat",
+  "din",
+  "dito",
+  "doon",
+  "gagawin",
+  "gayunman",
+  "ginagawa",
+  "ginawa",
+  "ginawang",
+  "gumawa",
+  "gusto",
+  "habang",
+  "hanggang",
+  "hindi",
+  "huwag",
+  "iba",
+  "ibaba",
+  "ibabaw",
+  "ibig",
+  "ikaw",
+  "ilagay",
+  "ilalim",
+  "ilan",
+  "inyong",
+  "isa",
+  "isang",
+  "itaas",
+  "ito",
+  "iyo",
+  "iyon",
+  "iyong",
+  "ka",
+  "kahit",
+  "kailangan",
+  "kailanman",
+  "kami",
+  "kanila",
+  "kanilang",
+  "kanino",
+  "kanya",
+  "kanyang",
+  "kapag",
+  "kapwa",
+  "karamihan",
+  "katiyakan",
+  "katulad",
+  "kaya",
+  "kaysa",
+  "ko",
+  "kong",
+  "kulang",
+  "kumuha",
+  "kung",
+  "laban",
+  "lahat",
+  "lamang",
+  "likod",
+  "lima",
+  "maaari",
+  "maaaring",
+  "maging",
+  "mahusay",
+  "makita",
+  "marami",
+  "marapat",
+  "masyado",
+  "may",
+  "mayroon",
+  "mga",
+  "minsan",
+  "mismo",
+  "mula",
+  "muli",
+  "na",
+  "nabanggit",
+  "naging",
+  "nagkaroon",
+  "nais",
+  "nakita",
+  "namin",
+  "napaka",
+  "narito",
+  "nasaan",
+  "ng",
+  "ngayon",
+  "ni",
+  "nila",
+  "nilang",
+  "nito",
+  "niya",
+  "niyang",
+  "noon",
+  "o",
+  "pa",
+  "paano",
+  "pababa",
+  "paggawa",
+  "pagitan",
+  "pagkakaroon",
+  "pagkatapos",
+  "palabas",
+  "pamamagitan",
+  "panahon",
+  "pangalawa",
+  "para",
+  "paraan",
+  "pareho",
+  "pataas",
+  "pero",
+  "pumunta",
+  "pumupunta",
+  "sa",
+  "saan",
+  "sabi",
+  "sabihin",
+  "sarili",
+  "sila",
+  "sino",
+  "siya",
+  "tatlo",
+  "tayo",
+  "tulad",
+  "tungkol",
+  "una",
+  "walang"
+]

kreuzberg 3.16.0__py3-none-any.whl → 3.17.1__py3-none-any.whl

kreuzberg 3.16.0py3-none-any.whl → 3.17.1py3-none-any.whl