kreuzberg 3.16.0__py3-none-any.whl → 3.17.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. kreuzberg/__init__.py +2 -0
  2. kreuzberg/_config.py +8 -9
  3. kreuzberg/_extractors/_base.py +0 -46
  4. kreuzberg/_extractors/_html.py +1 -1
  5. kreuzberg/_extractors/_pandoc.py +2 -2
  6. kreuzberg/_extractors/_pdf.py +4 -4
  7. kreuzberg/_gmft.py +2 -2
  8. kreuzberg/_language_detection.py +16 -39
  9. kreuzberg/_mcp/server.py +1 -1
  10. kreuzberg/_mime_types.py +1 -1
  11. kreuzberg/_ocr/_easyocr.py +4 -9
  12. kreuzberg/_ocr/_paddleocr.py +1 -1
  13. kreuzberg/_ocr/_tesseract.py +15 -25
  14. kreuzberg/_token_reduction/__init__.py +11 -0
  15. kreuzberg/_token_reduction/_reducer.py +439 -0
  16. kreuzberg/_token_reduction/_stopwords.py +116 -0
  17. kreuzberg/_token_reduction/stopwords/af_stopwords.json +53 -0
  18. kreuzberg/_token_reduction/stopwords/ar_stopwords.json +482 -0
  19. kreuzberg/_token_reduction/stopwords/bg_stopwords.json +261 -0
  20. kreuzberg/_token_reduction/stopwords/bn_stopwords.json +400 -0
  21. kreuzberg/_token_reduction/stopwords/br_stopwords.json +1205 -0
  22. kreuzberg/_token_reduction/stopwords/ca_stopwords.json +280 -0
  23. kreuzberg/_token_reduction/stopwords/cs_stopwords.json +425 -0
  24. kreuzberg/_token_reduction/stopwords/da_stopwords.json +172 -0
  25. kreuzberg/_token_reduction/stopwords/de_stopwords.json +622 -0
  26. kreuzberg/_token_reduction/stopwords/el_stopwords.json +849 -0
  27. kreuzberg/_token_reduction/stopwords/en_stopwords.json +1300 -0
  28. kreuzberg/_token_reduction/stopwords/eo_stopwords.json +175 -0
  29. kreuzberg/_token_reduction/stopwords/es_stopwords.json +734 -0
  30. kreuzberg/_token_reduction/stopwords/et_stopwords.json +37 -0
  31. kreuzberg/_token_reduction/stopwords/eu_stopwords.json +100 -0
  32. kreuzberg/_token_reduction/stopwords/fa_stopwords.json +801 -0
  33. kreuzberg/_token_reduction/stopwords/fi_stopwords.json +849 -0
  34. kreuzberg/_token_reduction/stopwords/fr_stopwords.json +693 -0
  35. kreuzberg/_token_reduction/stopwords/ga_stopwords.json +111 -0
  36. kreuzberg/_token_reduction/stopwords/gl_stopwords.json +162 -0
  37. kreuzberg/_token_reduction/stopwords/gu_stopwords.json +226 -0
  38. kreuzberg/_token_reduction/stopwords/ha_stopwords.json +41 -0
  39. kreuzberg/_token_reduction/stopwords/he_stopwords.json +196 -0
  40. kreuzberg/_token_reduction/stopwords/hi_stopwords.json +227 -0
  41. kreuzberg/_token_reduction/stopwords/hr_stopwords.json +181 -0
  42. kreuzberg/_token_reduction/stopwords/hu_stopwords.json +791 -0
  43. kreuzberg/_token_reduction/stopwords/hy_stopwords.json +47 -0
  44. kreuzberg/_token_reduction/stopwords/id_stopwords.json +760 -0
  45. kreuzberg/_token_reduction/stopwords/it_stopwords.json +634 -0
  46. kreuzberg/_token_reduction/stopwords/ja_stopwords.json +136 -0
  47. kreuzberg/_token_reduction/stopwords/kn_stopwords.json +84 -0
  48. kreuzberg/_token_reduction/stopwords/ko_stopwords.json +681 -0
  49. kreuzberg/_token_reduction/stopwords/ku_stopwords.json +64 -0
  50. kreuzberg/_token_reduction/stopwords/la_stopwords.json +51 -0
  51. kreuzberg/_token_reduction/stopwords/lt_stopwords.json +476 -0
  52. kreuzberg/_token_reduction/stopwords/lv_stopwords.json +163 -0
  53. kreuzberg/_token_reduction/stopwords/ml_stopwords.json +11 -0
  54. kreuzberg/_token_reduction/stopwords/mr_stopwords.json +101 -0
  55. kreuzberg/_token_reduction/stopwords/ms_stopwords.json +477 -0
  56. kreuzberg/_token_reduction/stopwords/ne_stopwords.json +490 -0
  57. kreuzberg/_token_reduction/stopwords/nl_stopwords.json +415 -0
  58. kreuzberg/_token_reduction/stopwords/no_stopwords.json +223 -0
  59. kreuzberg/_token_reduction/stopwords/pl_stopwords.json +331 -0
  60. kreuzberg/_token_reduction/stopwords/pt_stopwords.json +562 -0
  61. kreuzberg/_token_reduction/stopwords/ro_stopwords.json +436 -0
  62. kreuzberg/_token_reduction/stopwords/ru_stopwords.json +561 -0
  63. kreuzberg/_token_reduction/stopwords/si_stopwords.json +193 -0
  64. kreuzberg/_token_reduction/stopwords/sk_stopwords.json +420 -0
  65. kreuzberg/_token_reduction/stopwords/sl_stopwords.json +448 -0
  66. kreuzberg/_token_reduction/stopwords/so_stopwords.json +32 -0
  67. kreuzberg/_token_reduction/stopwords/st_stopwords.json +33 -0
  68. kreuzberg/_token_reduction/stopwords/sv_stopwords.json +420 -0
  69. kreuzberg/_token_reduction/stopwords/sw_stopwords.json +76 -0
  70. kreuzberg/_token_reduction/stopwords/ta_stopwords.json +129 -0
  71. kreuzberg/_token_reduction/stopwords/te_stopwords.json +54 -0
  72. kreuzberg/_token_reduction/stopwords/th_stopwords.json +118 -0
  73. kreuzberg/_token_reduction/stopwords/tl_stopwords.json +149 -0
  74. kreuzberg/_token_reduction/stopwords/tr_stopwords.json +506 -0
  75. kreuzberg/_token_reduction/stopwords/uk_stopwords.json +75 -0
  76. kreuzberg/_token_reduction/stopwords/ur_stopwords.json +519 -0
  77. kreuzberg/_token_reduction/stopwords/vi_stopwords.json +647 -0
  78. kreuzberg/_token_reduction/stopwords/yo_stopwords.json +62 -0
  79. kreuzberg/_token_reduction/stopwords/zh_stopwords.json +796 -0
  80. kreuzberg/_token_reduction/stopwords/zu_stopwords.json +31 -0
  81. kreuzberg/_types.py +50 -9
  82. kreuzberg/_utils/_image_preprocessing.py +1 -1
  83. kreuzberg/_utils/_ref.py +14 -6
  84. kreuzberg/exceptions.py +0 -1
  85. kreuzberg/extraction.py +33 -10
  86. {kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/METADATA +6 -5
  87. kreuzberg-3.17.1.dist-info/RECORD +128 -0
  88. kreuzberg-3.16.0.dist-info/RECORD +0 -61
  89. {kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/WHEEL +0 -0
  90. {kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/entry_points.txt +0 -0
  91. {kreuzberg-3.16.0.dist-info → kreuzberg-3.17.1.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,506 @@
1
+ [
2
+ "acaba",
3
+ "acep",
4
+ "adamakıllı",
5
+ "adeta",
6
+ "ait",
7
+ "altmýþ",
8
+ "altmış",
9
+ "altý",
10
+ "altı",
11
+ "ama",
12
+ "amma",
13
+ "anca",
14
+ "ancak",
15
+ "arada",
16
+ "artýk",
17
+ "aslında",
18
+ "aynen",
19
+ "ayrıca",
20
+ "az",
21
+ "açıkça",
22
+ "açıkçası",
23
+ "bana",
24
+ "bari",
25
+ "bazen",
26
+ "bazý",
27
+ "bazı",
28
+ "başkası",
29
+ "baţka",
30
+ "belki",
31
+ "ben",
32
+ "benden",
33
+ "beni",
34
+ "benim",
35
+ "beri",
36
+ "beriki",
37
+ "beþ",
38
+ "beş",
39
+ "beţ",
40
+ "bilcümle",
41
+ "bile",
42
+ "bin",
43
+ "binaen",
44
+ "binaenaleyh",
45
+ "bir",
46
+ "biraz",
47
+ "birazdan",
48
+ "birbiri",
49
+ "birden",
50
+ "birdenbire",
51
+ "biri",
52
+ "birice",
53
+ "birileri",
54
+ "birisi",
55
+ "birkaç",
56
+ "birkaçı",
57
+ "birkez",
58
+ "birlikte",
59
+ "birçok",
60
+ "birçoğu",
61
+ "birþey",
62
+ "birþeyi",
63
+ "birşey",
64
+ "birşeyi",
65
+ "birţey",
66
+ "bitevi",
67
+ "biteviye",
68
+ "bittabi",
69
+ "biz",
70
+ "bizatihi",
71
+ "bizce",
72
+ "bizcileyin",
73
+ "bizden",
74
+ "bize",
75
+ "bizi",
76
+ "bizim",
77
+ "bizimki",
78
+ "bizzat",
79
+ "boşuna",
80
+ "bu",
81
+ "buna",
82
+ "bunda",
83
+ "bundan",
84
+ "bunlar",
85
+ "bunları",
86
+ "bunların",
87
+ "bunu",
88
+ "bunun",
89
+ "buracıkta",
90
+ "burada",
91
+ "buradan",
92
+ "burası",
93
+ "böyle",
94
+ "böylece",
95
+ "böylecene",
96
+ "böylelikle",
97
+ "böylemesine",
98
+ "böylesine",
99
+ "büsbütün",
100
+ "bütün",
101
+ "cuk",
102
+ "cümlesi",
103
+ "da",
104
+ "daha",
105
+ "dahi",
106
+ "dahil",
107
+ "dahilen",
108
+ "daima",
109
+ "dair",
110
+ "dayanarak",
111
+ "de",
112
+ "defa",
113
+ "dek",
114
+ "demin",
115
+ "demincek",
116
+ "deminden",
117
+ "denli",
118
+ "derakap",
119
+ "derhal",
120
+ "derken",
121
+ "deđil",
122
+ "değil",
123
+ "değin",
124
+ "diye",
125
+ "diđer",
126
+ "diğer",
127
+ "diğeri",
128
+ "doksan",
129
+ "dokuz",
130
+ "dolayı",
131
+ "dolayısıyla",
132
+ "doğru",
133
+ "dört",
134
+ "edecek",
135
+ "eden",
136
+ "ederek",
137
+ "edilecek",
138
+ "ediliyor",
139
+ "edilmesi",
140
+ "ediyor",
141
+ "elbet",
142
+ "elbette",
143
+ "elli",
144
+ "emme",
145
+ "en",
146
+ "enikonu",
147
+ "epey",
148
+ "epeyce",
149
+ "epeyi",
150
+ "esasen",
151
+ "esnasında",
152
+ "etmesi",
153
+ "etraflı",
154
+ "etraflıca",
155
+ "etti",
156
+ "ettiği",
157
+ "ettiğini",
158
+ "evleviyetle",
159
+ "evvel",
160
+ "evvela",
161
+ "evvelce",
162
+ "evvelden",
163
+ "evvelemirde",
164
+ "evveli",
165
+ "eđer",
166
+ "eğer",
167
+ "fakat",
168
+ "filanca",
169
+ "gah",
170
+ "gayet",
171
+ "gayetle",
172
+ "gayri",
173
+ "gayrı",
174
+ "gelgelelim",
175
+ "gene",
176
+ "gerek",
177
+ "gerçi",
178
+ "geçende",
179
+ "geçenlerde",
180
+ "gibi",
181
+ "gibilerden",
182
+ "gibisinden",
183
+ "gine",
184
+ "göre",
185
+ "gırla",
186
+ "hakeza",
187
+ "halbuki",
188
+ "halen",
189
+ "halihazırda",
190
+ "haliyle",
191
+ "handiyse",
192
+ "hangi",
193
+ "hangisi",
194
+ "hani",
195
+ "hariç",
196
+ "hasebiyle",
197
+ "hasılı",
198
+ "hatta",
199
+ "hele",
200
+ "hem",
201
+ "henüz",
202
+ "hep",
203
+ "hepsi",
204
+ "her",
205
+ "herhangi",
206
+ "herkes",
207
+ "herkesin",
208
+ "hiç",
209
+ "hiçbir",
210
+ "hiçbiri",
211
+ "hoş",
212
+ "hulasaten",
213
+ "iken",
214
+ "iki",
215
+ "ila",
216
+ "ile",
217
+ "ilen",
218
+ "ilgili",
219
+ "ilk",
220
+ "illa",
221
+ "illaki",
222
+ "imdi",
223
+ "indinde",
224
+ "inen",
225
+ "insermi",
226
+ "ise",
227
+ "ister",
228
+ "itibaren",
229
+ "itibariyle",
230
+ "itibarıyla",
231
+ "iyi",
232
+ "iyice",
233
+ "iyicene",
234
+ "için",
235
+ "iş",
236
+ "işte",
237
+ "iţte",
238
+ "kadar",
239
+ "kaffesi",
240
+ "kah",
241
+ "kala",
242
+ "kanýmca",
243
+ "karşın",
244
+ "katrilyon",
245
+ "kaynak",
246
+ "kaçı",
247
+ "kelli",
248
+ "kendi",
249
+ "kendilerine",
250
+ "kendini",
251
+ "kendisi",
252
+ "kendisine",
253
+ "kendisini",
254
+ "kere",
255
+ "kez",
256
+ "keza",
257
+ "kezalik",
258
+ "keşke",
259
+ "keţke",
260
+ "ki",
261
+ "kim",
262
+ "kimden",
263
+ "kime",
264
+ "kimi",
265
+ "kimisi",
266
+ "kimse",
267
+ "kimsecik",
268
+ "kimsecikler",
269
+ "külliyen",
270
+ "kýrk",
271
+ "kýsaca",
272
+ "kırk",
273
+ "kısaca",
274
+ "lakin",
275
+ "leh",
276
+ "lütfen",
277
+ "maada",
278
+ "madem",
279
+ "mademki",
280
+ "mamafih",
281
+ "mebni",
282
+ "međer",
283
+ "meğer",
284
+ "meğerki",
285
+ "meğerse",
286
+ "milyar",
287
+ "milyon",
288
+ "mu",
289
+ "mü",
290
+ "mý",
291
+ "mı",
292
+ "nasýl",
293
+ "nasıl",
294
+ "nasılsa",
295
+ "nazaran",
296
+ "naşi",
297
+ "ne",
298
+ "neden",
299
+ "nedeniyle",
300
+ "nedenle",
301
+ "nedense",
302
+ "nerde",
303
+ "nerden",
304
+ "nerdeyse",
305
+ "nere",
306
+ "nerede",
307
+ "nereden",
308
+ "neredeyse",
309
+ "neresi",
310
+ "nereye",
311
+ "netekim",
312
+ "neye",
313
+ "neyi",
314
+ "neyse",
315
+ "nice",
316
+ "nihayet",
317
+ "nihayetinde",
318
+ "nitekim",
319
+ "niye",
320
+ "niçin",
321
+ "o",
322
+ "olan",
323
+ "olarak",
324
+ "oldu",
325
+ "olduklarını",
326
+ "oldukça",
327
+ "olduğu",
328
+ "olduğunu",
329
+ "olmadı",
330
+ "olmadığı",
331
+ "olmak",
332
+ "olması",
333
+ "olmayan",
334
+ "olmaz",
335
+ "olsa",
336
+ "olsun",
337
+ "olup",
338
+ "olur",
339
+ "olursa",
340
+ "oluyor",
341
+ "on",
342
+ "ona",
343
+ "onca",
344
+ "onculayın",
345
+ "onda",
346
+ "ondan",
347
+ "onlar",
348
+ "onlardan",
349
+ "onlari",
350
+ "onlarýn",
351
+ "onları",
352
+ "onların",
353
+ "onu",
354
+ "onun",
355
+ "oracık",
356
+ "oracıkta",
357
+ "orada",
358
+ "oradan",
359
+ "oranca",
360
+ "oranla",
361
+ "oraya",
362
+ "otuz",
363
+ "oysa",
364
+ "oysaki",
365
+ "pek",
366
+ "pekala",
367
+ "peki",
368
+ "pekçe",
369
+ "peyderpey",
370
+ "rağmen",
371
+ "sadece",
372
+ "sahi",
373
+ "sahiden",
374
+ "sana",
375
+ "sanki",
376
+ "sekiz",
377
+ "seksen",
378
+ "sen",
379
+ "senden",
380
+ "seni",
381
+ "senin",
382
+ "siz",
383
+ "sizden",
384
+ "sizi",
385
+ "sizin",
386
+ "sonra",
387
+ "sonradan",
388
+ "sonraları",
389
+ "sonunda",
390
+ "tabii",
391
+ "tam",
392
+ "tamam",
393
+ "tamamen",
394
+ "tamamıyla",
395
+ "tarafından",
396
+ "tek",
397
+ "trilyon",
398
+ "tüm",
399
+ "var",
400
+ "vardı",
401
+ "vasıtasıyla",
402
+ "ve",
403
+ "velev",
404
+ "velhasıl",
405
+ "velhasılıkelam",
406
+ "veya",
407
+ "veyahut",
408
+ "ya",
409
+ "yahut",
410
+ "yakinen",
411
+ "yakında",
412
+ "yakından",
413
+ "yakınlarda",
414
+ "yalnız",
415
+ "yalnızca",
416
+ "yani",
417
+ "yapacak",
418
+ "yapmak",
419
+ "yaptı",
420
+ "yaptıkları",
421
+ "yaptığı",
422
+ "yaptığını",
423
+ "yapılan",
424
+ "yapılması",
425
+ "yapıyor",
426
+ "yedi",
427
+ "yeniden",
428
+ "yenilerde",
429
+ "yerine",
430
+ "yetmiþ",
431
+ "yetmiş",
432
+ "yetmiţ",
433
+ "yine",
434
+ "yirmi",
435
+ "yok",
436
+ "yoksa",
437
+ "yoluyla",
438
+ "yüz",
439
+ "yüzünden",
440
+ "zarfında",
441
+ "zaten",
442
+ "zati",
443
+ "zira",
444
+ "çabuk",
445
+ "çabukça",
446
+ "çeşitli",
447
+ "çok",
448
+ "çokları",
449
+ "çoklarınca",
450
+ "çokluk",
451
+ "çoklukla",
452
+ "çokça",
453
+ "çoğu",
454
+ "çoğun",
455
+ "çoğunca",
456
+ "çoğunlukla",
457
+ "çünkü",
458
+ "öbür",
459
+ "öbürkü",
460
+ "öbürü",
461
+ "önce",
462
+ "önceden",
463
+ "önceleri",
464
+ "öncelikle",
465
+ "öteki",
466
+ "ötekisi",
467
+ "öyle",
468
+ "öylece",
469
+ "öylelikle",
470
+ "öylemesine",
471
+ "öz",
472
+ "üzere",
473
+ "üç",
474
+ "þey",
475
+ "þeyden",
476
+ "þeyi",
477
+ "þeyler",
478
+ "þu",
479
+ "þuna",
480
+ "þunda",
481
+ "þundan",
482
+ "þunu",
483
+ "şayet",
484
+ "şey",
485
+ "şeyden",
486
+ "şeyi",
487
+ "şeyler",
488
+ "şu",
489
+ "şuna",
490
+ "şuncacık",
491
+ "şunda",
492
+ "şundan",
493
+ "şunlar",
494
+ "şunları",
495
+ "şunu",
496
+ "şunun",
497
+ "şura",
498
+ "şuracık",
499
+ "şuracıkta",
500
+ "şurası",
501
+ "şöyle",
502
+ "ţayet",
503
+ "ţimdi",
504
+ "ţu",
505
+ "ţöyle"
506
+ ]
@@ -0,0 +1,75 @@
1
+ [
2
+ "авжеж",
3
+ "адже",
4
+ "але",
5
+ "б",
6
+ "без",
7
+ "був",
8
+ "була",
9
+ "були",
10
+ "було",
11
+ "бути",
12
+ "більш",
13
+ "вам",
14
+ "вас",
15
+ "весь",
16
+ "вздовж",
17
+ "ви",
18
+ "вниз",
19
+ "внизу",
20
+ "вона",
21
+ "вони",
22
+ "воно",
23
+ "все",
24
+ "всередині",
25
+ "всіх",
26
+ "від",
27
+ "він",
28
+ "да",
29
+ "давай",
30
+ "давати",
31
+ "де",
32
+ "дещо",
33
+ "для",
34
+ "до",
35
+ "з",
36
+ "завжди",
37
+ "замість",
38
+ "й",
39
+ "коли",
40
+ "ледве",
41
+ "майже",
42
+ "ми",
43
+ "навколо",
44
+ "навіть",
45
+ "нам",
46
+ "от",
47
+ "отже",
48
+ "отож",
49
+ "поза",
50
+ "про",
51
+ "під",
52
+ "та",
53
+ "так",
54
+ "такий",
55
+ "також",
56
+ "те",
57
+ "ти",
58
+ "тобто",
59
+ "тож",
60
+ "тощо",
61
+ "хоча",
62
+ "це",
63
+ "цей",
64
+ "чи",
65
+ "чого",
66
+ "що",
67
+ "як",
68
+ "який",
69
+ "якої",
70
+ "є",
71
+ "із",
72
+ "інших",
73
+ "їх",
74
+ "її"
75
+ ]