pensiev 0.25.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. memos/__init__.py +6 -0
  2. memos/cmds/__init__.py +0 -0
  3. memos/cmds/library.py +1289 -0
  4. memos/cmds/plugin.py +96 -0
  5. memos/commands.py +865 -0
  6. memos/config.py +225 -0
  7. memos/crud.py +605 -0
  8. memos/databases/__init__.py +0 -0
  9. memos/databases/initializers.py +481 -0
  10. memos/dataset_extractor_for_florence.py +165 -0
  11. memos/dataset_extractor_for_internvl2.py +192 -0
  12. memos/default_config.yaml +88 -0
  13. memos/embedding.py +129 -0
  14. memos/frame_extractor.py +53 -0
  15. memos/logging_config.py +35 -0
  16. memos/main.py +104 -0
  17. memos/migrations/alembic/README +1 -0
  18. memos/migrations/alembic/__pycache__/env.cpython-310.pyc +0 -0
  19. memos/migrations/alembic/env.py +108 -0
  20. memos/migrations/alembic/script.py.mako +30 -0
  21. memos/migrations/alembic/versions/00904ac8c6fc_add_indexes_to_entitymodel.py +63 -0
  22. memos/migrations/alembic/versions/04acdaf75664_add_indices_to_entitytags_and_metadata.py +86 -0
  23. memos/migrations/alembic/versions/12504c5b1d3c_add_extra_columns_for_embedding.py +67 -0
  24. memos/migrations/alembic/versions/31a1ad0e10b3_add_entity_plugin_status.py +71 -0
  25. memos/migrations/alembic/versions/__pycache__/00904ac8c6fc_add_indexes_to_entitymodel.cpython-310.pyc +0 -0
  26. memos/migrations/alembic/versions/__pycache__/04acdaf75664_add_indices_to_entitytags_and_metadata.cpython-310.pyc +0 -0
  27. memos/migrations/alembic/versions/__pycache__/12504c5b1d3c_add_extra_columns_for_embedding.cpython-310.pyc +0 -0
  28. memos/migrations/alembic/versions/__pycache__/20f5ecab014d_add_entity_plugin_status.cpython-310.pyc +0 -0
  29. memos/migrations/alembic/versions/__pycache__/31a1ad0e10b3_add_entity_plugin_status.cpython-310.pyc +0 -0
  30. memos/migrations/alembic/versions/__pycache__/4fcb062c5128_add_extra_columns_for_embedding.cpython-310.pyc +0 -0
  31. memos/migrations/alembic/versions/__pycache__/d10c55fbb7d2_add_index_for_entity_file_type_group_.cpython-310.pyc +0 -0
  32. memos/migrations/alembic/versions/__pycache__/f8f158182416_add_active_app_index.cpython-310.pyc +0 -0
  33. memos/migrations/alembic/versions/d10c55fbb7d2_add_index_for_entity_file_type_group_.py +44 -0
  34. memos/migrations/alembic/versions/f8f158182416_add_active_app_index.py +75 -0
  35. memos/migrations/alembic.ini +116 -0
  36. memos/migrations.py +19 -0
  37. memos/models.py +199 -0
  38. memos/plugins/__init__.py +0 -0
  39. memos/plugins/ocr/__init__.py +0 -0
  40. memos/plugins/ocr/main.py +251 -0
  41. memos/plugins/ocr/models/ch_PP-OCRv4_det_infer.onnx +0 -0
  42. memos/plugins/ocr/models/ch_PP-OCRv4_rec_infer.onnx +0 -0
  43. memos/plugins/ocr/models/ch_ppocr_mobile_v2.0_cls_train.onnx +0 -0
  44. memos/plugins/ocr/ppocr-gpu.yaml +43 -0
  45. memos/plugins/ocr/ppocr.yaml +44 -0
  46. memos/plugins/ocr/server.py +227 -0
  47. memos/plugins/ocr/temp_ppocr.yaml +42 -0
  48. memos/plugins/vlm/__init__.py +0 -0
  49. memos/plugins/vlm/main.py +251 -0
  50. memos/prepare_dataset.py +107 -0
  51. memos/process_webp.py +55 -0
  52. memos/read_metadata.py +32 -0
  53. memos/record.py +358 -0
  54. memos/schemas.py +289 -0
  55. memos/search.py +1198 -0
  56. memos/server.py +883 -0
  57. memos/shotsum.py +105 -0
  58. memos/shotsum_with_ocr.py +145 -0
  59. memos/simple_tokenizer/dict/README.md +31 -0
  60. memos/simple_tokenizer/dict/hmm_model.utf8 +34 -0
  61. memos/simple_tokenizer/dict/idf.utf8 +258826 -0
  62. memos/simple_tokenizer/dict/jieba.dict.utf8 +348982 -0
  63. memos/simple_tokenizer/dict/pos_dict/char_state_tab.utf8 +6653 -0
  64. memos/simple_tokenizer/dict/pos_dict/prob_emit.utf8 +166 -0
  65. memos/simple_tokenizer/dict/pos_dict/prob_start.utf8 +259 -0
  66. memos/simple_tokenizer/dict/pos_dict/prob_trans.utf8 +5222 -0
  67. memos/simple_tokenizer/dict/stop_words.utf8 +1534 -0
  68. memos/simple_tokenizer/dict/user.dict.utf8 +4 -0
  69. memos/simple_tokenizer/linux/libsimple.so +0 -0
  70. memos/simple_tokenizer/macos/libsimple.dylib +0 -0
  71. memos/simple_tokenizer/windows/simple.dll +0 -0
  72. memos/static/_app/immutable/assets/0.e250c031.css +1 -0
  73. memos/static/_app/immutable/assets/_layout.e7937cfe.css +1 -0
  74. memos/static/_app/immutable/chunks/index.5c08976b.js +1 -0
  75. memos/static/_app/immutable/chunks/index.60ee613b.js +4 -0
  76. memos/static/_app/immutable/chunks/runtime.a7926cf6.js +5 -0
  77. memos/static/_app/immutable/chunks/scheduler.5c1cff6e.js +1 -0
  78. memos/static/_app/immutable/chunks/singletons.583bdf4e.js +1 -0
  79. memos/static/_app/immutable/entry/app.666c1643.js +1 -0
  80. memos/static/_app/immutable/entry/start.aed5c701.js +3 -0
  81. memos/static/_app/immutable/nodes/0.5862ea38.js +7 -0
  82. memos/static/_app/immutable/nodes/1.35378a5e.js +1 -0
  83. memos/static/_app/immutable/nodes/2.1ccf9ea5.js +81 -0
  84. memos/static/_app/version.json +1 -0
  85. memos/static/app.html +36 -0
  86. memos/static/favicon.png +0 -0
  87. memos/static/logos/memos_logo_1024.png +0 -0
  88. memos/static/logos/memos_logo_1024@2x.png +0 -0
  89. memos/static/logos/memos_logo_128.png +0 -0
  90. memos/static/logos/memos_logo_128@2x.png +0 -0
  91. memos/static/logos/memos_logo_16.png +0 -0
  92. memos/static/logos/memos_logo_16@2x.png +0 -0
  93. memos/static/logos/memos_logo_256.png +0 -0
  94. memos/static/logos/memos_logo_256@2x.png +0 -0
  95. memos/static/logos/memos_logo_32.png +0 -0
  96. memos/static/logos/memos_logo_32@2x.png +0 -0
  97. memos/static/logos/memos_logo_512.png +0 -0
  98. memos/static/logos/memos_logo_512@2x.png +0 -0
  99. memos/static/logos/memos_logo_64.png +0 -0
  100. memos/static/logos/memos_logo_64@2x.png +0 -0
  101. memos/test_server.py +802 -0
  102. memos/utils.py +49 -0
  103. memos_ml_backends/florence2_server.py +176 -0
  104. memos_ml_backends/qwen2vl_server.py +182 -0
  105. memos_ml_backends/schemas.py +48 -0
  106. pensiev-0.25.5.dist-info/LICENSE +201 -0
  107. pensiev-0.25.5.dist-info/METADATA +541 -0
  108. pensiev-0.25.5.dist-info/RECORD +111 -0
  109. pensiev-0.25.5.dist-info/WHEEL +5 -0
  110. pensiev-0.25.5.dist-info/entry_points.txt +2 -0
  111. pensiev-0.25.5.dist-info/top_level.txt +2 -0
memos/simple_tokenizer/dict/pos_dict/prob_start.utf8 ADDED
@@ -0,0 +1,259 @@
1
+ #初始状态的概率
2
+ #格式
3
+ #状态:概率
4
+ B,a:-4.7623052146
5
+ B,ad:-6.68006603678
6
+ B,ag:-3.14e+100
7
+ B,an:-8.69708322302
8
+ B,b:-5.01837436211
9
+ B,bg:-3.14e+100
10
+ B,c:-3.42388018495
11
+ B,d:-3.97504752976
12
+ B,df:-8.88897423083
13
+ B,dg:-3.14e+100
14
+ B,e:-8.56355183039
15
+ B,en:-3.14e+100
16
+ B,f:-5.49163041848
17
+ B,g:-3.14e+100
18
+ B,h:-13.53336513
19
+ B,i:-6.11578472756
20
+ B,in:-3.14e+100
21
+ B,j:-5.05761912847
22
+ B,jn:-3.14e+100
23
+ B,k:-3.14e+100
24
+ B,l:-4.90588358466
25
+ B,ln:-3.14e+100
26
+ B,m:-3.6524299819
27
+ B,mg:-3.14e+100
28
+ B,mq:-6.7869530014
29
+ B,n:-1.69662577975
30
+ B,ng:-3.14e+100
31
+ B,nr:-2.23104959138
32
+ B,nrfg:-5.87372217541
33
+ B,nrt:-4.98564273352
34
+ B,ns:-2.8228438315
35
+ B,nt:-4.84609166818
36
+ B,nz:-3.94698846058
37
+ B,o:-8.43349870215
38
+ B,p:-4.20098413209
39
+ B,q:-6.99812385896
40
+ B,qe:-3.14e+100
41
+ B,qg:-3.14e+100
42
+ B,r:-3.40981877908
43
+ B,rg:-3.14e+100
44
+ B,rr:-12.4347528413
45
+ B,rz:-7.94611647157
46
+ B,s:-5.52267359084
47
+ B,t:-3.36474790945
48
+ B,tg:-3.14e+100
49
+ B,u:-9.1639172775
50
+ B,ud:-3.14e+100
51
+ B,ug:-3.14e+100
52
+ B,uj:-3.14e+100
53
+ B,ul:-3.14e+100
54
+ B,uv:-3.14e+100
55
+ B,uz:-3.14e+100
56
+ B,v:-2.67405848743
57
+ B,vd:-9.04472876024
58
+ B,vg:-3.14e+100
59
+ B,vi:-12.4347528413
60
+ B,vn:-4.33156108902
61
+ B,vq:-12.1470707689
62
+ B,w:-3.14e+100
63
+ B,x:-3.14e+100
64
+ B,y:-9.84448567586
65
+ B,yg:-3.14e+100
66
+ B,z:-7.04568111149
67
+ B,zg:-3.14e+100
68
+ E,a:-3.14e+100
69
+ E,ad:-3.14e+100
70
+ E,ag:-3.14e+100
71
+ E,an:-3.14e+100
72
+ E,b:-3.14e+100
73
+ E,bg:-3.14e+100
74
+ E,c:-3.14e+100
75
+ E,d:-3.14e+100
76
+ E,df:-3.14e+100
77
+ E,dg:-3.14e+100
78
+ E,e:-3.14e+100
79
+ E,en:-3.14e+100
80
+ E,f:-3.14e+100
81
+ E,g:-3.14e+100
82
+ E,h:-3.14e+100
83
+ E,i:-3.14e+100
84
+ E,in:-3.14e+100
85
+ E,j:-3.14e+100
86
+ E,jn:-3.14e+100
87
+ E,k:-3.14e+100
88
+ E,l:-3.14e+100
89
+ E,ln:-3.14e+100
90
+ E,m:-3.14e+100
91
+ E,mg:-3.14e+100
92
+ E,mq:-3.14e+100
93
+ E,n:-3.14e+100
94
+ E,ng:-3.14e+100
95
+ E,nr:-3.14e+100
96
+ E,nrfg:-3.14e+100
97
+ E,nrt:-3.14e+100
98
+ E,ns:-3.14e+100
99
+ E,nt:-3.14e+100
100
+ E,nz:-3.14e+100
101
+ E,o:-3.14e+100
102
+ E,p:-3.14e+100
103
+ E,q:-3.14e+100
104
+ E,qe:-3.14e+100
105
+ E,qg:-3.14e+100
106
+ E,r:-3.14e+100
107
+ E,rg:-3.14e+100
108
+ E,rr:-3.14e+100
109
+ E,rz:-3.14e+100
110
+ E,s:-3.14e+100
111
+ E,t:-3.14e+100
112
+ E,tg:-3.14e+100
113
+ E,u:-3.14e+100
114
+ E,ud:-3.14e+100
115
+ E,ug:-3.14e+100
116
+ E,uj:-3.14e+100
117
+ E,ul:-3.14e+100
118
+ E,uv:-3.14e+100
119
+ E,uz:-3.14e+100
120
+ E,v:-3.14e+100
121
+ E,vd:-3.14e+100
122
+ E,vg:-3.14e+100
123
+ E,vi:-3.14e+100
124
+ E,vn:-3.14e+100
125
+ E,vq:-3.14e+100
126
+ E,w:-3.14e+100
127
+ E,x:-3.14e+100
128
+ E,y:-3.14e+100
129
+ E,yg:-3.14e+100
130
+ E,z:-3.14e+100
131
+ E,zg:-3.14e+100
132
+ M,a:-3.14e+100
133
+ M,ad:-3.14e+100
134
+ M,ag:-3.14e+100
135
+ M,an:-3.14e+100
136
+ M,b:-3.14e+100
137
+ M,bg:-3.14e+100
138
+ M,c:-3.14e+100
139
+ M,d:-3.14e+100
140
+ M,df:-3.14e+100
141
+ M,dg:-3.14e+100
142
+ M,e:-3.14e+100
143
+ M,en:-3.14e+100
144
+ M,f:-3.14e+100
145
+ M,g:-3.14e+100
146
+ M,h:-3.14e+100
147
+ M,i:-3.14e+100
148
+ M,in:-3.14e+100
149
+ M,j:-3.14e+100
150
+ M,jn:-3.14e+100
151
+ M,k:-3.14e+100
152
+ M,l:-3.14e+100
153
+ M,ln:-3.14e+100
154
+ M,m:-3.14e+100
155
+ M,mg:-3.14e+100
156
+ M,mq:-3.14e+100
157
+ M,n:-3.14e+100
158
+ M,ng:-3.14e+100
159
+ M,nr:-3.14e+100
160
+ M,nrfg:-3.14e+100
161
+ M,nrt:-3.14e+100
162
+ M,ns:-3.14e+100
163
+ M,nt:-3.14e+100
164
+ M,nz:-3.14e+100
165
+ M,o:-3.14e+100
166
+ M,p:-3.14e+100
167
+ M,q:-3.14e+100
168
+ M,qe:-3.14e+100
169
+ M,qg:-3.14e+100
170
+ M,r:-3.14e+100
171
+ M,rg:-3.14e+100
172
+ M,rr:-3.14e+100
173
+ M,rz:-3.14e+100
174
+ M,s:-3.14e+100
175
+ M,t:-3.14e+100
176
+ M,tg:-3.14e+100
177
+ M,u:-3.14e+100
178
+ M,ud:-3.14e+100
179
+ M,ug:-3.14e+100
180
+ M,uj:-3.14e+100
181
+ M,ul:-3.14e+100
182
+ M,uv:-3.14e+100
183
+ M,uz:-3.14e+100
184
+ M,v:-3.14e+100
185
+ M,vd:-3.14e+100
186
+ M,vg:-3.14e+100
187
+ M,vi:-3.14e+100
188
+ M,vn:-3.14e+100
189
+ M,vq:-3.14e+100
190
+ M,w:-3.14e+100
191
+ M,x:-3.14e+100
192
+ M,y:-3.14e+100
193
+ M,yg:-3.14e+100
194
+ M,z:-3.14e+100
195
+ M,zg:-3.14e+100
196
+ S,a:-3.90253968313
197
+ S,ad:-11.0484584802
198
+ S,ag:-6.95411391796
199
+ S,an:-12.8402179494
200
+ S,b:-6.47288876397
201
+ S,bg:-3.14e+100
202
+ S,c:-4.78696679586
203
+ S,d:-3.90391976418
204
+ S,df:-3.14e+100
205
+ S,dg:-8.9483976513
206
+ S,e:-5.94251300628
207
+ S,en:-3.14e+100
208
+ S,f:-5.19482024998
209
+ S,g:-6.50782681533
210
+ S,h:-8.65056320738
211
+ S,i:-3.14e+100
212
+ S,in:-3.14e+100
213
+ S,j:-4.91199211964
214
+ S,jn:-3.14e+100
215
+ S,k:-6.94032059583
216
+ S,l:-3.14e+100
217
+ S,ln:-3.14e+100
218
+ S,m:-3.26920065212
219
+ S,mg:-10.8253149289
220
+ S,mq:-3.14e+100
221
+ S,n:-3.85514838976
222
+ S,ng:-4.9134348611
223
+ S,nr:-4.48366310396
224
+ S,nrfg:-3.14e+100
225
+ S,nrt:-3.14e+100
226
+ S,ns:-3.14e+100
227
+ S,nt:-12.1470707689
228
+ S,nz:-3.14e+100
229
+ S,o:-8.46446092775
230
+ S,p:-2.98684018136
231
+ S,q:-4.88865861826
232
+ S,qe:-3.14e+100
233
+ S,qg:-3.14e+100
234
+ S,r:-2.76353367841
235
+ S,rg:-10.2752685919
236
+ S,rr:-3.14e+100
237
+ S,rz:-3.14e+100
238
+ S,s:-3.14e+100
239
+ S,t:-3.14e+100
240
+ S,tg:-6.27284253188
241
+ S,u:-6.94032059583
242
+ S,ud:-7.72823016105
243
+ S,ug:-7.53940370266
244
+ S,uj:-6.85251045118
245
+ S,ul:-8.41537131755
246
+ S,uv:-8.15808672229
247
+ S,uz:-9.29925862537
248
+ S,v:-3.05329230341
249
+ S,vd:-3.14e+100
250
+ S,vg:-5.94301818437
251
+ S,vi:-3.14e+100
252
+ S,vn:-11.4539235883
253
+ S,vq:-3.14e+100
254
+ S,w:-3.14e+100
255
+ S,x:-8.42741965607
256
+ S,y:-6.19707946995
257
+ S,yg:-13.53336513
258
+ S,z:-3.14e+100
259
+ S,zg:-3.14e+100