nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (431) hide show
  1. migrations/0002_rollover_shards.py +1 -2
  2. migrations/0003_allfields_key.py +2 -37
  3. migrations/0004_rollover_shards.py +1 -2
  4. migrations/0005_rollover_shards.py +1 -2
  5. migrations/0006_rollover_shards.py +2 -4
  6. migrations/0008_cleanup_leftover_rollover_metadata.py +1 -2
  7. migrations/0009_upgrade_relations_and_texts_to_v2.py +5 -4
  8. migrations/0010_fix_corrupt_indexes.py +11 -12
  9. migrations/0011_materialize_labelset_ids.py +2 -18
  10. migrations/0012_rollover_shards.py +6 -12
  11. migrations/0013_rollover_shards.py +2 -4
  12. migrations/0014_rollover_shards.py +5 -7
  13. migrations/0015_targeted_rollover.py +6 -12
  14. migrations/0016_upgrade_to_paragraphs_v2.py +27 -32
  15. migrations/0017_multiple_writable_shards.py +3 -6
  16. migrations/0018_purge_orphan_kbslugs.py +59 -0
  17. migrations/0019_upgrade_to_paragraphs_v3.py +66 -0
  18. migrations/0020_drain_nodes_from_cluster.py +83 -0
  19. nucliadb/standalone/tests/unit/test_run.py → migrations/0021_overwrite_vectorsets_key.py +17 -18
  20. nucliadb/tests/unit/test_openapi.py → migrations/0022_fix_paragraph_deletion_bug.py +16 -11
  21. migrations/0023_backfill_pg_catalog.py +80 -0
  22. migrations/0025_assign_models_to_kbs_v2.py +113 -0
  23. migrations/0026_fix_high_cardinality_content_types.py +61 -0
  24. migrations/0027_rollover_texts3.py +73 -0
  25. nucliadb/ingest/fields/date.py → migrations/pg/0001_bootstrap.py +10 -12
  26. migrations/pg/0002_catalog.py +42 -0
  27. nucliadb/ingest/tests/unit/test_settings.py → migrations/pg/0003_catalog_kbid_index.py +5 -3
  28. nucliadb/common/cluster/base.py +41 -24
  29. nucliadb/common/cluster/discovery/base.py +6 -14
  30. nucliadb/common/cluster/discovery/k8s.py +9 -19
  31. nucliadb/common/cluster/discovery/manual.py +1 -3
  32. nucliadb/common/cluster/discovery/single.py +1 -2
  33. nucliadb/common/cluster/discovery/utils.py +1 -3
  34. nucliadb/common/cluster/grpc_node_dummy.py +11 -16
  35. nucliadb/common/cluster/index_node.py +10 -19
  36. nucliadb/common/cluster/manager.py +223 -102
  37. nucliadb/common/cluster/rebalance.py +42 -37
  38. nucliadb/common/cluster/rollover.py +377 -204
  39. nucliadb/common/cluster/settings.py +16 -9
  40. nucliadb/common/cluster/standalone/grpc_node_binding.py +24 -76
  41. nucliadb/common/cluster/standalone/index_node.py +4 -11
  42. nucliadb/common/cluster/standalone/service.py +2 -6
  43. nucliadb/common/cluster/standalone/utils.py +9 -6
  44. nucliadb/common/cluster/utils.py +43 -29
  45. nucliadb/common/constants.py +20 -0
  46. nucliadb/common/context/__init__.py +6 -4
  47. nucliadb/common/context/fastapi.py +8 -5
  48. nucliadb/{tests/knowledgeboxes/__init__.py → common/counters.py} +8 -2
  49. nucliadb/common/datamanagers/__init__.py +24 -5
  50. nucliadb/common/datamanagers/atomic.py +102 -0
  51. nucliadb/common/datamanagers/cluster.py +5 -5
  52. nucliadb/common/datamanagers/entities.py +6 -16
  53. nucliadb/common/datamanagers/fields.py +84 -0
  54. nucliadb/common/datamanagers/kb.py +101 -24
  55. nucliadb/common/datamanagers/labels.py +26 -56
  56. nucliadb/common/datamanagers/processing.py +2 -6
  57. nucliadb/common/datamanagers/resources.py +214 -117
  58. nucliadb/common/datamanagers/rollover.py +77 -16
  59. nucliadb/{ingest/orm → common/datamanagers}/synonyms.py +16 -28
  60. nucliadb/common/datamanagers/utils.py +19 -11
  61. nucliadb/common/datamanagers/vectorsets.py +110 -0
  62. nucliadb/common/external_index_providers/base.py +257 -0
  63. nucliadb/{ingest/tests/unit/test_cache.py → common/external_index_providers/exceptions.py} +9 -8
  64. nucliadb/common/external_index_providers/manager.py +101 -0
  65. nucliadb/common/external_index_providers/pinecone.py +933 -0
  66. nucliadb/common/external_index_providers/settings.py +52 -0
  67. nucliadb/common/http_clients/auth.py +3 -6
  68. nucliadb/common/http_clients/processing.py +6 -11
  69. nucliadb/common/http_clients/utils.py +1 -3
  70. nucliadb/common/ids.py +240 -0
  71. nucliadb/common/locking.py +43 -13
  72. nucliadb/common/maindb/driver.py +11 -35
  73. nucliadb/common/maindb/exceptions.py +6 -6
  74. nucliadb/common/maindb/local.py +22 -9
  75. nucliadb/common/maindb/pg.py +206 -111
  76. nucliadb/common/maindb/utils.py +13 -44
  77. nucliadb/common/models_utils/from_proto.py +479 -0
  78. nucliadb/common/models_utils/to_proto.py +60 -0
  79. nucliadb/common/nidx.py +260 -0
  80. nucliadb/export_import/datamanager.py +25 -19
  81. nucliadb/export_import/exceptions.py +8 -0
  82. nucliadb/export_import/exporter.py +20 -7
  83. nucliadb/export_import/importer.py +6 -11
  84. nucliadb/export_import/models.py +5 -5
  85. nucliadb/export_import/tasks.py +4 -4
  86. nucliadb/export_import/utils.py +94 -54
  87. nucliadb/health.py +1 -3
  88. nucliadb/ingest/app.py +15 -11
  89. nucliadb/ingest/consumer/auditing.py +30 -147
  90. nucliadb/ingest/consumer/consumer.py +96 -52
  91. nucliadb/ingest/consumer/materializer.py +10 -12
  92. nucliadb/ingest/consumer/pull.py +12 -27
  93. nucliadb/ingest/consumer/service.py +20 -19
  94. nucliadb/ingest/consumer/shard_creator.py +7 -14
  95. nucliadb/ingest/consumer/utils.py +1 -3
  96. nucliadb/ingest/fields/base.py +139 -188
  97. nucliadb/ingest/fields/conversation.py +18 -5
  98. nucliadb/ingest/fields/exceptions.py +1 -4
  99. nucliadb/ingest/fields/file.py +7 -25
  100. nucliadb/ingest/fields/link.py +11 -16
  101. nucliadb/ingest/fields/text.py +9 -4
  102. nucliadb/ingest/orm/brain.py +255 -262
  103. nucliadb/ingest/orm/broker_message.py +181 -0
  104. nucliadb/ingest/orm/entities.py +36 -51
  105. nucliadb/ingest/orm/exceptions.py +12 -0
  106. nucliadb/ingest/orm/knowledgebox.py +334 -278
  107. nucliadb/ingest/orm/processor/__init__.py +2 -697
  108. nucliadb/ingest/orm/processor/auditing.py +117 -0
  109. nucliadb/ingest/orm/processor/data_augmentation.py +164 -0
  110. nucliadb/ingest/orm/processor/pgcatalog.py +84 -0
  111. nucliadb/ingest/orm/processor/processor.py +752 -0
  112. nucliadb/ingest/orm/processor/sequence_manager.py +1 -1
  113. nucliadb/ingest/orm/resource.py +280 -520
  114. nucliadb/ingest/orm/utils.py +25 -31
  115. nucliadb/ingest/partitions.py +3 -9
  116. nucliadb/ingest/processing.py +76 -81
  117. nucliadb/ingest/py.typed +0 -0
  118. nucliadb/ingest/serialize.py +37 -173
  119. nucliadb/ingest/service/__init__.py +1 -3
  120. nucliadb/ingest/service/writer.py +186 -577
  121. nucliadb/ingest/settings.py +13 -22
  122. nucliadb/ingest/utils.py +3 -6
  123. nucliadb/learning_proxy.py +264 -51
  124. nucliadb/metrics_exporter.py +30 -19
  125. nucliadb/middleware/__init__.py +1 -3
  126. nucliadb/migrator/command.py +1 -3
  127. nucliadb/migrator/datamanager.py +13 -13
  128. nucliadb/migrator/migrator.py +57 -37
  129. nucliadb/migrator/settings.py +2 -1
  130. nucliadb/migrator/utils.py +18 -10
  131. nucliadb/purge/__init__.py +139 -33
  132. nucliadb/purge/orphan_shards.py +7 -13
  133. nucliadb/reader/__init__.py +1 -3
  134. nucliadb/reader/api/models.py +3 -14
  135. nucliadb/reader/api/v1/__init__.py +0 -1
  136. nucliadb/reader/api/v1/download.py +27 -94
  137. nucliadb/reader/api/v1/export_import.py +4 -4
  138. nucliadb/reader/api/v1/knowledgebox.py +13 -13
  139. nucliadb/reader/api/v1/learning_config.py +8 -12
  140. nucliadb/reader/api/v1/resource.py +67 -93
  141. nucliadb/reader/api/v1/services.py +70 -125
  142. nucliadb/reader/app.py +16 -46
  143. nucliadb/reader/lifecycle.py +18 -4
  144. nucliadb/reader/py.typed +0 -0
  145. nucliadb/reader/reader/notifications.py +10 -31
  146. nucliadb/search/__init__.py +1 -3
  147. nucliadb/search/api/v1/__init__.py +2 -2
  148. nucliadb/search/api/v1/ask.py +112 -0
  149. nucliadb/search/api/v1/catalog.py +184 -0
  150. nucliadb/search/api/v1/feedback.py +17 -25
  151. nucliadb/search/api/v1/find.py +41 -41
  152. nucliadb/search/api/v1/knowledgebox.py +90 -62
  153. nucliadb/search/api/v1/predict_proxy.py +2 -2
  154. nucliadb/search/api/v1/resource/ask.py +66 -117
  155. nucliadb/search/api/v1/resource/search.py +51 -72
  156. nucliadb/search/api/v1/router.py +1 -0
  157. nucliadb/search/api/v1/search.py +50 -197
  158. nucliadb/search/api/v1/suggest.py +40 -54
  159. nucliadb/search/api/v1/summarize.py +9 -5
  160. nucliadb/search/api/v1/utils.py +2 -1
  161. nucliadb/search/app.py +16 -48
  162. nucliadb/search/lifecycle.py +10 -3
  163. nucliadb/search/predict.py +176 -188
  164. nucliadb/search/py.typed +0 -0
  165. nucliadb/search/requesters/utils.py +41 -63
  166. nucliadb/search/search/cache.py +149 -20
  167. nucliadb/search/search/chat/ask.py +918 -0
  168. nucliadb/search/{tests/unit/test_run.py → search/chat/exceptions.py} +14 -13
  169. nucliadb/search/search/chat/images.py +41 -17
  170. nucliadb/search/search/chat/prompt.py +851 -282
  171. nucliadb/search/search/chat/query.py +274 -267
  172. nucliadb/{writer/resource/slug.py → search/search/cut.py} +8 -6
  173. nucliadb/search/search/fetch.py +43 -36
  174. nucliadb/search/search/filters.py +9 -15
  175. nucliadb/search/search/find.py +214 -54
  176. nucliadb/search/search/find_merge.py +408 -391
  177. nucliadb/search/search/hydrator.py +191 -0
  178. nucliadb/search/search/merge.py +198 -234
  179. nucliadb/search/search/metrics.py +73 -2
  180. nucliadb/search/search/paragraphs.py +64 -106
  181. nucliadb/search/search/pgcatalog.py +233 -0
  182. nucliadb/search/search/predict_proxy.py +1 -1
  183. nucliadb/search/search/query.py +386 -257
  184. nucliadb/search/search/query_parser/exceptions.py +22 -0
  185. nucliadb/search/search/query_parser/models.py +101 -0
  186. nucliadb/search/search/query_parser/parser.py +183 -0
  187. nucliadb/search/search/rank_fusion.py +204 -0
  188. nucliadb/search/search/rerankers.py +270 -0
  189. nucliadb/search/search/shards.py +4 -38
  190. nucliadb/search/search/summarize.py +14 -18
  191. nucliadb/search/search/utils.py +27 -4
  192. nucliadb/search/settings.py +15 -1
  193. nucliadb/standalone/api_router.py +4 -10
  194. nucliadb/standalone/app.py +17 -14
  195. nucliadb/standalone/auth.py +7 -21
  196. nucliadb/standalone/config.py +9 -12
  197. nucliadb/standalone/introspect.py +5 -5
  198. nucliadb/standalone/lifecycle.py +26 -25
  199. nucliadb/standalone/migrations.py +58 -0
  200. nucliadb/standalone/purge.py +9 -8
  201. nucliadb/standalone/py.typed +0 -0
  202. nucliadb/standalone/run.py +25 -18
  203. nucliadb/standalone/settings.py +10 -14
  204. nucliadb/standalone/versions.py +15 -5
  205. nucliadb/tasks/consumer.py +8 -12
  206. nucliadb/tasks/producer.py +7 -6
  207. nucliadb/tests/config.py +53 -0
  208. nucliadb/train/__init__.py +1 -3
  209. nucliadb/train/api/utils.py +1 -2
  210. nucliadb/train/api/v1/shards.py +2 -2
  211. nucliadb/train/api/v1/trainset.py +4 -6
  212. nucliadb/train/app.py +14 -47
  213. nucliadb/train/generator.py +10 -19
  214. nucliadb/train/generators/field_classifier.py +7 -19
  215. nucliadb/train/generators/field_streaming.py +156 -0
  216. nucliadb/train/generators/image_classifier.py +12 -18
  217. nucliadb/train/generators/paragraph_classifier.py +5 -9
  218. nucliadb/train/generators/paragraph_streaming.py +6 -9
  219. nucliadb/train/generators/question_answer_streaming.py +19 -20
  220. nucliadb/train/generators/sentence_classifier.py +9 -15
  221. nucliadb/train/generators/token_classifier.py +45 -36
  222. nucliadb/train/generators/utils.py +14 -18
  223. nucliadb/train/lifecycle.py +7 -3
  224. nucliadb/train/nodes.py +23 -32
  225. nucliadb/train/py.typed +0 -0
  226. nucliadb/train/servicer.py +13 -21
  227. nucliadb/train/settings.py +2 -6
  228. nucliadb/train/types.py +13 -10
  229. nucliadb/train/upload.py +3 -6
  230. nucliadb/train/uploader.py +20 -25
  231. nucliadb/train/utils.py +1 -1
  232. nucliadb/writer/__init__.py +1 -3
  233. nucliadb/writer/api/constants.py +0 -5
  234. nucliadb/{ingest/fields/keywordset.py → writer/api/utils.py} +13 -10
  235. nucliadb/writer/api/v1/export_import.py +102 -49
  236. nucliadb/writer/api/v1/field.py +196 -620
  237. nucliadb/writer/api/v1/knowledgebox.py +221 -71
  238. nucliadb/writer/api/v1/learning_config.py +2 -2
  239. nucliadb/writer/api/v1/resource.py +114 -216
  240. nucliadb/writer/api/v1/services.py +64 -132
  241. nucliadb/writer/api/v1/slug.py +61 -0
  242. nucliadb/writer/api/v1/transaction.py +67 -0
  243. nucliadb/writer/api/v1/upload.py +184 -215
  244. nucliadb/writer/app.py +11 -61
  245. nucliadb/writer/back_pressure.py +62 -43
  246. nucliadb/writer/exceptions.py +0 -4
  247. nucliadb/writer/lifecycle.py +21 -15
  248. nucliadb/writer/py.typed +0 -0
  249. nucliadb/writer/resource/audit.py +2 -1
  250. nucliadb/writer/resource/basic.py +48 -62
  251. nucliadb/writer/resource/field.py +45 -135
  252. nucliadb/writer/resource/origin.py +1 -2
  253. nucliadb/writer/settings.py +14 -5
  254. nucliadb/writer/tus/__init__.py +17 -15
  255. nucliadb/writer/tus/azure.py +111 -0
  256. nucliadb/writer/tus/dm.py +17 -5
  257. nucliadb/writer/tus/exceptions.py +1 -3
  258. nucliadb/writer/tus/gcs.py +56 -84
  259. nucliadb/writer/tus/local.py +21 -37
  260. nucliadb/writer/tus/s3.py +28 -68
  261. nucliadb/writer/tus/storage.py +5 -56
  262. nucliadb/writer/vectorsets.py +125 -0
  263. nucliadb-6.2.1.post2777.dist-info/METADATA +148 -0
  264. nucliadb-6.2.1.post2777.dist-info/RECORD +343 -0
  265. {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/WHEEL +1 -1
  266. nucliadb/common/maindb/redis.py +0 -194
  267. nucliadb/common/maindb/tikv.py +0 -412
  268. nucliadb/ingest/fields/layout.py +0 -58
  269. nucliadb/ingest/tests/conftest.py +0 -30
  270. nucliadb/ingest/tests/fixtures.py +0 -771
  271. nucliadb/ingest/tests/integration/consumer/__init__.py +0 -18
  272. nucliadb/ingest/tests/integration/consumer/test_auditing.py +0 -80
  273. nucliadb/ingest/tests/integration/consumer/test_materializer.py +0 -89
  274. nucliadb/ingest/tests/integration/consumer/test_pull.py +0 -144
  275. nucliadb/ingest/tests/integration/consumer/test_service.py +0 -81
  276. nucliadb/ingest/tests/integration/consumer/test_shard_creator.py +0 -68
  277. nucliadb/ingest/tests/integration/ingest/test_ingest.py +0 -691
  278. nucliadb/ingest/tests/integration/ingest/test_processing_engine.py +0 -95
  279. nucliadb/ingest/tests/integration/ingest/test_relations.py +0 -272
  280. nucliadb/ingest/tests/unit/consumer/__init__.py +0 -18
  281. nucliadb/ingest/tests/unit/consumer/test_auditing.py +0 -140
  282. nucliadb/ingest/tests/unit/consumer/test_consumer.py +0 -69
  283. nucliadb/ingest/tests/unit/consumer/test_pull.py +0 -60
  284. nucliadb/ingest/tests/unit/consumer/test_shard_creator.py +0 -139
  285. nucliadb/ingest/tests/unit/consumer/test_utils.py +0 -67
  286. nucliadb/ingest/tests/unit/orm/__init__.py +0 -19
  287. nucliadb/ingest/tests/unit/orm/test_brain.py +0 -247
  288. nucliadb/ingest/tests/unit/orm/test_processor.py +0 -131
  289. nucliadb/ingest/tests/unit/orm/test_resource.py +0 -275
  290. nucliadb/ingest/tests/unit/test_partitions.py +0 -40
  291. nucliadb/ingest/tests/unit/test_processing.py +0 -171
  292. nucliadb/middleware/transaction.py +0 -117
  293. nucliadb/reader/api/v1/learning_collector.py +0 -63
  294. nucliadb/reader/tests/__init__.py +0 -19
  295. nucliadb/reader/tests/conftest.py +0 -31
  296. nucliadb/reader/tests/fixtures.py +0 -136
  297. nucliadb/reader/tests/test_list_resources.py +0 -75
  298. nucliadb/reader/tests/test_reader_file_download.py +0 -273
  299. nucliadb/reader/tests/test_reader_resource.py +0 -379
  300. nucliadb/reader/tests/test_reader_resource_field.py +0 -219
  301. nucliadb/search/api/v1/chat.py +0 -258
  302. nucliadb/search/api/v1/resource/chat.py +0 -94
  303. nucliadb/search/tests/__init__.py +0 -19
  304. nucliadb/search/tests/conftest.py +0 -33
  305. nucliadb/search/tests/fixtures.py +0 -199
  306. nucliadb/search/tests/node.py +0 -465
  307. nucliadb/search/tests/unit/__init__.py +0 -18
  308. nucliadb/search/tests/unit/api/__init__.py +0 -19
  309. nucliadb/search/tests/unit/api/v1/__init__.py +0 -19
  310. nucliadb/search/tests/unit/api/v1/resource/__init__.py +0 -19
  311. nucliadb/search/tests/unit/api/v1/resource/test_ask.py +0 -67
  312. nucliadb/search/tests/unit/api/v1/resource/test_chat.py +0 -97
  313. nucliadb/search/tests/unit/api/v1/test_chat.py +0 -96
  314. nucliadb/search/tests/unit/api/v1/test_predict_proxy.py +0 -98
  315. nucliadb/search/tests/unit/api/v1/test_summarize.py +0 -93
  316. nucliadb/search/tests/unit/search/__init__.py +0 -18
  317. nucliadb/search/tests/unit/search/requesters/__init__.py +0 -18
  318. nucliadb/search/tests/unit/search/requesters/test_utils.py +0 -210
  319. nucliadb/search/tests/unit/search/search/__init__.py +0 -19
  320. nucliadb/search/tests/unit/search/search/test_shards.py +0 -45
  321. nucliadb/search/tests/unit/search/search/test_utils.py +0 -82
  322. nucliadb/search/tests/unit/search/test_chat_prompt.py +0 -266
  323. nucliadb/search/tests/unit/search/test_fetch.py +0 -108
  324. nucliadb/search/tests/unit/search/test_filters.py +0 -125
  325. nucliadb/search/tests/unit/search/test_paragraphs.py +0 -157
  326. nucliadb/search/tests/unit/search/test_predict_proxy.py +0 -106
  327. nucliadb/search/tests/unit/search/test_query.py +0 -201
  328. nucliadb/search/tests/unit/test_app.py +0 -79
  329. nucliadb/search/tests/unit/test_find_merge.py +0 -112
  330. nucliadb/search/tests/unit/test_merge.py +0 -34
  331. nucliadb/search/tests/unit/test_predict.py +0 -584
  332. nucliadb/standalone/tests/__init__.py +0 -19
  333. nucliadb/standalone/tests/conftest.py +0 -33
  334. nucliadb/standalone/tests/fixtures.py +0 -38
  335. nucliadb/standalone/tests/unit/__init__.py +0 -18
  336. nucliadb/standalone/tests/unit/test_api_router.py +0 -61
  337. nucliadb/standalone/tests/unit/test_auth.py +0 -169
  338. nucliadb/standalone/tests/unit/test_introspect.py +0 -35
  339. nucliadb/standalone/tests/unit/test_versions.py +0 -68
  340. nucliadb/tests/benchmarks/__init__.py +0 -19
  341. nucliadb/tests/benchmarks/test_search.py +0 -99
  342. nucliadb/tests/conftest.py +0 -32
  343. nucliadb/tests/fixtures.py +0 -736
  344. nucliadb/tests/knowledgeboxes/philosophy_books.py +0 -203
  345. nucliadb/tests/knowledgeboxes/ten_dummy_resources.py +0 -109
  346. nucliadb/tests/migrations/__init__.py +0 -19
  347. nucliadb/tests/migrations/test_migration_0017.py +0 -80
  348. nucliadb/tests/tikv.py +0 -240
  349. nucliadb/tests/unit/__init__.py +0 -19
  350. nucliadb/tests/unit/common/__init__.py +0 -19
  351. nucliadb/tests/unit/common/cluster/__init__.py +0 -19
  352. nucliadb/tests/unit/common/cluster/discovery/__init__.py +0 -19
  353. nucliadb/tests/unit/common/cluster/discovery/test_k8s.py +0 -170
  354. nucliadb/tests/unit/common/cluster/standalone/__init__.py +0 -18
  355. nucliadb/tests/unit/common/cluster/standalone/test_service.py +0 -113
  356. nucliadb/tests/unit/common/cluster/standalone/test_utils.py +0 -59
  357. nucliadb/tests/unit/common/cluster/test_cluster.py +0 -399
  358. nucliadb/tests/unit/common/cluster/test_kb_shard_manager.py +0 -178
  359. nucliadb/tests/unit/common/cluster/test_rollover.py +0 -279
  360. nucliadb/tests/unit/common/maindb/__init__.py +0 -18
  361. nucliadb/tests/unit/common/maindb/test_driver.py +0 -127
  362. nucliadb/tests/unit/common/maindb/test_tikv.py +0 -53
  363. nucliadb/tests/unit/common/maindb/test_utils.py +0 -81
  364. nucliadb/tests/unit/common/test_context.py +0 -36
  365. nucliadb/tests/unit/export_import/__init__.py +0 -19
  366. nucliadb/tests/unit/export_import/test_datamanager.py +0 -37
  367. nucliadb/tests/unit/export_import/test_utils.py +0 -294
  368. nucliadb/tests/unit/migrator/__init__.py +0 -19
  369. nucliadb/tests/unit/migrator/test_migrator.py +0 -87
  370. nucliadb/tests/unit/tasks/__init__.py +0 -19
  371. nucliadb/tests/unit/tasks/conftest.py +0 -42
  372. nucliadb/tests/unit/tasks/test_consumer.py +0 -93
  373. nucliadb/tests/unit/tasks/test_producer.py +0 -95
  374. nucliadb/tests/unit/tasks/test_tasks.py +0 -60
  375. nucliadb/tests/unit/test_field_ids.py +0 -49
  376. nucliadb/tests/unit/test_health.py +0 -84
  377. nucliadb/tests/unit/test_kb_slugs.py +0 -54
  378. nucliadb/tests/unit/test_learning_proxy.py +0 -252
  379. nucliadb/tests/unit/test_metrics_exporter.py +0 -77
  380. nucliadb/tests/unit/test_purge.py +0 -138
  381. nucliadb/tests/utils/__init__.py +0 -74
  382. nucliadb/tests/utils/aiohttp_session.py +0 -44
  383. nucliadb/tests/utils/broker_messages/__init__.py +0 -167
  384. nucliadb/tests/utils/broker_messages/fields.py +0 -181
  385. nucliadb/tests/utils/broker_messages/helpers.py +0 -33
  386. nucliadb/tests/utils/entities.py +0 -78
  387. nucliadb/train/api/v1/check.py +0 -60
  388. nucliadb/train/tests/__init__.py +0 -19
  389. nucliadb/train/tests/conftest.py +0 -29
  390. nucliadb/train/tests/fixtures.py +0 -342
  391. nucliadb/train/tests/test_field_classification.py +0 -122
  392. nucliadb/train/tests/test_get_entities.py +0 -80
  393. nucliadb/train/tests/test_get_info.py +0 -51
  394. nucliadb/train/tests/test_get_ontology.py +0 -34
  395. nucliadb/train/tests/test_get_ontology_count.py +0 -63
  396. nucliadb/train/tests/test_image_classification.py +0 -222
  397. nucliadb/train/tests/test_list_fields.py +0 -39
  398. nucliadb/train/tests/test_list_paragraphs.py +0 -73
  399. nucliadb/train/tests/test_list_resources.py +0 -39
  400. nucliadb/train/tests/test_list_sentences.py +0 -71
  401. nucliadb/train/tests/test_paragraph_classification.py +0 -123
  402. nucliadb/train/tests/test_paragraph_streaming.py +0 -118
  403. nucliadb/train/tests/test_question_answer_streaming.py +0 -239
  404. nucliadb/train/tests/test_sentence_classification.py +0 -143
  405. nucliadb/train/tests/test_token_classification.py +0 -136
  406. nucliadb/train/tests/utils.py +0 -108
  407. nucliadb/writer/layouts/__init__.py +0 -51
  408. nucliadb/writer/layouts/v1.py +0 -59
  409. nucliadb/writer/resource/vectors.py +0 -120
  410. nucliadb/writer/tests/__init__.py +0 -19
  411. nucliadb/writer/tests/conftest.py +0 -31
  412. nucliadb/writer/tests/fixtures.py +0 -192
  413. nucliadb/writer/tests/test_fields.py +0 -486
  414. nucliadb/writer/tests/test_files.py +0 -743
  415. nucliadb/writer/tests/test_knowledgebox.py +0 -49
  416. nucliadb/writer/tests/test_reprocess_file_field.py +0 -139
  417. nucliadb/writer/tests/test_resources.py +0 -546
  418. nucliadb/writer/tests/test_service.py +0 -137
  419. nucliadb/writer/tests/test_tus.py +0 -203
  420. nucliadb/writer/tests/utils.py +0 -35
  421. nucliadb/writer/tus/pg.py +0 -125
  422. nucliadb-2.46.1.post382.dist-info/METADATA +0 -134
  423. nucliadb-2.46.1.post382.dist-info/RECORD +0 -451
  424. {nucliadb/ingest/tests → migrations/pg}/__init__.py +0 -0
  425. /nucliadb/{ingest/tests/integration → common/external_index_providers}/__init__.py +0 -0
  426. /nucliadb/{ingest/tests/integration/ingest → common/models_utils}/__init__.py +0 -0
  427. /nucliadb/{ingest/tests/unit → search/search/query_parser}/__init__.py +0 -0
  428. /nucliadb/{ingest/tests → tests}/vectors.py +0 -0
  429. {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/entry_points.txt +0 -0
  430. {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/top_level.txt +0 -0
  431. {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/zip-safe +0 -0
@@ -1,192 +0,0 @@
1
- # Copyright (C) 2021 Bosutech XXI S.L.
2
- #
3
- # nucliadb is offered under the AGPL v3.0 and as commercial software.
4
- # For commercial licensing, contact us at info@nuclia.com.
5
- #
6
- # AGPL:
7
- # This program is free software: you can redistribute it and/or modify
8
- # it under the terms of the GNU Affero General Public License as
9
- # published by the Free Software Foundation, either version 3 of the
10
- # License, or (at your option) any later version.
11
- #
12
- # This program is distributed in the hope that it will be useful,
13
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
- # GNU Affero General Public License for more details.
16
- #
17
- # You should have received a copy of the GNU Affero General Public License
18
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
- #
20
- from enum import Enum
21
- from typing import AsyncIterator, Callable, Optional
22
- from unittest import mock
23
-
24
- import pytest
25
- from httpx import AsyncClient
26
- from pytest_lazy_fixtures import lazy_fixture
27
- from redis import asyncio as aioredis
28
-
29
- from nucliadb.ingest.tests.fixtures import IngestFixture
30
- from nucliadb.writer import API_PREFIX
31
- from nucliadb.writer.api.v1.router import KB_PREFIX, KBS_PREFIX
32
- from nucliadb.writer.settings import settings
33
- from nucliadb.writer.tus import clear_storage
34
- from nucliadb_models.resource import NucliaDBRoles
35
- from nucliadb_utils.settings import (
36
- FileBackendConfig,
37
- nuclia_settings,
38
- nucliadb_settings,
39
- storage_settings,
40
- )
41
- from nucliadb_utils.tests.conftest import get_testing_storage_backend
42
- from nucliadb_utils.utilities import Utility, clean_utility, set_utility
43
-
44
-
45
- @pytest.fixture(scope="function")
46
- def disabled_back_pressure():
47
- with mock.patch(
48
- "nucliadb.writer.back_pressure.is_back_pressure_enabled", return_value=False
49
- ) as mocked:
50
- yield mocked
51
-
52
-
53
- @pytest.fixture(scope="function")
54
- async def writer_api(
55
- disabled_back_pressure,
56
- redis,
57
- storage_writer,
58
- grpc_servicer: IngestFixture,
59
- transaction_utility,
60
- processing_utility,
61
- tus_manager,
62
- ) -> AsyncIterator[Callable[[list[Enum], str, str], AsyncClient]]:
63
- nucliadb_settings.nucliadb_ingest = grpc_servicer.host
64
- from nucliadb.writer.app import create_application
65
-
66
- application = create_application()
67
-
68
- def make_client_fixture(
69
- roles: Optional[list[Enum]] = None,
70
- user: str = "",
71
- version: str = "1",
72
- ) -> AsyncClient:
73
- roles = roles or []
74
- client_base_url = "http://test"
75
- client_base_url = f"{client_base_url}/{API_PREFIX}/v{version}"
76
-
77
- client = AsyncClient(app=application, base_url=client_base_url) # type: ignore
78
- client.headers["X-NUCLIADB-ROLES"] = ";".join(
79
- map(lambda role: role.value, roles)
80
- )
81
- client.headers["X-NUCLIADB-USER"] = user
82
-
83
- return client
84
-
85
- driver = aioredis.from_url(f"redis://{redis[0]}:{redis[1]}")
86
- await driver.flushall()
87
-
88
- await application.router.startup()
89
-
90
- yield make_client_fixture
91
-
92
- await application.router.shutdown()
93
- clear_storage()
94
-
95
- await driver.flushall()
96
- await driver.close(close_connection_pool=True)
97
-
98
-
99
- @pytest.fixture(scope="function")
100
- def gcs_storage_writer(gcs):
101
- storage_settings.file_backend = FileBackendConfig.GCS
102
- storage_settings.gcs_endpoint_url = gcs
103
- storage_settings.gcs_bucket = "test_{kbid}"
104
-
105
-
106
- @pytest.fixture(scope="function")
107
- def s3_storage_writer(s3):
108
- storage_settings.file_backend = FileBackendConfig.S3
109
- storage_settings.s3_endpoint = s3
110
- storage_settings.s3_client_id = ""
111
- storage_settings.s3_client_secret = ""
112
- storage_settings.s3_bucket = "test-{kbid}"
113
-
114
-
115
- @pytest.fixture(scope="function")
116
- def pg_storage_writer(pg):
117
- storage_settings.file_backend = FileBackendConfig.PG
118
- url = f"postgresql://postgres:postgres@{pg[0]}:{pg[1]}/postgres"
119
- storage_settings.driver_pg_url = url
120
-
121
-
122
- def lazy_storage_writer_fixture():
123
- backend = get_testing_storage_backend()
124
- if backend == "gcs":
125
- return [lazy_fixture.lf("gcs_storage_writer")]
126
- elif backend == "s3":
127
- return [lazy_fixture.lf("s3_storage_writer")]
128
- elif backend == "pg":
129
- return [lazy_fixture.lf("pg_storage_writer")]
130
- else:
131
- print(f"Unknown storage backend {backend}, using gcs")
132
- return [lazy_fixture.lf("gcs_storage_writer")]
133
-
134
-
135
- @pytest.fixture(scope="function", params=lazy_storage_writer_fixture())
136
- async def storage_writer(request):
137
- """
138
- Generic storage fixture that allows us to run the same tests for different storage backends.
139
- """
140
- storage_driver = request.param
141
- set_utility(Utility.STORAGE, storage_driver)
142
-
143
- yield storage_driver
144
-
145
- clean_utility(Utility.STORAGE)
146
-
147
-
148
- @pytest.fixture(scope="function")
149
- async def knowledgebox_writer(writer_api):
150
- async with writer_api(roles=[NucliaDBRoles.MANAGER]) as client:
151
- resp = await client.post(
152
- f"/{KBS_PREFIX}",
153
- json={
154
- "slug": "kbid1",
155
- "title": "My Knowledge Box",
156
- },
157
- )
158
- assert resp.status_code == 201
159
- kbid = resp.json().get("uuid")
160
- assert kbid is not None
161
- yield kbid
162
-
163
-
164
- @pytest.fixture(scope="function")
165
- async def resource(redis, writer_api, knowledgebox_writer):
166
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
167
- resp = await client.post(
168
- f"/{KB_PREFIX}/{knowledgebox_writer}/resources",
169
- headers={"X-Synchronous": "true"},
170
- json={
171
- "slug": "resource1",
172
- "title": "Resource 1",
173
- },
174
- )
175
- assert resp.status_code == 201
176
- uuid = resp.json()["uuid"]
177
-
178
- return uuid
179
-
180
-
181
- @pytest.fixture(scope="function")
182
- async def processing_utility():
183
- nuclia_settings.dummy_processing = True
184
- nuclia_settings.onprem = True
185
- nuclia_settings.nuclia_jwt_key = "foobarkey"
186
-
187
-
188
- @pytest.fixture(scope="function")
189
- async def tus_manager(redis):
190
- settings.dm_redis_host = redis[0]
191
- settings.dm_redis_port = redis[1]
192
- yield
@@ -1,486 +0,0 @@
1
- # Copyright (C) 2021 Bosutech XXI S.L.
2
- #
3
- # nucliadb is offered under the AGPL v3.0 and as commercial software.
4
- # For commercial licensing, contact us at info@nuclia.com.
5
- #
6
- # AGPL:
7
- # This program is free software: you can redistribute it and/or modify
8
- # it under the terms of the GNU Affero General Public License as
9
- # published by the Free Software Foundation, either version 3 of the
10
- # License, or (at your option) any later version.
11
- #
12
- # This program is distributed in the hope that it will be useful,
13
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
- # GNU Affero General Public License for more details.
16
- #
17
- # You should have received a copy of the GNU Affero General Public License
18
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
- #
20
-
21
- from copy import deepcopy
22
- from datetime import datetime
23
- from os.path import dirname
24
-
25
- import pytest
26
-
27
- from nucliadb.writer.api.v1.router import (
28
- KB_PREFIX,
29
- RESOURCE_PREFIX,
30
- RESOURCES_PREFIX,
31
- RSLUG_PREFIX,
32
- )
33
- from nucliadb.writer.tests.utils import load_file_as_FileB64_payload
34
- from nucliadb_models.resource import NucliaDBRoles
35
-
36
- TEST_FILE = {f"{dirname(__file__)}/orm/"}
37
- TEST_TEXT_PAYLOAD = {"body": "test1", "format": "PLAIN"}
38
- TEST_LINK_PAYLOAD = {
39
- "added": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
40
- "headers": {},
41
- "cookies": {},
42
- "uri": "http://some-link.com",
43
- "language": "en",
44
- "localstorage": {},
45
- "css_selector": "main",
46
- }
47
- TEST_KEYWORDSETS_PAYLOAD = {"keywords": [{"value": "kw1"}, {"value": "kw2"}]}
48
- TEST_DATETIMES_PAYLOAD = {"value": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}
49
- TEST_CONVERSATION_PAYLOAD = {
50
- "messages": [
51
- {
52
- "timestamp": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
53
- "who": "Bob",
54
- "to": ["Alice", "Charlie"],
55
- "content": {
56
- "text": "Hi people!",
57
- "format": "PLAIN",
58
- "files": [
59
- load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg")
60
- ],
61
- },
62
- "ident": "message_id_001",
63
- }
64
- ]
65
- }
66
- TEST_LAYOUT_PAYLOAD = {
67
- "body": {
68
- "blocks": {
69
- "block1": {
70
- "x": 0,
71
- "y": 0,
72
- "cols": 1,
73
- "rows": 2,
74
- "type": "TITLE",
75
- "ident": "main_title",
76
- "payload": "This is a Test Title",
77
- "file": load_file_as_FileB64_payload(
78
- "/assets/image001.jpg", "image/jpg"
79
- ),
80
- }
81
- }
82
- },
83
- "format": "NUCLIAv1",
84
- }
85
-
86
- TEST_FILE_PAYLOAD = {
87
- "language": "en",
88
- "password": "xxxxxx",
89
- "file": load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg"),
90
- }
91
-
92
- TEST_EXTERNAL_FILE_PAYLOAD = {
93
- "file": {
94
- "uri": "https://mysite.com/files/myfile.pdf",
95
- "extra_headers": {"foo": "bar"},
96
- }
97
- }
98
-
99
- TEST_CONVERSATION_APPEND_MESSAGES_PAYLOAD = [
100
- {
101
- "timestamp": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
102
- "who": "Bob",
103
- "to": ["Alice", "Charlie"],
104
- "content": {
105
- "text": "Hi people!",
106
- "format": "PLAIN",
107
- "attachments": [
108
- load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg")
109
- ],
110
- },
111
- "ident": "message_id_001",
112
- }
113
- ]
114
-
115
- TEST_LAYOUT_APPEND_BLOCKS_PAYLOAD = {
116
- "block1": {
117
- "x": 0,
118
- "y": 0,
119
- "cols": 1,
120
- "rows": 2,
121
- "type": "TITLE",
122
- "ident": "main_title",
123
- "payload": "This is a Test Title",
124
- "file": load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg"),
125
- }
126
- }
127
-
128
-
129
- @pytest.mark.asyncio
130
- async def test_resource_field_add(writer_api, knowledgebox_writer):
131
- knowledgebox_id = knowledgebox_writer
132
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
133
- resp = await client.post(
134
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
135
- headers={"X-SYNCHRONOUS": "True"},
136
- json={"slug": "resource1", "title": "My resource"},
137
- )
138
- assert resp.status_code == 201
139
- data = resp.json()
140
- assert "uuid" in data
141
- assert "seqid" in data
142
- rid = data["uuid"]
143
-
144
- # Text
145
- resp = await client.put(
146
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/text/text1",
147
- json=TEST_TEXT_PAYLOAD,
148
- )
149
- assert resp.status_code == 201
150
- data = resp.json()
151
- assert "seqid" in data
152
-
153
- # Link
154
- resp = await client.put(
155
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/link/link1",
156
- json=TEST_LINK_PAYLOAD,
157
- )
158
- assert resp.status_code == 201
159
- data = resp.json()
160
- assert "seqid" in data
161
-
162
- # Keywordset
163
- resp = await client.put(
164
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/keywordset/kws1",
165
- json=TEST_KEYWORDSETS_PAYLOAD,
166
- )
167
- assert resp.status_code == 201
168
- data = resp.json()
169
- assert "seqid" in data
170
-
171
- # Datetimes
172
-
173
- resp = await client.put(
174
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/datetime/date1",
175
- json=TEST_DATETIMES_PAYLOAD,
176
- )
177
- assert resp.status_code == 201
178
- data = resp.json()
179
- assert "seqid" in data
180
-
181
- # Conversation
182
- resp = await client.put(
183
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/conversation/conv1",
184
- json=TEST_CONVERSATION_PAYLOAD,
185
- )
186
-
187
- assert resp.status_code == 201
188
- data = resp.json()
189
- assert "seqid" in data
190
-
191
- # Layout
192
- resp = await client.put(
193
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/layout/layout1",
194
- json=TEST_LAYOUT_PAYLOAD,
195
- )
196
-
197
- assert resp.status_code == 201
198
- data = resp.json()
199
- assert "seqid" in data
200
-
201
- # File
202
- resp = await client.put(
203
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1",
204
- json=TEST_FILE_PAYLOAD,
205
- )
206
- assert resp.status_code == 201
207
- data = resp.json()
208
- assert "seqid" in data
209
-
210
- # File without storing it in the internal BrokerMessage, only send to process
211
- resp = await client.put(
212
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1",
213
- json=TEST_FILE_PAYLOAD,
214
- headers={"x_skip_store": "1"},
215
- )
216
- assert resp.status_code == 201
217
- data = resp.json()
218
- assert "seqid" in data
219
-
220
- # File field pointing to an externally hosted file
221
- resp = await client.put(
222
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/externalfile",
223
- json=TEST_EXTERNAL_FILE_PAYLOAD,
224
- headers={"X-SYNCHRONOUS": "True"},
225
- )
226
- assert resp.status_code == 201
227
- data = resp.json()
228
- assert "seqid" in data
229
-
230
-
231
- @pytest.mark.asyncio
232
- async def test_resource_field_append_extra(writer_api, knowledgebox_writer):
233
- knowledgebox_id = knowledgebox_writer
234
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
235
- resp = await client.post(
236
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
237
- headers={"X-SYNCHRONOUS": "True"},
238
- json={
239
- "slug": "resource1",
240
- "title": "My resource",
241
- "layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
242
- "conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
243
- },
244
- )
245
- assert resp.status_code == 201
246
- data = resp.json()
247
- assert "uuid" in data
248
- assert "seqid" in data
249
- rid = data["uuid"]
250
-
251
- # Conversation
252
- resp = await client.put(
253
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/conversation/conv1/messages",
254
- json=TEST_CONVERSATION_APPEND_MESSAGES_PAYLOAD,
255
- )
256
- assert resp.status_code == 200
257
- data = resp.json()
258
- assert "seqid" in data
259
-
260
- # Layout
261
- resp = await client.put(
262
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/layout/layout1/blocks",
263
- json=TEST_LAYOUT_APPEND_BLOCKS_PAYLOAD,
264
- )
265
-
266
- assert resp.status_code == 200
267
- data = resp.json()
268
- assert "seqid" in data
269
-
270
-
271
- @pytest.mark.asyncio
272
- async def test_resource_field_delete(writer_api, knowledgebox_writer):
273
- knowledgebox_id = knowledgebox_writer
274
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
275
- resp = await client.post(
276
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
277
- headers={"X-SYNCHRONOUS": "True"},
278
- json={
279
- "slug": "resource1",
280
- "title": "My resource",
281
- "texts": {"text1": TEST_TEXT_PAYLOAD},
282
- "links": {"link1": TEST_LINK_PAYLOAD},
283
- "files": {"file1": TEST_FILE_PAYLOAD},
284
- "layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
285
- "conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
286
- "keywordsets": {"keywordset1": TEST_KEYWORDSETS_PAYLOAD},
287
- "datetimes": {"datetime1": TEST_DATETIMES_PAYLOAD},
288
- },
289
- )
290
-
291
- assert resp.status_code == 201
292
- data = resp.json()
293
- rid = data["uuid"]
294
-
295
- # Text
296
- resp = await client.delete(
297
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/text/text1"
298
- )
299
- assert resp.status_code == 204
300
-
301
- # Link
302
- resp = await client.delete(
303
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/link/link1"
304
- )
305
- assert resp.status_code == 204
306
-
307
- # Keywords
308
- resp = await client.delete(
309
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/keywordset/kws1"
310
- )
311
- assert resp.status_code == 204
312
-
313
- # Datetimes
314
-
315
- resp = await client.delete(
316
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/datetime/date1"
317
- )
318
- assert resp.status_code == 204
319
-
320
- # Conversation
321
- resp = await client.delete(
322
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/conversation/conv1"
323
- )
324
-
325
- # Layout
326
- resp = await client.delete(
327
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/layout/layout1"
328
- )
329
- assert resp.status_code == 204
330
-
331
- # File
332
- resp = await client.delete(
333
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1"
334
- )
335
- assert resp.status_code == 204
336
-
337
-
338
- @pytest.mark.asyncio
339
- @pytest.mark.parametrize(
340
- "endpoint,payload",
341
- [
342
- ("text/text1", TEST_TEXT_PAYLOAD),
343
- ("link/link1", TEST_LINK_PAYLOAD),
344
- ("keywordset/kws1", TEST_KEYWORDSETS_PAYLOAD),
345
- ("datetime/date1", TEST_DATETIMES_PAYLOAD),
346
- ("conversation/conv1", TEST_CONVERSATION_PAYLOAD),
347
- ("conversation/conv1/messages", TEST_CONVERSATION_APPEND_MESSAGES_PAYLOAD),
348
- ("layout/layout1", TEST_LAYOUT_PAYLOAD),
349
- ("layout/layout1/blocks", TEST_LAYOUT_APPEND_BLOCKS_PAYLOAD),
350
- ("file/file1", TEST_FILE_PAYLOAD),
351
- ],
352
- )
353
- async def test_sync_ops(writer_api, knowledgebox_writer, endpoint, payload):
354
- knowledgebox_id = knowledgebox_writer
355
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
356
- HEADERS = {"X-SYNCHRONOUS": "True"}
357
- # Create a resource
358
- resp = await client.post(
359
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
360
- headers=HEADERS,
361
- json={
362
- "slug": "resource1",
363
- "title": "My resource",
364
- "layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
365
- "conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
366
- },
367
- )
368
- assert resp.status_code == 201
369
- data = resp.json()
370
- rid = data["uuid"]
371
-
372
- resource_path = f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}"
373
- resp = await client.put(
374
- f"{resource_path}/{endpoint}",
375
- headers={"X-SYNCHRONOUS": "True"},
376
- json=payload,
377
- )
378
- assert resp.status_code in (201, 200)
379
-
380
-
381
- @pytest.mark.asyncio
382
- async def test_external_file_field(writer_api, knowledgebox_writer):
383
- knowledgebox_id = knowledgebox_writer
384
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
385
- resp = await client.post(
386
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
387
- json={"slug": "resource1", "title": "My resource"},
388
- headers={"X-SYNCHRONOUS": "True"},
389
- )
390
- assert resp.status_code == 201
391
- rid = resp.json()["uuid"]
392
-
393
- # File field pointing to an externally hosted file
394
- resp = await client.put(
395
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/externalfile",
396
- json=TEST_EXTERNAL_FILE_PAYLOAD,
397
- headers={"X-SYNCHRONOUS": "True"},
398
- )
399
- assert resp.status_code == 201
400
-
401
-
402
- @pytest.mark.asyncio
403
- async def test_file_field_validation(writer_api, knowledgebox_writer):
404
- knowledgebox_id = knowledgebox_writer
405
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
406
- resp = await client.post(
407
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
408
- json={"slug": "resource1", "title": "My resource"},
409
- headers={"X-SYNCHRONOUS": "True"},
410
- )
411
- assert resp.status_code == 201
412
- rid = resp.json()["uuid"]
413
-
414
- # Remove a required key from the payload
415
- payload = deepcopy(TEST_FILE_PAYLOAD)
416
- payload["file"].pop("md5")
417
-
418
- resp = await client.put(
419
- f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1",
420
- json=payload,
421
- headers={"X-SYNCHRONOUS": "True"},
422
- )
423
- assert resp.status_code == 201
424
-
425
-
426
- @pytest.mark.parametrize(
427
- "method,endpoint,payload",
428
- [
429
- ["put", "/text/{field_id}", TEST_TEXT_PAYLOAD],
430
- ["put", "/link/{field_id}", TEST_LINK_PAYLOAD],
431
- ["put", "/keywordset/{field_id}", TEST_KEYWORDSETS_PAYLOAD],
432
- ["put", "/datetime/{field_id}", TEST_DATETIMES_PAYLOAD],
433
- ["put", "/layout/{field_id}", TEST_LAYOUT_PAYLOAD],
434
- ["put", "/conversation/{field_id}", TEST_CONVERSATION_PAYLOAD],
435
- ["put", "/file/{field_id}", TEST_FILE_PAYLOAD],
436
- ["delete", "", None],
437
- ],
438
- )
439
- @pytest.mark.asyncio()
440
- async def test_field_endpoints_by_slug(
441
- writer_api,
442
- knowledgebox_ingest,
443
- method,
444
- endpoint,
445
- payload,
446
- ):
447
- async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
448
- slug = "my-resource"
449
- field_id = "myfield"
450
- field_type = "text"
451
-
452
- resp = await client.post(
453
- f"/{KB_PREFIX}/{knowledgebox_ingest}/{RESOURCES_PREFIX}",
454
- headers={"X-SYNCHRONOUS": "True"},
455
- json={"slug": slug},
456
- )
457
- assert resp.status_code == 201
458
-
459
- extra_params = {}
460
- if payload is not None:
461
- extra_params["json"] = payload
462
- op = getattr(client, method)
463
-
464
- # Try first a non-existing slug should return 404
465
- url = endpoint.format(
466
- field_id=field_id,
467
- field_type=field_type,
468
- )
469
-
470
- resp = await op(
471
- f"/{KB_PREFIX}/{knowledgebox_ingest}/{RSLUG_PREFIX}/idonotexist" + url,
472
- **extra_params,
473
- )
474
- assert resp.status_code == 404
475
- assert resp.json()["detail"] == "Resource does not exist"
476
-
477
- # Try the happy path now
478
- url = endpoint.format(
479
- field_id=field_id,
480
- field_type=field_type,
481
- )
482
- resp = await op(
483
- f"/{KB_PREFIX}/{knowledgebox_ingest}/{RSLUG_PREFIX}/{slug}" + url,
484
- **extra_params,
485
- )
486
- assert str(resp.status_code).startswith("2")