nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- migrations/0002_rollover_shards.py +1 -2
- migrations/0003_allfields_key.py +2 -37
- migrations/0004_rollover_shards.py +1 -2
- migrations/0005_rollover_shards.py +1 -2
- migrations/0006_rollover_shards.py +2 -4
- migrations/0008_cleanup_leftover_rollover_metadata.py +1 -2
- migrations/0009_upgrade_relations_and_texts_to_v2.py +5 -4
- migrations/0010_fix_corrupt_indexes.py +11 -12
- migrations/0011_materialize_labelset_ids.py +2 -18
- migrations/0012_rollover_shards.py +6 -12
- migrations/0013_rollover_shards.py +2 -4
- migrations/0014_rollover_shards.py +5 -7
- migrations/0015_targeted_rollover.py +6 -12
- migrations/0016_upgrade_to_paragraphs_v2.py +27 -32
- migrations/0017_multiple_writable_shards.py +3 -6
- migrations/0018_purge_orphan_kbslugs.py +59 -0
- migrations/0019_upgrade_to_paragraphs_v3.py +66 -0
- migrations/0020_drain_nodes_from_cluster.py +83 -0
- nucliadb/standalone/tests/unit/test_run.py → migrations/0021_overwrite_vectorsets_key.py +17 -18
- nucliadb/tests/unit/test_openapi.py → migrations/0022_fix_paragraph_deletion_bug.py +16 -11
- migrations/0023_backfill_pg_catalog.py +80 -0
- migrations/0025_assign_models_to_kbs_v2.py +113 -0
- migrations/0026_fix_high_cardinality_content_types.py +61 -0
- migrations/0027_rollover_texts3.py +73 -0
- nucliadb/ingest/fields/date.py → migrations/pg/0001_bootstrap.py +10 -12
- migrations/pg/0002_catalog.py +42 -0
- nucliadb/ingest/tests/unit/test_settings.py → migrations/pg/0003_catalog_kbid_index.py +5 -3
- nucliadb/common/cluster/base.py +41 -24
- nucliadb/common/cluster/discovery/base.py +6 -14
- nucliadb/common/cluster/discovery/k8s.py +9 -19
- nucliadb/common/cluster/discovery/manual.py +1 -3
- nucliadb/common/cluster/discovery/single.py +1 -2
- nucliadb/common/cluster/discovery/utils.py +1 -3
- nucliadb/common/cluster/grpc_node_dummy.py +11 -16
- nucliadb/common/cluster/index_node.py +10 -19
- nucliadb/common/cluster/manager.py +223 -102
- nucliadb/common/cluster/rebalance.py +42 -37
- nucliadb/common/cluster/rollover.py +377 -204
- nucliadb/common/cluster/settings.py +16 -9
- nucliadb/common/cluster/standalone/grpc_node_binding.py +24 -76
- nucliadb/common/cluster/standalone/index_node.py +4 -11
- nucliadb/common/cluster/standalone/service.py +2 -6
- nucliadb/common/cluster/standalone/utils.py +9 -6
- nucliadb/common/cluster/utils.py +43 -29
- nucliadb/common/constants.py +20 -0
- nucliadb/common/context/__init__.py +6 -4
- nucliadb/common/context/fastapi.py +8 -5
- nucliadb/{tests/knowledgeboxes/__init__.py → common/counters.py} +8 -2
- nucliadb/common/datamanagers/__init__.py +24 -5
- nucliadb/common/datamanagers/atomic.py +102 -0
- nucliadb/common/datamanagers/cluster.py +5 -5
- nucliadb/common/datamanagers/entities.py +6 -16
- nucliadb/common/datamanagers/fields.py +84 -0
- nucliadb/common/datamanagers/kb.py +101 -24
- nucliadb/common/datamanagers/labels.py +26 -56
- nucliadb/common/datamanagers/processing.py +2 -6
- nucliadb/common/datamanagers/resources.py +214 -117
- nucliadb/common/datamanagers/rollover.py +77 -16
- nucliadb/{ingest/orm → common/datamanagers}/synonyms.py +16 -28
- nucliadb/common/datamanagers/utils.py +19 -11
- nucliadb/common/datamanagers/vectorsets.py +110 -0
- nucliadb/common/external_index_providers/base.py +257 -0
- nucliadb/{ingest/tests/unit/test_cache.py → common/external_index_providers/exceptions.py} +9 -8
- nucliadb/common/external_index_providers/manager.py +101 -0
- nucliadb/common/external_index_providers/pinecone.py +933 -0
- nucliadb/common/external_index_providers/settings.py +52 -0
- nucliadb/common/http_clients/auth.py +3 -6
- nucliadb/common/http_clients/processing.py +6 -11
- nucliadb/common/http_clients/utils.py +1 -3
- nucliadb/common/ids.py +240 -0
- nucliadb/common/locking.py +43 -13
- nucliadb/common/maindb/driver.py +11 -35
- nucliadb/common/maindb/exceptions.py +6 -6
- nucliadb/common/maindb/local.py +22 -9
- nucliadb/common/maindb/pg.py +206 -111
- nucliadb/common/maindb/utils.py +13 -44
- nucliadb/common/models_utils/from_proto.py +479 -0
- nucliadb/common/models_utils/to_proto.py +60 -0
- nucliadb/common/nidx.py +260 -0
- nucliadb/export_import/datamanager.py +25 -19
- nucliadb/export_import/exceptions.py +8 -0
- nucliadb/export_import/exporter.py +20 -7
- nucliadb/export_import/importer.py +6 -11
- nucliadb/export_import/models.py +5 -5
- nucliadb/export_import/tasks.py +4 -4
- nucliadb/export_import/utils.py +94 -54
- nucliadb/health.py +1 -3
- nucliadb/ingest/app.py +15 -11
- nucliadb/ingest/consumer/auditing.py +30 -147
- nucliadb/ingest/consumer/consumer.py +96 -52
- nucliadb/ingest/consumer/materializer.py +10 -12
- nucliadb/ingest/consumer/pull.py +12 -27
- nucliadb/ingest/consumer/service.py +20 -19
- nucliadb/ingest/consumer/shard_creator.py +7 -14
- nucliadb/ingest/consumer/utils.py +1 -3
- nucliadb/ingest/fields/base.py +139 -188
- nucliadb/ingest/fields/conversation.py +18 -5
- nucliadb/ingest/fields/exceptions.py +1 -4
- nucliadb/ingest/fields/file.py +7 -25
- nucliadb/ingest/fields/link.py +11 -16
- nucliadb/ingest/fields/text.py +9 -4
- nucliadb/ingest/orm/brain.py +255 -262
- nucliadb/ingest/orm/broker_message.py +181 -0
- nucliadb/ingest/orm/entities.py +36 -51
- nucliadb/ingest/orm/exceptions.py +12 -0
- nucliadb/ingest/orm/knowledgebox.py +334 -278
- nucliadb/ingest/orm/processor/__init__.py +2 -697
- nucliadb/ingest/orm/processor/auditing.py +117 -0
- nucliadb/ingest/orm/processor/data_augmentation.py +164 -0
- nucliadb/ingest/orm/processor/pgcatalog.py +84 -0
- nucliadb/ingest/orm/processor/processor.py +752 -0
- nucliadb/ingest/orm/processor/sequence_manager.py +1 -1
- nucliadb/ingest/orm/resource.py +280 -520
- nucliadb/ingest/orm/utils.py +25 -31
- nucliadb/ingest/partitions.py +3 -9
- nucliadb/ingest/processing.py +76 -81
- nucliadb/ingest/py.typed +0 -0
- nucliadb/ingest/serialize.py +37 -173
- nucliadb/ingest/service/__init__.py +1 -3
- nucliadb/ingest/service/writer.py +186 -577
- nucliadb/ingest/settings.py +13 -22
- nucliadb/ingest/utils.py +3 -6
- nucliadb/learning_proxy.py +264 -51
- nucliadb/metrics_exporter.py +30 -19
- nucliadb/middleware/__init__.py +1 -3
- nucliadb/migrator/command.py +1 -3
- nucliadb/migrator/datamanager.py +13 -13
- nucliadb/migrator/migrator.py +57 -37
- nucliadb/migrator/settings.py +2 -1
- nucliadb/migrator/utils.py +18 -10
- nucliadb/purge/__init__.py +139 -33
- nucliadb/purge/orphan_shards.py +7 -13
- nucliadb/reader/__init__.py +1 -3
- nucliadb/reader/api/models.py +3 -14
- nucliadb/reader/api/v1/__init__.py +0 -1
- nucliadb/reader/api/v1/download.py +27 -94
- nucliadb/reader/api/v1/export_import.py +4 -4
- nucliadb/reader/api/v1/knowledgebox.py +13 -13
- nucliadb/reader/api/v1/learning_config.py +8 -12
- nucliadb/reader/api/v1/resource.py +67 -93
- nucliadb/reader/api/v1/services.py +70 -125
- nucliadb/reader/app.py +16 -46
- nucliadb/reader/lifecycle.py +18 -4
- nucliadb/reader/py.typed +0 -0
- nucliadb/reader/reader/notifications.py +10 -31
- nucliadb/search/__init__.py +1 -3
- nucliadb/search/api/v1/__init__.py +2 -2
- nucliadb/search/api/v1/ask.py +112 -0
- nucliadb/search/api/v1/catalog.py +184 -0
- nucliadb/search/api/v1/feedback.py +17 -25
- nucliadb/search/api/v1/find.py +41 -41
- nucliadb/search/api/v1/knowledgebox.py +90 -62
- nucliadb/search/api/v1/predict_proxy.py +2 -2
- nucliadb/search/api/v1/resource/ask.py +66 -117
- nucliadb/search/api/v1/resource/search.py +51 -72
- nucliadb/search/api/v1/router.py +1 -0
- nucliadb/search/api/v1/search.py +50 -197
- nucliadb/search/api/v1/suggest.py +40 -54
- nucliadb/search/api/v1/summarize.py +9 -5
- nucliadb/search/api/v1/utils.py +2 -1
- nucliadb/search/app.py +16 -48
- nucliadb/search/lifecycle.py +10 -3
- nucliadb/search/predict.py +176 -188
- nucliadb/search/py.typed +0 -0
- nucliadb/search/requesters/utils.py +41 -63
- nucliadb/search/search/cache.py +149 -20
- nucliadb/search/search/chat/ask.py +918 -0
- nucliadb/search/{tests/unit/test_run.py → search/chat/exceptions.py} +14 -13
- nucliadb/search/search/chat/images.py +41 -17
- nucliadb/search/search/chat/prompt.py +851 -282
- nucliadb/search/search/chat/query.py +274 -267
- nucliadb/{writer/resource/slug.py → search/search/cut.py} +8 -6
- nucliadb/search/search/fetch.py +43 -36
- nucliadb/search/search/filters.py +9 -15
- nucliadb/search/search/find.py +214 -54
- nucliadb/search/search/find_merge.py +408 -391
- nucliadb/search/search/hydrator.py +191 -0
- nucliadb/search/search/merge.py +198 -234
- nucliadb/search/search/metrics.py +73 -2
- nucliadb/search/search/paragraphs.py +64 -106
- nucliadb/search/search/pgcatalog.py +233 -0
- nucliadb/search/search/predict_proxy.py +1 -1
- nucliadb/search/search/query.py +386 -257
- nucliadb/search/search/query_parser/exceptions.py +22 -0
- nucliadb/search/search/query_parser/models.py +101 -0
- nucliadb/search/search/query_parser/parser.py +183 -0
- nucliadb/search/search/rank_fusion.py +204 -0
- nucliadb/search/search/rerankers.py +270 -0
- nucliadb/search/search/shards.py +4 -38
- nucliadb/search/search/summarize.py +14 -18
- nucliadb/search/search/utils.py +27 -4
- nucliadb/search/settings.py +15 -1
- nucliadb/standalone/api_router.py +4 -10
- nucliadb/standalone/app.py +17 -14
- nucliadb/standalone/auth.py +7 -21
- nucliadb/standalone/config.py +9 -12
- nucliadb/standalone/introspect.py +5 -5
- nucliadb/standalone/lifecycle.py +26 -25
- nucliadb/standalone/migrations.py +58 -0
- nucliadb/standalone/purge.py +9 -8
- nucliadb/standalone/py.typed +0 -0
- nucliadb/standalone/run.py +25 -18
- nucliadb/standalone/settings.py +10 -14
- nucliadb/standalone/versions.py +15 -5
- nucliadb/tasks/consumer.py +8 -12
- nucliadb/tasks/producer.py +7 -6
- nucliadb/tests/config.py +53 -0
- nucliadb/train/__init__.py +1 -3
- nucliadb/train/api/utils.py +1 -2
- nucliadb/train/api/v1/shards.py +2 -2
- nucliadb/train/api/v1/trainset.py +4 -6
- nucliadb/train/app.py +14 -47
- nucliadb/train/generator.py +10 -19
- nucliadb/train/generators/field_classifier.py +7 -19
- nucliadb/train/generators/field_streaming.py +156 -0
- nucliadb/train/generators/image_classifier.py +12 -18
- nucliadb/train/generators/paragraph_classifier.py +5 -9
- nucliadb/train/generators/paragraph_streaming.py +6 -9
- nucliadb/train/generators/question_answer_streaming.py +19 -20
- nucliadb/train/generators/sentence_classifier.py +9 -15
- nucliadb/train/generators/token_classifier.py +45 -36
- nucliadb/train/generators/utils.py +14 -18
- nucliadb/train/lifecycle.py +7 -3
- nucliadb/train/nodes.py +23 -32
- nucliadb/train/py.typed +0 -0
- nucliadb/train/servicer.py +13 -21
- nucliadb/train/settings.py +2 -6
- nucliadb/train/types.py +13 -10
- nucliadb/train/upload.py +3 -6
- nucliadb/train/uploader.py +20 -25
- nucliadb/train/utils.py +1 -1
- nucliadb/writer/__init__.py +1 -3
- nucliadb/writer/api/constants.py +0 -5
- nucliadb/{ingest/fields/keywordset.py → writer/api/utils.py} +13 -10
- nucliadb/writer/api/v1/export_import.py +102 -49
- nucliadb/writer/api/v1/field.py +196 -620
- nucliadb/writer/api/v1/knowledgebox.py +221 -71
- nucliadb/writer/api/v1/learning_config.py +2 -2
- nucliadb/writer/api/v1/resource.py +114 -216
- nucliadb/writer/api/v1/services.py +64 -132
- nucliadb/writer/api/v1/slug.py +61 -0
- nucliadb/writer/api/v1/transaction.py +67 -0
- nucliadb/writer/api/v1/upload.py +184 -215
- nucliadb/writer/app.py +11 -61
- nucliadb/writer/back_pressure.py +62 -43
- nucliadb/writer/exceptions.py +0 -4
- nucliadb/writer/lifecycle.py +21 -15
- nucliadb/writer/py.typed +0 -0
- nucliadb/writer/resource/audit.py +2 -1
- nucliadb/writer/resource/basic.py +48 -62
- nucliadb/writer/resource/field.py +45 -135
- nucliadb/writer/resource/origin.py +1 -2
- nucliadb/writer/settings.py +14 -5
- nucliadb/writer/tus/__init__.py +17 -15
- nucliadb/writer/tus/azure.py +111 -0
- nucliadb/writer/tus/dm.py +17 -5
- nucliadb/writer/tus/exceptions.py +1 -3
- nucliadb/writer/tus/gcs.py +56 -84
- nucliadb/writer/tus/local.py +21 -37
- nucliadb/writer/tus/s3.py +28 -68
- nucliadb/writer/tus/storage.py +5 -56
- nucliadb/writer/vectorsets.py +125 -0
- nucliadb-6.2.1.post2777.dist-info/METADATA +148 -0
- nucliadb-6.2.1.post2777.dist-info/RECORD +343 -0
- {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/WHEEL +1 -1
- nucliadb/common/maindb/redis.py +0 -194
- nucliadb/common/maindb/tikv.py +0 -412
- nucliadb/ingest/fields/layout.py +0 -58
- nucliadb/ingest/tests/conftest.py +0 -30
- nucliadb/ingest/tests/fixtures.py +0 -771
- nucliadb/ingest/tests/integration/consumer/__init__.py +0 -18
- nucliadb/ingest/tests/integration/consumer/test_auditing.py +0 -80
- nucliadb/ingest/tests/integration/consumer/test_materializer.py +0 -89
- nucliadb/ingest/tests/integration/consumer/test_pull.py +0 -144
- nucliadb/ingest/tests/integration/consumer/test_service.py +0 -81
- nucliadb/ingest/tests/integration/consumer/test_shard_creator.py +0 -68
- nucliadb/ingest/tests/integration/ingest/test_ingest.py +0 -691
- nucliadb/ingest/tests/integration/ingest/test_processing_engine.py +0 -95
- nucliadb/ingest/tests/integration/ingest/test_relations.py +0 -272
- nucliadb/ingest/tests/unit/consumer/__init__.py +0 -18
- nucliadb/ingest/tests/unit/consumer/test_auditing.py +0 -140
- nucliadb/ingest/tests/unit/consumer/test_consumer.py +0 -69
- nucliadb/ingest/tests/unit/consumer/test_pull.py +0 -60
- nucliadb/ingest/tests/unit/consumer/test_shard_creator.py +0 -139
- nucliadb/ingest/tests/unit/consumer/test_utils.py +0 -67
- nucliadb/ingest/tests/unit/orm/__init__.py +0 -19
- nucliadb/ingest/tests/unit/orm/test_brain.py +0 -247
- nucliadb/ingest/tests/unit/orm/test_processor.py +0 -131
- nucliadb/ingest/tests/unit/orm/test_resource.py +0 -275
- nucliadb/ingest/tests/unit/test_partitions.py +0 -40
- nucliadb/ingest/tests/unit/test_processing.py +0 -171
- nucliadb/middleware/transaction.py +0 -117
- nucliadb/reader/api/v1/learning_collector.py +0 -63
- nucliadb/reader/tests/__init__.py +0 -19
- nucliadb/reader/tests/conftest.py +0 -31
- nucliadb/reader/tests/fixtures.py +0 -136
- nucliadb/reader/tests/test_list_resources.py +0 -75
- nucliadb/reader/tests/test_reader_file_download.py +0 -273
- nucliadb/reader/tests/test_reader_resource.py +0 -379
- nucliadb/reader/tests/test_reader_resource_field.py +0 -219
- nucliadb/search/api/v1/chat.py +0 -258
- nucliadb/search/api/v1/resource/chat.py +0 -94
- nucliadb/search/tests/__init__.py +0 -19
- nucliadb/search/tests/conftest.py +0 -33
- nucliadb/search/tests/fixtures.py +0 -199
- nucliadb/search/tests/node.py +0 -465
- nucliadb/search/tests/unit/__init__.py +0 -18
- nucliadb/search/tests/unit/api/__init__.py +0 -19
- nucliadb/search/tests/unit/api/v1/__init__.py +0 -19
- nucliadb/search/tests/unit/api/v1/resource/__init__.py +0 -19
- nucliadb/search/tests/unit/api/v1/resource/test_ask.py +0 -67
- nucliadb/search/tests/unit/api/v1/resource/test_chat.py +0 -97
- nucliadb/search/tests/unit/api/v1/test_chat.py +0 -96
- nucliadb/search/tests/unit/api/v1/test_predict_proxy.py +0 -98
- nucliadb/search/tests/unit/api/v1/test_summarize.py +0 -93
- nucliadb/search/tests/unit/search/__init__.py +0 -18
- nucliadb/search/tests/unit/search/requesters/__init__.py +0 -18
- nucliadb/search/tests/unit/search/requesters/test_utils.py +0 -210
- nucliadb/search/tests/unit/search/search/__init__.py +0 -19
- nucliadb/search/tests/unit/search/search/test_shards.py +0 -45
- nucliadb/search/tests/unit/search/search/test_utils.py +0 -82
- nucliadb/search/tests/unit/search/test_chat_prompt.py +0 -266
- nucliadb/search/tests/unit/search/test_fetch.py +0 -108
- nucliadb/search/tests/unit/search/test_filters.py +0 -125
- nucliadb/search/tests/unit/search/test_paragraphs.py +0 -157
- nucliadb/search/tests/unit/search/test_predict_proxy.py +0 -106
- nucliadb/search/tests/unit/search/test_query.py +0 -201
- nucliadb/search/tests/unit/test_app.py +0 -79
- nucliadb/search/tests/unit/test_find_merge.py +0 -112
- nucliadb/search/tests/unit/test_merge.py +0 -34
- nucliadb/search/tests/unit/test_predict.py +0 -584
- nucliadb/standalone/tests/__init__.py +0 -19
- nucliadb/standalone/tests/conftest.py +0 -33
- nucliadb/standalone/tests/fixtures.py +0 -38
- nucliadb/standalone/tests/unit/__init__.py +0 -18
- nucliadb/standalone/tests/unit/test_api_router.py +0 -61
- nucliadb/standalone/tests/unit/test_auth.py +0 -169
- nucliadb/standalone/tests/unit/test_introspect.py +0 -35
- nucliadb/standalone/tests/unit/test_versions.py +0 -68
- nucliadb/tests/benchmarks/__init__.py +0 -19
- nucliadb/tests/benchmarks/test_search.py +0 -99
- nucliadb/tests/conftest.py +0 -32
- nucliadb/tests/fixtures.py +0 -736
- nucliadb/tests/knowledgeboxes/philosophy_books.py +0 -203
- nucliadb/tests/knowledgeboxes/ten_dummy_resources.py +0 -109
- nucliadb/tests/migrations/__init__.py +0 -19
- nucliadb/tests/migrations/test_migration_0017.py +0 -80
- nucliadb/tests/tikv.py +0 -240
- nucliadb/tests/unit/__init__.py +0 -19
- nucliadb/tests/unit/common/__init__.py +0 -19
- nucliadb/tests/unit/common/cluster/__init__.py +0 -19
- nucliadb/tests/unit/common/cluster/discovery/__init__.py +0 -19
- nucliadb/tests/unit/common/cluster/discovery/test_k8s.py +0 -170
- nucliadb/tests/unit/common/cluster/standalone/__init__.py +0 -18
- nucliadb/tests/unit/common/cluster/standalone/test_service.py +0 -113
- nucliadb/tests/unit/common/cluster/standalone/test_utils.py +0 -59
- nucliadb/tests/unit/common/cluster/test_cluster.py +0 -399
- nucliadb/tests/unit/common/cluster/test_kb_shard_manager.py +0 -178
- nucliadb/tests/unit/common/cluster/test_rollover.py +0 -279
- nucliadb/tests/unit/common/maindb/__init__.py +0 -18
- nucliadb/tests/unit/common/maindb/test_driver.py +0 -127
- nucliadb/tests/unit/common/maindb/test_tikv.py +0 -53
- nucliadb/tests/unit/common/maindb/test_utils.py +0 -81
- nucliadb/tests/unit/common/test_context.py +0 -36
- nucliadb/tests/unit/export_import/__init__.py +0 -19
- nucliadb/tests/unit/export_import/test_datamanager.py +0 -37
- nucliadb/tests/unit/export_import/test_utils.py +0 -294
- nucliadb/tests/unit/migrator/__init__.py +0 -19
- nucliadb/tests/unit/migrator/test_migrator.py +0 -87
- nucliadb/tests/unit/tasks/__init__.py +0 -19
- nucliadb/tests/unit/tasks/conftest.py +0 -42
- nucliadb/tests/unit/tasks/test_consumer.py +0 -93
- nucliadb/tests/unit/tasks/test_producer.py +0 -95
- nucliadb/tests/unit/tasks/test_tasks.py +0 -60
- nucliadb/tests/unit/test_field_ids.py +0 -49
- nucliadb/tests/unit/test_health.py +0 -84
- nucliadb/tests/unit/test_kb_slugs.py +0 -54
- nucliadb/tests/unit/test_learning_proxy.py +0 -252
- nucliadb/tests/unit/test_metrics_exporter.py +0 -77
- nucliadb/tests/unit/test_purge.py +0 -138
- nucliadb/tests/utils/__init__.py +0 -74
- nucliadb/tests/utils/aiohttp_session.py +0 -44
- nucliadb/tests/utils/broker_messages/__init__.py +0 -167
- nucliadb/tests/utils/broker_messages/fields.py +0 -181
- nucliadb/tests/utils/broker_messages/helpers.py +0 -33
- nucliadb/tests/utils/entities.py +0 -78
- nucliadb/train/api/v1/check.py +0 -60
- nucliadb/train/tests/__init__.py +0 -19
- nucliadb/train/tests/conftest.py +0 -29
- nucliadb/train/tests/fixtures.py +0 -342
- nucliadb/train/tests/test_field_classification.py +0 -122
- nucliadb/train/tests/test_get_entities.py +0 -80
- nucliadb/train/tests/test_get_info.py +0 -51
- nucliadb/train/tests/test_get_ontology.py +0 -34
- nucliadb/train/tests/test_get_ontology_count.py +0 -63
- nucliadb/train/tests/test_image_classification.py +0 -222
- nucliadb/train/tests/test_list_fields.py +0 -39
- nucliadb/train/tests/test_list_paragraphs.py +0 -73
- nucliadb/train/tests/test_list_resources.py +0 -39
- nucliadb/train/tests/test_list_sentences.py +0 -71
- nucliadb/train/tests/test_paragraph_classification.py +0 -123
- nucliadb/train/tests/test_paragraph_streaming.py +0 -118
- nucliadb/train/tests/test_question_answer_streaming.py +0 -239
- nucliadb/train/tests/test_sentence_classification.py +0 -143
- nucliadb/train/tests/test_token_classification.py +0 -136
- nucliadb/train/tests/utils.py +0 -108
- nucliadb/writer/layouts/__init__.py +0 -51
- nucliadb/writer/layouts/v1.py +0 -59
- nucliadb/writer/resource/vectors.py +0 -120
- nucliadb/writer/tests/__init__.py +0 -19
- nucliadb/writer/tests/conftest.py +0 -31
- nucliadb/writer/tests/fixtures.py +0 -192
- nucliadb/writer/tests/test_fields.py +0 -486
- nucliadb/writer/tests/test_files.py +0 -743
- nucliadb/writer/tests/test_knowledgebox.py +0 -49
- nucliadb/writer/tests/test_reprocess_file_field.py +0 -139
- nucliadb/writer/tests/test_resources.py +0 -546
- nucliadb/writer/tests/test_service.py +0 -137
- nucliadb/writer/tests/test_tus.py +0 -203
- nucliadb/writer/tests/utils.py +0 -35
- nucliadb/writer/tus/pg.py +0 -125
- nucliadb-2.46.1.post382.dist-info/METADATA +0 -134
- nucliadb-2.46.1.post382.dist-info/RECORD +0 -451
- {nucliadb/ingest/tests → migrations/pg}/__init__.py +0 -0
- /nucliadb/{ingest/tests/integration → common/external_index_providers}/__init__.py +0 -0
- /nucliadb/{ingest/tests/integration/ingest → common/models_utils}/__init__.py +0 -0
- /nucliadb/{ingest/tests/unit → search/search/query_parser}/__init__.py +0 -0
- /nucliadb/{ingest/tests → tests}/vectors.py +0 -0
- {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/entry_points.txt +0 -0
- {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/top_level.txt +0 -0
- {nucliadb-2.46.1.post382.dist-info → nucliadb-6.2.1.post2777.dist-info}/zip-safe +0 -0
@@ -1,192 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
20
|
-
from enum import Enum
|
21
|
-
from typing import AsyncIterator, Callable, Optional
|
22
|
-
from unittest import mock
|
23
|
-
|
24
|
-
import pytest
|
25
|
-
from httpx import AsyncClient
|
26
|
-
from pytest_lazy_fixtures import lazy_fixture
|
27
|
-
from redis import asyncio as aioredis
|
28
|
-
|
29
|
-
from nucliadb.ingest.tests.fixtures import IngestFixture
|
30
|
-
from nucliadb.writer import API_PREFIX
|
31
|
-
from nucliadb.writer.api.v1.router import KB_PREFIX, KBS_PREFIX
|
32
|
-
from nucliadb.writer.settings import settings
|
33
|
-
from nucliadb.writer.tus import clear_storage
|
34
|
-
from nucliadb_models.resource import NucliaDBRoles
|
35
|
-
from nucliadb_utils.settings import (
|
36
|
-
FileBackendConfig,
|
37
|
-
nuclia_settings,
|
38
|
-
nucliadb_settings,
|
39
|
-
storage_settings,
|
40
|
-
)
|
41
|
-
from nucliadb_utils.tests.conftest import get_testing_storage_backend
|
42
|
-
from nucliadb_utils.utilities import Utility, clean_utility, set_utility
|
43
|
-
|
44
|
-
|
45
|
-
@pytest.fixture(scope="function")
|
46
|
-
def disabled_back_pressure():
|
47
|
-
with mock.patch(
|
48
|
-
"nucliadb.writer.back_pressure.is_back_pressure_enabled", return_value=False
|
49
|
-
) as mocked:
|
50
|
-
yield mocked
|
51
|
-
|
52
|
-
|
53
|
-
@pytest.fixture(scope="function")
|
54
|
-
async def writer_api(
|
55
|
-
disabled_back_pressure,
|
56
|
-
redis,
|
57
|
-
storage_writer,
|
58
|
-
grpc_servicer: IngestFixture,
|
59
|
-
transaction_utility,
|
60
|
-
processing_utility,
|
61
|
-
tus_manager,
|
62
|
-
) -> AsyncIterator[Callable[[list[Enum], str, str], AsyncClient]]:
|
63
|
-
nucliadb_settings.nucliadb_ingest = grpc_servicer.host
|
64
|
-
from nucliadb.writer.app import create_application
|
65
|
-
|
66
|
-
application = create_application()
|
67
|
-
|
68
|
-
def make_client_fixture(
|
69
|
-
roles: Optional[list[Enum]] = None,
|
70
|
-
user: str = "",
|
71
|
-
version: str = "1",
|
72
|
-
) -> AsyncClient:
|
73
|
-
roles = roles or []
|
74
|
-
client_base_url = "http://test"
|
75
|
-
client_base_url = f"{client_base_url}/{API_PREFIX}/v{version}"
|
76
|
-
|
77
|
-
client = AsyncClient(app=application, base_url=client_base_url) # type: ignore
|
78
|
-
client.headers["X-NUCLIADB-ROLES"] = ";".join(
|
79
|
-
map(lambda role: role.value, roles)
|
80
|
-
)
|
81
|
-
client.headers["X-NUCLIADB-USER"] = user
|
82
|
-
|
83
|
-
return client
|
84
|
-
|
85
|
-
driver = aioredis.from_url(f"redis://{redis[0]}:{redis[1]}")
|
86
|
-
await driver.flushall()
|
87
|
-
|
88
|
-
await application.router.startup()
|
89
|
-
|
90
|
-
yield make_client_fixture
|
91
|
-
|
92
|
-
await application.router.shutdown()
|
93
|
-
clear_storage()
|
94
|
-
|
95
|
-
await driver.flushall()
|
96
|
-
await driver.close(close_connection_pool=True)
|
97
|
-
|
98
|
-
|
99
|
-
@pytest.fixture(scope="function")
|
100
|
-
def gcs_storage_writer(gcs):
|
101
|
-
storage_settings.file_backend = FileBackendConfig.GCS
|
102
|
-
storage_settings.gcs_endpoint_url = gcs
|
103
|
-
storage_settings.gcs_bucket = "test_{kbid}"
|
104
|
-
|
105
|
-
|
106
|
-
@pytest.fixture(scope="function")
|
107
|
-
def s3_storage_writer(s3):
|
108
|
-
storage_settings.file_backend = FileBackendConfig.S3
|
109
|
-
storage_settings.s3_endpoint = s3
|
110
|
-
storage_settings.s3_client_id = ""
|
111
|
-
storage_settings.s3_client_secret = ""
|
112
|
-
storage_settings.s3_bucket = "test-{kbid}"
|
113
|
-
|
114
|
-
|
115
|
-
@pytest.fixture(scope="function")
|
116
|
-
def pg_storage_writer(pg):
|
117
|
-
storage_settings.file_backend = FileBackendConfig.PG
|
118
|
-
url = f"postgresql://postgres:postgres@{pg[0]}:{pg[1]}/postgres"
|
119
|
-
storage_settings.driver_pg_url = url
|
120
|
-
|
121
|
-
|
122
|
-
def lazy_storage_writer_fixture():
|
123
|
-
backend = get_testing_storage_backend()
|
124
|
-
if backend == "gcs":
|
125
|
-
return [lazy_fixture.lf("gcs_storage_writer")]
|
126
|
-
elif backend == "s3":
|
127
|
-
return [lazy_fixture.lf("s3_storage_writer")]
|
128
|
-
elif backend == "pg":
|
129
|
-
return [lazy_fixture.lf("pg_storage_writer")]
|
130
|
-
else:
|
131
|
-
print(f"Unknown storage backend {backend}, using gcs")
|
132
|
-
return [lazy_fixture.lf("gcs_storage_writer")]
|
133
|
-
|
134
|
-
|
135
|
-
@pytest.fixture(scope="function", params=lazy_storage_writer_fixture())
|
136
|
-
async def storage_writer(request):
|
137
|
-
"""
|
138
|
-
Generic storage fixture that allows us to run the same tests for different storage backends.
|
139
|
-
"""
|
140
|
-
storage_driver = request.param
|
141
|
-
set_utility(Utility.STORAGE, storage_driver)
|
142
|
-
|
143
|
-
yield storage_driver
|
144
|
-
|
145
|
-
clean_utility(Utility.STORAGE)
|
146
|
-
|
147
|
-
|
148
|
-
@pytest.fixture(scope="function")
|
149
|
-
async def knowledgebox_writer(writer_api):
|
150
|
-
async with writer_api(roles=[NucliaDBRoles.MANAGER]) as client:
|
151
|
-
resp = await client.post(
|
152
|
-
f"/{KBS_PREFIX}",
|
153
|
-
json={
|
154
|
-
"slug": "kbid1",
|
155
|
-
"title": "My Knowledge Box",
|
156
|
-
},
|
157
|
-
)
|
158
|
-
assert resp.status_code == 201
|
159
|
-
kbid = resp.json().get("uuid")
|
160
|
-
assert kbid is not None
|
161
|
-
yield kbid
|
162
|
-
|
163
|
-
|
164
|
-
@pytest.fixture(scope="function")
|
165
|
-
async def resource(redis, writer_api, knowledgebox_writer):
|
166
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
167
|
-
resp = await client.post(
|
168
|
-
f"/{KB_PREFIX}/{knowledgebox_writer}/resources",
|
169
|
-
headers={"X-Synchronous": "true"},
|
170
|
-
json={
|
171
|
-
"slug": "resource1",
|
172
|
-
"title": "Resource 1",
|
173
|
-
},
|
174
|
-
)
|
175
|
-
assert resp.status_code == 201
|
176
|
-
uuid = resp.json()["uuid"]
|
177
|
-
|
178
|
-
return uuid
|
179
|
-
|
180
|
-
|
181
|
-
@pytest.fixture(scope="function")
|
182
|
-
async def processing_utility():
|
183
|
-
nuclia_settings.dummy_processing = True
|
184
|
-
nuclia_settings.onprem = True
|
185
|
-
nuclia_settings.nuclia_jwt_key = "foobarkey"
|
186
|
-
|
187
|
-
|
188
|
-
@pytest.fixture(scope="function")
|
189
|
-
async def tus_manager(redis):
|
190
|
-
settings.dm_redis_host = redis[0]
|
191
|
-
settings.dm_redis_port = redis[1]
|
192
|
-
yield
|
@@ -1,486 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
20
|
-
|
21
|
-
from copy import deepcopy
|
22
|
-
from datetime import datetime
|
23
|
-
from os.path import dirname
|
24
|
-
|
25
|
-
import pytest
|
26
|
-
|
27
|
-
from nucliadb.writer.api.v1.router import (
|
28
|
-
KB_PREFIX,
|
29
|
-
RESOURCE_PREFIX,
|
30
|
-
RESOURCES_PREFIX,
|
31
|
-
RSLUG_PREFIX,
|
32
|
-
)
|
33
|
-
from nucliadb.writer.tests.utils import load_file_as_FileB64_payload
|
34
|
-
from nucliadb_models.resource import NucliaDBRoles
|
35
|
-
|
36
|
-
TEST_FILE = {f"{dirname(__file__)}/orm/"}
|
37
|
-
TEST_TEXT_PAYLOAD = {"body": "test1", "format": "PLAIN"}
|
38
|
-
TEST_LINK_PAYLOAD = {
|
39
|
-
"added": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
40
|
-
"headers": {},
|
41
|
-
"cookies": {},
|
42
|
-
"uri": "http://some-link.com",
|
43
|
-
"language": "en",
|
44
|
-
"localstorage": {},
|
45
|
-
"css_selector": "main",
|
46
|
-
}
|
47
|
-
TEST_KEYWORDSETS_PAYLOAD = {"keywords": [{"value": "kw1"}, {"value": "kw2"}]}
|
48
|
-
TEST_DATETIMES_PAYLOAD = {"value": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}
|
49
|
-
TEST_CONVERSATION_PAYLOAD = {
|
50
|
-
"messages": [
|
51
|
-
{
|
52
|
-
"timestamp": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
53
|
-
"who": "Bob",
|
54
|
-
"to": ["Alice", "Charlie"],
|
55
|
-
"content": {
|
56
|
-
"text": "Hi people!",
|
57
|
-
"format": "PLAIN",
|
58
|
-
"files": [
|
59
|
-
load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg")
|
60
|
-
],
|
61
|
-
},
|
62
|
-
"ident": "message_id_001",
|
63
|
-
}
|
64
|
-
]
|
65
|
-
}
|
66
|
-
TEST_LAYOUT_PAYLOAD = {
|
67
|
-
"body": {
|
68
|
-
"blocks": {
|
69
|
-
"block1": {
|
70
|
-
"x": 0,
|
71
|
-
"y": 0,
|
72
|
-
"cols": 1,
|
73
|
-
"rows": 2,
|
74
|
-
"type": "TITLE",
|
75
|
-
"ident": "main_title",
|
76
|
-
"payload": "This is a Test Title",
|
77
|
-
"file": load_file_as_FileB64_payload(
|
78
|
-
"/assets/image001.jpg", "image/jpg"
|
79
|
-
),
|
80
|
-
}
|
81
|
-
}
|
82
|
-
},
|
83
|
-
"format": "NUCLIAv1",
|
84
|
-
}
|
85
|
-
|
86
|
-
TEST_FILE_PAYLOAD = {
|
87
|
-
"language": "en",
|
88
|
-
"password": "xxxxxx",
|
89
|
-
"file": load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg"),
|
90
|
-
}
|
91
|
-
|
92
|
-
TEST_EXTERNAL_FILE_PAYLOAD = {
|
93
|
-
"file": {
|
94
|
-
"uri": "https://mysite.com/files/myfile.pdf",
|
95
|
-
"extra_headers": {"foo": "bar"},
|
96
|
-
}
|
97
|
-
}
|
98
|
-
|
99
|
-
TEST_CONVERSATION_APPEND_MESSAGES_PAYLOAD = [
|
100
|
-
{
|
101
|
-
"timestamp": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
102
|
-
"who": "Bob",
|
103
|
-
"to": ["Alice", "Charlie"],
|
104
|
-
"content": {
|
105
|
-
"text": "Hi people!",
|
106
|
-
"format": "PLAIN",
|
107
|
-
"attachments": [
|
108
|
-
load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg")
|
109
|
-
],
|
110
|
-
},
|
111
|
-
"ident": "message_id_001",
|
112
|
-
}
|
113
|
-
]
|
114
|
-
|
115
|
-
TEST_LAYOUT_APPEND_BLOCKS_PAYLOAD = {
|
116
|
-
"block1": {
|
117
|
-
"x": 0,
|
118
|
-
"y": 0,
|
119
|
-
"cols": 1,
|
120
|
-
"rows": 2,
|
121
|
-
"type": "TITLE",
|
122
|
-
"ident": "main_title",
|
123
|
-
"payload": "This is a Test Title",
|
124
|
-
"file": load_file_as_FileB64_payload("/assets/image001.jpg", "image/jpg"),
|
125
|
-
}
|
126
|
-
}
|
127
|
-
|
128
|
-
|
129
|
-
@pytest.mark.asyncio
|
130
|
-
async def test_resource_field_add(writer_api, knowledgebox_writer):
|
131
|
-
knowledgebox_id = knowledgebox_writer
|
132
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
133
|
-
resp = await client.post(
|
134
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
|
135
|
-
headers={"X-SYNCHRONOUS": "True"},
|
136
|
-
json={"slug": "resource1", "title": "My resource"},
|
137
|
-
)
|
138
|
-
assert resp.status_code == 201
|
139
|
-
data = resp.json()
|
140
|
-
assert "uuid" in data
|
141
|
-
assert "seqid" in data
|
142
|
-
rid = data["uuid"]
|
143
|
-
|
144
|
-
# Text
|
145
|
-
resp = await client.put(
|
146
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/text/text1",
|
147
|
-
json=TEST_TEXT_PAYLOAD,
|
148
|
-
)
|
149
|
-
assert resp.status_code == 201
|
150
|
-
data = resp.json()
|
151
|
-
assert "seqid" in data
|
152
|
-
|
153
|
-
# Link
|
154
|
-
resp = await client.put(
|
155
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/link/link1",
|
156
|
-
json=TEST_LINK_PAYLOAD,
|
157
|
-
)
|
158
|
-
assert resp.status_code == 201
|
159
|
-
data = resp.json()
|
160
|
-
assert "seqid" in data
|
161
|
-
|
162
|
-
# Keywordset
|
163
|
-
resp = await client.put(
|
164
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/keywordset/kws1",
|
165
|
-
json=TEST_KEYWORDSETS_PAYLOAD,
|
166
|
-
)
|
167
|
-
assert resp.status_code == 201
|
168
|
-
data = resp.json()
|
169
|
-
assert "seqid" in data
|
170
|
-
|
171
|
-
# Datetimes
|
172
|
-
|
173
|
-
resp = await client.put(
|
174
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/datetime/date1",
|
175
|
-
json=TEST_DATETIMES_PAYLOAD,
|
176
|
-
)
|
177
|
-
assert resp.status_code == 201
|
178
|
-
data = resp.json()
|
179
|
-
assert "seqid" in data
|
180
|
-
|
181
|
-
# Conversation
|
182
|
-
resp = await client.put(
|
183
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/conversation/conv1",
|
184
|
-
json=TEST_CONVERSATION_PAYLOAD,
|
185
|
-
)
|
186
|
-
|
187
|
-
assert resp.status_code == 201
|
188
|
-
data = resp.json()
|
189
|
-
assert "seqid" in data
|
190
|
-
|
191
|
-
# Layout
|
192
|
-
resp = await client.put(
|
193
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/layout/layout1",
|
194
|
-
json=TEST_LAYOUT_PAYLOAD,
|
195
|
-
)
|
196
|
-
|
197
|
-
assert resp.status_code == 201
|
198
|
-
data = resp.json()
|
199
|
-
assert "seqid" in data
|
200
|
-
|
201
|
-
# File
|
202
|
-
resp = await client.put(
|
203
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1",
|
204
|
-
json=TEST_FILE_PAYLOAD,
|
205
|
-
)
|
206
|
-
assert resp.status_code == 201
|
207
|
-
data = resp.json()
|
208
|
-
assert "seqid" in data
|
209
|
-
|
210
|
-
# File without storing it in the internal BrokerMessage, only send to process
|
211
|
-
resp = await client.put(
|
212
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1",
|
213
|
-
json=TEST_FILE_PAYLOAD,
|
214
|
-
headers={"x_skip_store": "1"},
|
215
|
-
)
|
216
|
-
assert resp.status_code == 201
|
217
|
-
data = resp.json()
|
218
|
-
assert "seqid" in data
|
219
|
-
|
220
|
-
# File field pointing to an externally hosted file
|
221
|
-
resp = await client.put(
|
222
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/externalfile",
|
223
|
-
json=TEST_EXTERNAL_FILE_PAYLOAD,
|
224
|
-
headers={"X-SYNCHRONOUS": "True"},
|
225
|
-
)
|
226
|
-
assert resp.status_code == 201
|
227
|
-
data = resp.json()
|
228
|
-
assert "seqid" in data
|
229
|
-
|
230
|
-
|
231
|
-
@pytest.mark.asyncio
|
232
|
-
async def test_resource_field_append_extra(writer_api, knowledgebox_writer):
|
233
|
-
knowledgebox_id = knowledgebox_writer
|
234
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
235
|
-
resp = await client.post(
|
236
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
|
237
|
-
headers={"X-SYNCHRONOUS": "True"},
|
238
|
-
json={
|
239
|
-
"slug": "resource1",
|
240
|
-
"title": "My resource",
|
241
|
-
"layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
|
242
|
-
"conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
|
243
|
-
},
|
244
|
-
)
|
245
|
-
assert resp.status_code == 201
|
246
|
-
data = resp.json()
|
247
|
-
assert "uuid" in data
|
248
|
-
assert "seqid" in data
|
249
|
-
rid = data["uuid"]
|
250
|
-
|
251
|
-
# Conversation
|
252
|
-
resp = await client.put(
|
253
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/conversation/conv1/messages",
|
254
|
-
json=TEST_CONVERSATION_APPEND_MESSAGES_PAYLOAD,
|
255
|
-
)
|
256
|
-
assert resp.status_code == 200
|
257
|
-
data = resp.json()
|
258
|
-
assert "seqid" in data
|
259
|
-
|
260
|
-
# Layout
|
261
|
-
resp = await client.put(
|
262
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/layout/layout1/blocks",
|
263
|
-
json=TEST_LAYOUT_APPEND_BLOCKS_PAYLOAD,
|
264
|
-
)
|
265
|
-
|
266
|
-
assert resp.status_code == 200
|
267
|
-
data = resp.json()
|
268
|
-
assert "seqid" in data
|
269
|
-
|
270
|
-
|
271
|
-
@pytest.mark.asyncio
|
272
|
-
async def test_resource_field_delete(writer_api, knowledgebox_writer):
|
273
|
-
knowledgebox_id = knowledgebox_writer
|
274
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
275
|
-
resp = await client.post(
|
276
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
|
277
|
-
headers={"X-SYNCHRONOUS": "True"},
|
278
|
-
json={
|
279
|
-
"slug": "resource1",
|
280
|
-
"title": "My resource",
|
281
|
-
"texts": {"text1": TEST_TEXT_PAYLOAD},
|
282
|
-
"links": {"link1": TEST_LINK_PAYLOAD},
|
283
|
-
"files": {"file1": TEST_FILE_PAYLOAD},
|
284
|
-
"layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
|
285
|
-
"conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
|
286
|
-
"keywordsets": {"keywordset1": TEST_KEYWORDSETS_PAYLOAD},
|
287
|
-
"datetimes": {"datetime1": TEST_DATETIMES_PAYLOAD},
|
288
|
-
},
|
289
|
-
)
|
290
|
-
|
291
|
-
assert resp.status_code == 201
|
292
|
-
data = resp.json()
|
293
|
-
rid = data["uuid"]
|
294
|
-
|
295
|
-
# Text
|
296
|
-
resp = await client.delete(
|
297
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/text/text1"
|
298
|
-
)
|
299
|
-
assert resp.status_code == 204
|
300
|
-
|
301
|
-
# Link
|
302
|
-
resp = await client.delete(
|
303
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/link/link1"
|
304
|
-
)
|
305
|
-
assert resp.status_code == 204
|
306
|
-
|
307
|
-
# Keywords
|
308
|
-
resp = await client.delete(
|
309
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/keywordset/kws1"
|
310
|
-
)
|
311
|
-
assert resp.status_code == 204
|
312
|
-
|
313
|
-
# Datetimes
|
314
|
-
|
315
|
-
resp = await client.delete(
|
316
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/datetime/date1"
|
317
|
-
)
|
318
|
-
assert resp.status_code == 204
|
319
|
-
|
320
|
-
# Conversation
|
321
|
-
resp = await client.delete(
|
322
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/conversation/conv1"
|
323
|
-
)
|
324
|
-
|
325
|
-
# Layout
|
326
|
-
resp = await client.delete(
|
327
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/layout/layout1"
|
328
|
-
)
|
329
|
-
assert resp.status_code == 204
|
330
|
-
|
331
|
-
# File
|
332
|
-
resp = await client.delete(
|
333
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1"
|
334
|
-
)
|
335
|
-
assert resp.status_code == 204
|
336
|
-
|
337
|
-
|
338
|
-
@pytest.mark.asyncio
|
339
|
-
@pytest.mark.parametrize(
|
340
|
-
"endpoint,payload",
|
341
|
-
[
|
342
|
-
("text/text1", TEST_TEXT_PAYLOAD),
|
343
|
-
("link/link1", TEST_LINK_PAYLOAD),
|
344
|
-
("keywordset/kws1", TEST_KEYWORDSETS_PAYLOAD),
|
345
|
-
("datetime/date1", TEST_DATETIMES_PAYLOAD),
|
346
|
-
("conversation/conv1", TEST_CONVERSATION_PAYLOAD),
|
347
|
-
("conversation/conv1/messages", TEST_CONVERSATION_APPEND_MESSAGES_PAYLOAD),
|
348
|
-
("layout/layout1", TEST_LAYOUT_PAYLOAD),
|
349
|
-
("layout/layout1/blocks", TEST_LAYOUT_APPEND_BLOCKS_PAYLOAD),
|
350
|
-
("file/file1", TEST_FILE_PAYLOAD),
|
351
|
-
],
|
352
|
-
)
|
353
|
-
async def test_sync_ops(writer_api, knowledgebox_writer, endpoint, payload):
|
354
|
-
knowledgebox_id = knowledgebox_writer
|
355
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
356
|
-
HEADERS = {"X-SYNCHRONOUS": "True"}
|
357
|
-
# Create a resource
|
358
|
-
resp = await client.post(
|
359
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
|
360
|
-
headers=HEADERS,
|
361
|
-
json={
|
362
|
-
"slug": "resource1",
|
363
|
-
"title": "My resource",
|
364
|
-
"layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
|
365
|
-
"conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
|
366
|
-
},
|
367
|
-
)
|
368
|
-
assert resp.status_code == 201
|
369
|
-
data = resp.json()
|
370
|
-
rid = data["uuid"]
|
371
|
-
|
372
|
-
resource_path = f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}"
|
373
|
-
resp = await client.put(
|
374
|
-
f"{resource_path}/{endpoint}",
|
375
|
-
headers={"X-SYNCHRONOUS": "True"},
|
376
|
-
json=payload,
|
377
|
-
)
|
378
|
-
assert resp.status_code in (201, 200)
|
379
|
-
|
380
|
-
|
381
|
-
@pytest.mark.asyncio
|
382
|
-
async def test_external_file_field(writer_api, knowledgebox_writer):
|
383
|
-
knowledgebox_id = knowledgebox_writer
|
384
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
385
|
-
resp = await client.post(
|
386
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
|
387
|
-
json={"slug": "resource1", "title": "My resource"},
|
388
|
-
headers={"X-SYNCHRONOUS": "True"},
|
389
|
-
)
|
390
|
-
assert resp.status_code == 201
|
391
|
-
rid = resp.json()["uuid"]
|
392
|
-
|
393
|
-
# File field pointing to an externally hosted file
|
394
|
-
resp = await client.put(
|
395
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/externalfile",
|
396
|
-
json=TEST_EXTERNAL_FILE_PAYLOAD,
|
397
|
-
headers={"X-SYNCHRONOUS": "True"},
|
398
|
-
)
|
399
|
-
assert resp.status_code == 201
|
400
|
-
|
401
|
-
|
402
|
-
@pytest.mark.asyncio
|
403
|
-
async def test_file_field_validation(writer_api, knowledgebox_writer):
|
404
|
-
knowledgebox_id = knowledgebox_writer
|
405
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
406
|
-
resp = await client.post(
|
407
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
|
408
|
-
json={"slug": "resource1", "title": "My resource"},
|
409
|
-
headers={"X-SYNCHRONOUS": "True"},
|
410
|
-
)
|
411
|
-
assert resp.status_code == 201
|
412
|
-
rid = resp.json()["uuid"]
|
413
|
-
|
414
|
-
# Remove a required key from the payload
|
415
|
-
payload = deepcopy(TEST_FILE_PAYLOAD)
|
416
|
-
payload["file"].pop("md5")
|
417
|
-
|
418
|
-
resp = await client.put(
|
419
|
-
f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}/file/file1",
|
420
|
-
json=payload,
|
421
|
-
headers={"X-SYNCHRONOUS": "True"},
|
422
|
-
)
|
423
|
-
assert resp.status_code == 201
|
424
|
-
|
425
|
-
|
426
|
-
@pytest.mark.parametrize(
|
427
|
-
"method,endpoint,payload",
|
428
|
-
[
|
429
|
-
["put", "/text/{field_id}", TEST_TEXT_PAYLOAD],
|
430
|
-
["put", "/link/{field_id}", TEST_LINK_PAYLOAD],
|
431
|
-
["put", "/keywordset/{field_id}", TEST_KEYWORDSETS_PAYLOAD],
|
432
|
-
["put", "/datetime/{field_id}", TEST_DATETIMES_PAYLOAD],
|
433
|
-
["put", "/layout/{field_id}", TEST_LAYOUT_PAYLOAD],
|
434
|
-
["put", "/conversation/{field_id}", TEST_CONVERSATION_PAYLOAD],
|
435
|
-
["put", "/file/{field_id}", TEST_FILE_PAYLOAD],
|
436
|
-
["delete", "", None],
|
437
|
-
],
|
438
|
-
)
|
439
|
-
@pytest.mark.asyncio()
|
440
|
-
async def test_field_endpoints_by_slug(
|
441
|
-
writer_api,
|
442
|
-
knowledgebox_ingest,
|
443
|
-
method,
|
444
|
-
endpoint,
|
445
|
-
payload,
|
446
|
-
):
|
447
|
-
async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
|
448
|
-
slug = "my-resource"
|
449
|
-
field_id = "myfield"
|
450
|
-
field_type = "text"
|
451
|
-
|
452
|
-
resp = await client.post(
|
453
|
-
f"/{KB_PREFIX}/{knowledgebox_ingest}/{RESOURCES_PREFIX}",
|
454
|
-
headers={"X-SYNCHRONOUS": "True"},
|
455
|
-
json={"slug": slug},
|
456
|
-
)
|
457
|
-
assert resp.status_code == 201
|
458
|
-
|
459
|
-
extra_params = {}
|
460
|
-
if payload is not None:
|
461
|
-
extra_params["json"] = payload
|
462
|
-
op = getattr(client, method)
|
463
|
-
|
464
|
-
# Try first a non-existing slug should return 404
|
465
|
-
url = endpoint.format(
|
466
|
-
field_id=field_id,
|
467
|
-
field_type=field_type,
|
468
|
-
)
|
469
|
-
|
470
|
-
resp = await op(
|
471
|
-
f"/{KB_PREFIX}/{knowledgebox_ingest}/{RSLUG_PREFIX}/idonotexist" + url,
|
472
|
-
**extra_params,
|
473
|
-
)
|
474
|
-
assert resp.status_code == 404
|
475
|
-
assert resp.json()["detail"] == "Resource does not exist"
|
476
|
-
|
477
|
-
# Try the happy path now
|
478
|
-
url = endpoint.format(
|
479
|
-
field_id=field_id,
|
480
|
-
field_type=field_type,
|
481
|
-
)
|
482
|
-
resp = await op(
|
483
|
-
f"/{KB_PREFIX}/{knowledgebox_ingest}/{RSLUG_PREFIX}/{slug}" + url,
|
484
|
-
**extra_params,
|
485
|
-
)
|
486
|
-
assert str(resp.status_code).startswith("2")
|