nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- migrations/0003_allfields_key.py +1 -35
- migrations/0009_upgrade_relations_and_texts_to_v2.py +4 -2
- migrations/0010_fix_corrupt_indexes.py +10 -10
- migrations/0011_materialize_labelset_ids.py +1 -16
- migrations/0012_rollover_shards.py +5 -10
- migrations/0014_rollover_shards.py +4 -5
- migrations/0015_targeted_rollover.py +5 -10
- migrations/0016_upgrade_to_paragraphs_v2.py +25 -28
- migrations/0017_multiple_writable_shards.py +2 -4
- migrations/0018_purge_orphan_kbslugs.py +5 -7
- migrations/0019_upgrade_to_paragraphs_v3.py +25 -28
- migrations/0020_drain_nodes_from_cluster.py +3 -3
- nucliadb/standalone/tests/unit/test_run.py → migrations/0021_overwrite_vectorsets_key.py +16 -19
- nucliadb/tests/unit/test_openapi.py → migrations/0022_fix_paragraph_deletion_bug.py +16 -11
- migrations/0023_backfill_pg_catalog.py +80 -0
- migrations/0025_assign_models_to_kbs_v2.py +113 -0
- migrations/0026_fix_high_cardinality_content_types.py +61 -0
- migrations/0027_rollover_texts3.py +73 -0
- nucliadb/ingest/fields/date.py → migrations/pg/0001_bootstrap.py +10 -12
- migrations/pg/0002_catalog.py +42 -0
- nucliadb/ingest/tests/unit/test_settings.py → migrations/pg/0003_catalog_kbid_index.py +5 -3
- nucliadb/common/cluster/base.py +30 -16
- nucliadb/common/cluster/discovery/base.py +6 -14
- nucliadb/common/cluster/discovery/k8s.py +9 -19
- nucliadb/common/cluster/discovery/manual.py +1 -3
- nucliadb/common/cluster/discovery/utils.py +1 -3
- nucliadb/common/cluster/grpc_node_dummy.py +3 -11
- nucliadb/common/cluster/index_node.py +10 -19
- nucliadb/common/cluster/manager.py +174 -59
- nucliadb/common/cluster/rebalance.py +27 -29
- nucliadb/common/cluster/rollover.py +353 -194
- nucliadb/common/cluster/settings.py +6 -0
- nucliadb/common/cluster/standalone/grpc_node_binding.py +13 -64
- nucliadb/common/cluster/standalone/index_node.py +4 -11
- nucliadb/common/cluster/standalone/service.py +2 -6
- nucliadb/common/cluster/standalone/utils.py +2 -6
- nucliadb/common/cluster/utils.py +29 -22
- nucliadb/common/constants.py +20 -0
- nucliadb/common/context/__init__.py +3 -0
- nucliadb/common/context/fastapi.py +8 -5
- nucliadb/{tests/knowledgeboxes/__init__.py → common/counters.py} +8 -2
- nucliadb/common/datamanagers/__init__.py +7 -1
- nucliadb/common/datamanagers/atomic.py +22 -4
- nucliadb/common/datamanagers/cluster.py +5 -5
- nucliadb/common/datamanagers/entities.py +6 -16
- nucliadb/common/datamanagers/fields.py +84 -0
- nucliadb/common/datamanagers/kb.py +83 -37
- nucliadb/common/datamanagers/labels.py +26 -56
- nucliadb/common/datamanagers/processing.py +2 -6
- nucliadb/common/datamanagers/resources.py +41 -103
- nucliadb/common/datamanagers/rollover.py +76 -15
- nucliadb/common/datamanagers/synonyms.py +1 -1
- nucliadb/common/datamanagers/utils.py +15 -6
- nucliadb/common/datamanagers/vectorsets.py +110 -0
- nucliadb/common/external_index_providers/base.py +257 -0
- nucliadb/{ingest/tests/unit/orm/test_orm_utils.py → common/external_index_providers/exceptions.py} +9 -8
- nucliadb/common/external_index_providers/manager.py +101 -0
- nucliadb/common/external_index_providers/pinecone.py +933 -0
- nucliadb/common/external_index_providers/settings.py +52 -0
- nucliadb/common/http_clients/auth.py +3 -6
- nucliadb/common/http_clients/processing.py +6 -11
- nucliadb/common/http_clients/utils.py +1 -3
- nucliadb/common/ids.py +240 -0
- nucliadb/common/locking.py +29 -7
- nucliadb/common/maindb/driver.py +11 -35
- nucliadb/common/maindb/exceptions.py +3 -0
- nucliadb/common/maindb/local.py +22 -9
- nucliadb/common/maindb/pg.py +206 -111
- nucliadb/common/maindb/utils.py +11 -42
- nucliadb/common/models_utils/from_proto.py +479 -0
- nucliadb/common/models_utils/to_proto.py +60 -0
- nucliadb/common/nidx.py +260 -0
- nucliadb/export_import/datamanager.py +25 -19
- nucliadb/export_import/exporter.py +5 -11
- nucliadb/export_import/importer.py +5 -7
- nucliadb/export_import/models.py +3 -3
- nucliadb/export_import/tasks.py +4 -4
- nucliadb/export_import/utils.py +25 -37
- nucliadb/health.py +1 -3
- nucliadb/ingest/app.py +15 -11
- nucliadb/ingest/consumer/auditing.py +21 -19
- nucliadb/ingest/consumer/consumer.py +82 -47
- nucliadb/ingest/consumer/materializer.py +5 -12
- nucliadb/ingest/consumer/pull.py +12 -27
- nucliadb/ingest/consumer/service.py +19 -17
- nucliadb/ingest/consumer/shard_creator.py +2 -4
- nucliadb/ingest/consumer/utils.py +1 -3
- nucliadb/ingest/fields/base.py +137 -105
- nucliadb/ingest/fields/conversation.py +18 -5
- nucliadb/ingest/fields/exceptions.py +1 -4
- nucliadb/ingest/fields/file.py +7 -16
- nucliadb/ingest/fields/link.py +5 -10
- nucliadb/ingest/fields/text.py +9 -4
- nucliadb/ingest/orm/brain.py +200 -213
- nucliadb/ingest/orm/broker_message.py +181 -0
- nucliadb/ingest/orm/entities.py +36 -51
- nucliadb/ingest/orm/exceptions.py +12 -0
- nucliadb/ingest/orm/knowledgebox.py +322 -197
- nucliadb/ingest/orm/processor/__init__.py +2 -700
- nucliadb/ingest/orm/processor/auditing.py +4 -23
- nucliadb/ingest/orm/processor/data_augmentation.py +164 -0
- nucliadb/ingest/orm/processor/pgcatalog.py +84 -0
- nucliadb/ingest/orm/processor/processor.py +752 -0
- nucliadb/ingest/orm/processor/sequence_manager.py +1 -1
- nucliadb/ingest/orm/resource.py +249 -402
- nucliadb/ingest/orm/utils.py +4 -4
- nucliadb/ingest/partitions.py +3 -9
- nucliadb/ingest/processing.py +64 -73
- nucliadb/ingest/py.typed +0 -0
- nucliadb/ingest/serialize.py +37 -167
- nucliadb/ingest/service/__init__.py +1 -3
- nucliadb/ingest/service/writer.py +185 -412
- nucliadb/ingest/settings.py +10 -20
- nucliadb/ingest/utils.py +3 -6
- nucliadb/learning_proxy.py +242 -55
- nucliadb/metrics_exporter.py +30 -19
- nucliadb/middleware/__init__.py +1 -3
- nucliadb/migrator/command.py +1 -3
- nucliadb/migrator/datamanager.py +13 -13
- nucliadb/migrator/migrator.py +47 -30
- nucliadb/migrator/utils.py +18 -10
- nucliadb/purge/__init__.py +139 -33
- nucliadb/purge/orphan_shards.py +7 -13
- nucliadb/reader/__init__.py +1 -3
- nucliadb/reader/api/models.py +1 -12
- nucliadb/reader/api/v1/__init__.py +0 -1
- nucliadb/reader/api/v1/download.py +21 -88
- nucliadb/reader/api/v1/export_import.py +1 -1
- nucliadb/reader/api/v1/knowledgebox.py +10 -10
- nucliadb/reader/api/v1/learning_config.py +2 -6
- nucliadb/reader/api/v1/resource.py +62 -88
- nucliadb/reader/api/v1/services.py +64 -83
- nucliadb/reader/app.py +12 -29
- nucliadb/reader/lifecycle.py +18 -4
- nucliadb/reader/py.typed +0 -0
- nucliadb/reader/reader/notifications.py +10 -28
- nucliadb/search/__init__.py +1 -3
- nucliadb/search/api/v1/__init__.py +1 -2
- nucliadb/search/api/v1/ask.py +17 -10
- nucliadb/search/api/v1/catalog.py +184 -0
- nucliadb/search/api/v1/feedback.py +16 -24
- nucliadb/search/api/v1/find.py +36 -36
- nucliadb/search/api/v1/knowledgebox.py +89 -60
- nucliadb/search/api/v1/resource/ask.py +2 -8
- nucliadb/search/api/v1/resource/search.py +49 -70
- nucliadb/search/api/v1/search.py +44 -210
- nucliadb/search/api/v1/suggest.py +39 -54
- nucliadb/search/app.py +12 -32
- nucliadb/search/lifecycle.py +10 -3
- nucliadb/search/predict.py +136 -187
- nucliadb/search/py.typed +0 -0
- nucliadb/search/requesters/utils.py +25 -58
- nucliadb/search/search/cache.py +149 -20
- nucliadb/search/search/chat/ask.py +571 -123
- nucliadb/search/{tests/unit/test_run.py → search/chat/exceptions.py} +14 -14
- nucliadb/search/search/chat/images.py +41 -17
- nucliadb/search/search/chat/prompt.py +817 -266
- nucliadb/search/search/chat/query.py +213 -309
- nucliadb/{tests/migrations/__init__.py → search/search/cut.py} +8 -8
- nucliadb/search/search/fetch.py +43 -36
- nucliadb/search/search/filters.py +9 -15
- nucliadb/search/search/find.py +214 -53
- nucliadb/search/search/find_merge.py +408 -391
- nucliadb/search/search/hydrator.py +191 -0
- nucliadb/search/search/merge.py +187 -223
- nucliadb/search/search/metrics.py +73 -2
- nucliadb/search/search/paragraphs.py +64 -106
- nucliadb/search/search/pgcatalog.py +233 -0
- nucliadb/search/search/predict_proxy.py +1 -1
- nucliadb/search/search/query.py +305 -150
- nucliadb/search/search/query_parser/exceptions.py +22 -0
- nucliadb/search/search/query_parser/models.py +101 -0
- nucliadb/search/search/query_parser/parser.py +183 -0
- nucliadb/search/search/rank_fusion.py +204 -0
- nucliadb/search/search/rerankers.py +270 -0
- nucliadb/search/search/shards.py +3 -32
- nucliadb/search/search/summarize.py +7 -18
- nucliadb/search/search/utils.py +27 -4
- nucliadb/search/settings.py +15 -1
- nucliadb/standalone/api_router.py +4 -10
- nucliadb/standalone/app.py +8 -14
- nucliadb/standalone/auth.py +7 -21
- nucliadb/standalone/config.py +7 -10
- nucliadb/standalone/lifecycle.py +26 -25
- nucliadb/standalone/migrations.py +1 -3
- nucliadb/standalone/purge.py +1 -1
- nucliadb/standalone/py.typed +0 -0
- nucliadb/standalone/run.py +3 -6
- nucliadb/standalone/settings.py +9 -16
- nucliadb/standalone/versions.py +15 -5
- nucliadb/tasks/consumer.py +8 -12
- nucliadb/tasks/producer.py +7 -6
- nucliadb/tests/config.py +53 -0
- nucliadb/train/__init__.py +1 -3
- nucliadb/train/api/utils.py +1 -2
- nucliadb/train/api/v1/shards.py +1 -1
- nucliadb/train/api/v1/trainset.py +2 -4
- nucliadb/train/app.py +10 -31
- nucliadb/train/generator.py +10 -19
- nucliadb/train/generators/field_classifier.py +7 -19
- nucliadb/train/generators/field_streaming.py +156 -0
- nucliadb/train/generators/image_classifier.py +12 -18
- nucliadb/train/generators/paragraph_classifier.py +5 -9
- nucliadb/train/generators/paragraph_streaming.py +6 -9
- nucliadb/train/generators/question_answer_streaming.py +19 -20
- nucliadb/train/generators/sentence_classifier.py +9 -15
- nucliadb/train/generators/token_classifier.py +48 -39
- nucliadb/train/generators/utils.py +14 -18
- nucliadb/train/lifecycle.py +7 -3
- nucliadb/train/nodes.py +23 -32
- nucliadb/train/py.typed +0 -0
- nucliadb/train/servicer.py +13 -21
- nucliadb/train/settings.py +2 -6
- nucliadb/train/types.py +13 -10
- nucliadb/train/upload.py +3 -6
- nucliadb/train/uploader.py +19 -23
- nucliadb/train/utils.py +1 -1
- nucliadb/writer/__init__.py +1 -3
- nucliadb/{ingest/fields/keywordset.py → writer/api/utils.py} +13 -10
- nucliadb/writer/api/v1/export_import.py +67 -14
- nucliadb/writer/api/v1/field.py +16 -269
- nucliadb/writer/api/v1/knowledgebox.py +218 -68
- nucliadb/writer/api/v1/resource.py +68 -88
- nucliadb/writer/api/v1/services.py +51 -70
- nucliadb/writer/api/v1/slug.py +61 -0
- nucliadb/writer/api/v1/transaction.py +67 -0
- nucliadb/writer/api/v1/upload.py +114 -113
- nucliadb/writer/app.py +6 -43
- nucliadb/writer/back_pressure.py +16 -38
- nucliadb/writer/exceptions.py +0 -4
- nucliadb/writer/lifecycle.py +21 -15
- nucliadb/writer/py.typed +0 -0
- nucliadb/writer/resource/audit.py +2 -1
- nucliadb/writer/resource/basic.py +48 -46
- nucliadb/writer/resource/field.py +25 -127
- nucliadb/writer/resource/origin.py +1 -2
- nucliadb/writer/settings.py +6 -2
- nucliadb/writer/tus/__init__.py +17 -15
- nucliadb/writer/tus/azure.py +111 -0
- nucliadb/writer/tus/dm.py +17 -5
- nucliadb/writer/tus/exceptions.py +1 -3
- nucliadb/writer/tus/gcs.py +49 -84
- nucliadb/writer/tus/local.py +21 -37
- nucliadb/writer/tus/s3.py +28 -68
- nucliadb/writer/tus/storage.py +5 -56
- nucliadb/writer/vectorsets.py +125 -0
- nucliadb-6.2.1.post2777.dist-info/METADATA +148 -0
- nucliadb-6.2.1.post2777.dist-info/RECORD +343 -0
- {nucliadb-4.0.0.post542.dist-info → nucliadb-6.2.1.post2777.dist-info}/WHEEL +1 -1
- nucliadb/common/maindb/redis.py +0 -194
- nucliadb/common/maindb/tikv.py +0 -433
- nucliadb/ingest/fields/layout.py +0 -58
- nucliadb/ingest/tests/conftest.py +0 -30
- nucliadb/ingest/tests/fixtures.py +0 -764
- nucliadb/ingest/tests/integration/consumer/__init__.py +0 -18
- nucliadb/ingest/tests/integration/consumer/test_auditing.py +0 -78
- nucliadb/ingest/tests/integration/consumer/test_materializer.py +0 -126
- nucliadb/ingest/tests/integration/consumer/test_pull.py +0 -144
- nucliadb/ingest/tests/integration/consumer/test_service.py +0 -81
- nucliadb/ingest/tests/integration/consumer/test_shard_creator.py +0 -68
- nucliadb/ingest/tests/integration/ingest/test_ingest.py +0 -684
- nucliadb/ingest/tests/integration/ingest/test_processing_engine.py +0 -95
- nucliadb/ingest/tests/integration/ingest/test_relations.py +0 -272
- nucliadb/ingest/tests/unit/consumer/__init__.py +0 -18
- nucliadb/ingest/tests/unit/consumer/test_auditing.py +0 -139
- nucliadb/ingest/tests/unit/consumer/test_consumer.py +0 -69
- nucliadb/ingest/tests/unit/consumer/test_pull.py +0 -60
- nucliadb/ingest/tests/unit/consumer/test_shard_creator.py +0 -140
- nucliadb/ingest/tests/unit/consumer/test_utils.py +0 -67
- nucliadb/ingest/tests/unit/orm/__init__.py +0 -19
- nucliadb/ingest/tests/unit/orm/test_brain.py +0 -247
- nucliadb/ingest/tests/unit/orm/test_brain_vectors.py +0 -74
- nucliadb/ingest/tests/unit/orm/test_processor.py +0 -131
- nucliadb/ingest/tests/unit/orm/test_resource.py +0 -331
- nucliadb/ingest/tests/unit/test_cache.py +0 -31
- nucliadb/ingest/tests/unit/test_partitions.py +0 -40
- nucliadb/ingest/tests/unit/test_processing.py +0 -171
- nucliadb/middleware/transaction.py +0 -117
- nucliadb/reader/api/v1/learning_collector.py +0 -63
- nucliadb/reader/tests/__init__.py +0 -19
- nucliadb/reader/tests/conftest.py +0 -31
- nucliadb/reader/tests/fixtures.py +0 -136
- nucliadb/reader/tests/test_list_resources.py +0 -75
- nucliadb/reader/tests/test_reader_file_download.py +0 -273
- nucliadb/reader/tests/test_reader_resource.py +0 -353
- nucliadb/reader/tests/test_reader_resource_field.py +0 -219
- nucliadb/search/api/v1/chat.py +0 -263
- nucliadb/search/api/v1/resource/chat.py +0 -174
- nucliadb/search/tests/__init__.py +0 -19
- nucliadb/search/tests/conftest.py +0 -33
- nucliadb/search/tests/fixtures.py +0 -199
- nucliadb/search/tests/node.py +0 -466
- nucliadb/search/tests/unit/__init__.py +0 -18
- nucliadb/search/tests/unit/api/__init__.py +0 -19
- nucliadb/search/tests/unit/api/v1/__init__.py +0 -19
- nucliadb/search/tests/unit/api/v1/resource/__init__.py +0 -19
- nucliadb/search/tests/unit/api/v1/resource/test_chat.py +0 -98
- nucliadb/search/tests/unit/api/v1/test_ask.py +0 -120
- nucliadb/search/tests/unit/api/v1/test_chat.py +0 -96
- nucliadb/search/tests/unit/api/v1/test_predict_proxy.py +0 -98
- nucliadb/search/tests/unit/api/v1/test_summarize.py +0 -99
- nucliadb/search/tests/unit/search/__init__.py +0 -18
- nucliadb/search/tests/unit/search/requesters/__init__.py +0 -18
- nucliadb/search/tests/unit/search/requesters/test_utils.py +0 -211
- nucliadb/search/tests/unit/search/search/__init__.py +0 -19
- nucliadb/search/tests/unit/search/search/test_shards.py +0 -45
- nucliadb/search/tests/unit/search/search/test_utils.py +0 -82
- nucliadb/search/tests/unit/search/test_chat_prompt.py +0 -270
- nucliadb/search/tests/unit/search/test_fetch.py +0 -108
- nucliadb/search/tests/unit/search/test_filters.py +0 -125
- nucliadb/search/tests/unit/search/test_paragraphs.py +0 -157
- nucliadb/search/tests/unit/search/test_predict_proxy.py +0 -106
- nucliadb/search/tests/unit/search/test_query.py +0 -153
- nucliadb/search/tests/unit/test_app.py +0 -79
- nucliadb/search/tests/unit/test_find_merge.py +0 -112
- nucliadb/search/tests/unit/test_merge.py +0 -34
- nucliadb/search/tests/unit/test_predict.py +0 -525
- nucliadb/standalone/tests/__init__.py +0 -19
- nucliadb/standalone/tests/conftest.py +0 -33
- nucliadb/standalone/tests/fixtures.py +0 -38
- nucliadb/standalone/tests/unit/__init__.py +0 -18
- nucliadb/standalone/tests/unit/test_api_router.py +0 -61
- nucliadb/standalone/tests/unit/test_auth.py +0 -169
- nucliadb/standalone/tests/unit/test_introspect.py +0 -35
- nucliadb/standalone/tests/unit/test_migrations.py +0 -63
- nucliadb/standalone/tests/unit/test_versions.py +0 -68
- nucliadb/tests/benchmarks/__init__.py +0 -19
- nucliadb/tests/benchmarks/test_search.py +0 -99
- nucliadb/tests/conftest.py +0 -32
- nucliadb/tests/fixtures.py +0 -735
- nucliadb/tests/knowledgeboxes/philosophy_books.py +0 -202
- nucliadb/tests/knowledgeboxes/ten_dummy_resources.py +0 -107
- nucliadb/tests/migrations/test_migration_0017.py +0 -76
- nucliadb/tests/migrations/test_migration_0018.py +0 -95
- nucliadb/tests/tikv.py +0 -240
- nucliadb/tests/unit/__init__.py +0 -19
- nucliadb/tests/unit/common/__init__.py +0 -19
- nucliadb/tests/unit/common/cluster/__init__.py +0 -19
- nucliadb/tests/unit/common/cluster/discovery/__init__.py +0 -19
- nucliadb/tests/unit/common/cluster/discovery/test_k8s.py +0 -172
- nucliadb/tests/unit/common/cluster/standalone/__init__.py +0 -18
- nucliadb/tests/unit/common/cluster/standalone/test_service.py +0 -114
- nucliadb/tests/unit/common/cluster/standalone/test_utils.py +0 -61
- nucliadb/tests/unit/common/cluster/test_cluster.py +0 -408
- nucliadb/tests/unit/common/cluster/test_kb_shard_manager.py +0 -173
- nucliadb/tests/unit/common/cluster/test_rebalance.py +0 -38
- nucliadb/tests/unit/common/cluster/test_rollover.py +0 -282
- nucliadb/tests/unit/common/maindb/__init__.py +0 -18
- nucliadb/tests/unit/common/maindb/test_driver.py +0 -127
- nucliadb/tests/unit/common/maindb/test_tikv.py +0 -53
- nucliadb/tests/unit/common/maindb/test_utils.py +0 -92
- nucliadb/tests/unit/common/test_context.py +0 -36
- nucliadb/tests/unit/export_import/__init__.py +0 -19
- nucliadb/tests/unit/export_import/test_datamanager.py +0 -37
- nucliadb/tests/unit/export_import/test_utils.py +0 -301
- nucliadb/tests/unit/migrator/__init__.py +0 -19
- nucliadb/tests/unit/migrator/test_migrator.py +0 -87
- nucliadb/tests/unit/tasks/__init__.py +0 -19
- nucliadb/tests/unit/tasks/conftest.py +0 -42
- nucliadb/tests/unit/tasks/test_consumer.py +0 -92
- nucliadb/tests/unit/tasks/test_producer.py +0 -95
- nucliadb/tests/unit/tasks/test_tasks.py +0 -58
- nucliadb/tests/unit/test_field_ids.py +0 -49
- nucliadb/tests/unit/test_health.py +0 -86
- nucliadb/tests/unit/test_kb_slugs.py +0 -54
- nucliadb/tests/unit/test_learning_proxy.py +0 -252
- nucliadb/tests/unit/test_metrics_exporter.py +0 -77
- nucliadb/tests/unit/test_purge.py +0 -136
- nucliadb/tests/utils/__init__.py +0 -74
- nucliadb/tests/utils/aiohttp_session.py +0 -44
- nucliadb/tests/utils/broker_messages/__init__.py +0 -171
- nucliadb/tests/utils/broker_messages/fields.py +0 -197
- nucliadb/tests/utils/broker_messages/helpers.py +0 -33
- nucliadb/tests/utils/entities.py +0 -78
- nucliadb/train/api/v1/check.py +0 -60
- nucliadb/train/tests/__init__.py +0 -19
- nucliadb/train/tests/conftest.py +0 -29
- nucliadb/train/tests/fixtures.py +0 -342
- nucliadb/train/tests/test_field_classification.py +0 -122
- nucliadb/train/tests/test_get_entities.py +0 -80
- nucliadb/train/tests/test_get_info.py +0 -51
- nucliadb/train/tests/test_get_ontology.py +0 -34
- nucliadb/train/tests/test_get_ontology_count.py +0 -63
- nucliadb/train/tests/test_image_classification.py +0 -221
- nucliadb/train/tests/test_list_fields.py +0 -39
- nucliadb/train/tests/test_list_paragraphs.py +0 -73
- nucliadb/train/tests/test_list_resources.py +0 -39
- nucliadb/train/tests/test_list_sentences.py +0 -71
- nucliadb/train/tests/test_paragraph_classification.py +0 -123
- nucliadb/train/tests/test_paragraph_streaming.py +0 -118
- nucliadb/train/tests/test_question_answer_streaming.py +0 -239
- nucliadb/train/tests/test_sentence_classification.py +0 -143
- nucliadb/train/tests/test_token_classification.py +0 -136
- nucliadb/train/tests/utils.py +0 -101
- nucliadb/writer/layouts/__init__.py +0 -51
- nucliadb/writer/layouts/v1.py +0 -59
- nucliadb/writer/tests/__init__.py +0 -19
- nucliadb/writer/tests/conftest.py +0 -31
- nucliadb/writer/tests/fixtures.py +0 -191
- nucliadb/writer/tests/test_fields.py +0 -475
- nucliadb/writer/tests/test_files.py +0 -740
- nucliadb/writer/tests/test_knowledgebox.py +0 -49
- nucliadb/writer/tests/test_reprocess_file_field.py +0 -133
- nucliadb/writer/tests/test_resources.py +0 -476
- nucliadb/writer/tests/test_service.py +0 -137
- nucliadb/writer/tests/test_tus.py +0 -203
- nucliadb/writer/tests/utils.py +0 -35
- nucliadb/writer/tus/pg.py +0 -125
- nucliadb-4.0.0.post542.dist-info/METADATA +0 -135
- nucliadb-4.0.0.post542.dist-info/RECORD +0 -462
- {nucliadb/ingest/tests → migrations/pg}/__init__.py +0 -0
- /nucliadb/{ingest/tests/integration → common/external_index_providers}/__init__.py +0 -0
- /nucliadb/{ingest/tests/integration/ingest → common/models_utils}/__init__.py +0 -0
- /nucliadb/{ingest/tests/unit → search/search/query_parser}/__init__.py +0 -0
- /nucliadb/{ingest/tests → tests}/vectors.py +0 -0
- {nucliadb-4.0.0.post542.dist-info → nucliadb-6.2.1.post2777.dist-info}/entry_points.txt +0 -0
- {nucliadb-4.0.0.post542.dist-info → nucliadb-6.2.1.post2777.dist-info}/top_level.txt +0 -0
- {nucliadb-4.0.0.post542.dist-info → nucliadb-6.2.1.post2777.dist-info}/zip-safe +0 -0
nucliadb/tests/tikv.py
DELETED
@@ -1,240 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
20
|
-
import os
|
21
|
-
import platform
|
22
|
-
import signal
|
23
|
-
import subprocess
|
24
|
-
import tarfile
|
25
|
-
import tempfile
|
26
|
-
import time
|
27
|
-
from io import BytesIO
|
28
|
-
|
29
|
-
import pytest
|
30
|
-
import requests
|
31
|
-
from tikv_client import TransactionClient # type: ignore
|
32
|
-
|
33
|
-
|
34
|
-
class TiKVd(object):
|
35
|
-
def __init__(
|
36
|
-
self,
|
37
|
-
port=20160,
|
38
|
-
pd_port=2379,
|
39
|
-
peer_port=2380,
|
40
|
-
tikv_bin_name="tikv-server",
|
41
|
-
pd_bin_name="pd-server",
|
42
|
-
host="127.0.0.1",
|
43
|
-
path="",
|
44
|
-
debug=False,
|
45
|
-
):
|
46
|
-
self.port = port
|
47
|
-
self.pd_port = pd_port
|
48
|
-
self.peer_port = peer_port
|
49
|
-
self.tikv_bin_name = tikv_bin_name
|
50
|
-
self.pd_bin_name = pd_bin_name
|
51
|
-
self.path = path
|
52
|
-
self.host = host
|
53
|
-
self.tmpfolder = None
|
54
|
-
self.debug = debug
|
55
|
-
self.proc = None
|
56
|
-
self.proc2 = None
|
57
|
-
|
58
|
-
def start(self):
|
59
|
-
self.tmpfolder = tempfile.TemporaryDirectory()
|
60
|
-
|
61
|
-
cmd = [
|
62
|
-
f"{self.path}/{self.tikv_bin_name}",
|
63
|
-
f"--pd-endpoints={self.host}:{self.pd_port}",
|
64
|
-
f"--addr={self.host}:{self.port}",
|
65
|
-
f"--data-dir={self.tmpfolder.name}/tikv1",
|
66
|
-
f"--log-file={self.tmpfolder.name}/tikv1.log",
|
67
|
-
]
|
68
|
-
|
69
|
-
if self.debug:
|
70
|
-
self.proc2 = subprocess.Popen(cmd)
|
71
|
-
else:
|
72
|
-
self.proc2 = subprocess.Popen(
|
73
|
-
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
74
|
-
)
|
75
|
-
|
76
|
-
if self.debug:
|
77
|
-
if self.proc2 is None:
|
78
|
-
print(
|
79
|
-
"[\031[0;33mDEBUG\033[0;0m] Failed to start server listening on port %d started."
|
80
|
-
% self.port
|
81
|
-
)
|
82
|
-
else:
|
83
|
-
print(
|
84
|
-
"[\033[0;33mDEBUG\033[0;0m] Server listening on port %d started."
|
85
|
-
% self.port
|
86
|
-
)
|
87
|
-
cmd = [
|
88
|
-
f"{self.path}/{self.pd_bin_name}",
|
89
|
-
"--name=pd",
|
90
|
-
f"--data-dir={self.tmpfolder.name}",
|
91
|
-
f"--client-urls=http://{self.host}:{self.pd_port}",
|
92
|
-
f"--peer-urls=http://{self.host}:{self.peer_port}",
|
93
|
-
f"--initial-cluster=pd=http://{self.host}:{self.peer_port}",
|
94
|
-
]
|
95
|
-
|
96
|
-
if self.debug:
|
97
|
-
self.proc = subprocess.Popen(cmd)
|
98
|
-
else:
|
99
|
-
self.proc = subprocess.Popen(
|
100
|
-
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
101
|
-
)
|
102
|
-
|
103
|
-
if self.debug:
|
104
|
-
if self.proc is None:
|
105
|
-
print(
|
106
|
-
"[\031[0;33mDEBUG\033[0;0m] Failed to start server listening on port %d started."
|
107
|
-
% self.pd_port
|
108
|
-
)
|
109
|
-
else:
|
110
|
-
print(
|
111
|
-
"[\033[0;33mDEBUG\033[0;0m] Server listening on port %d started."
|
112
|
-
% self.pd_port
|
113
|
-
)
|
114
|
-
return self.proc
|
115
|
-
|
116
|
-
def stop(self):
|
117
|
-
if self.debug:
|
118
|
-
print(
|
119
|
-
"[\033[0;33mDEBUG\033[0;0m] Server listening on %d will stop."
|
120
|
-
% self.port
|
121
|
-
)
|
122
|
-
|
123
|
-
if self.debug:
|
124
|
-
if self.proc is None:
|
125
|
-
print(
|
126
|
-
"[\033[0;31mDEBUG\033[0;0m] Failed terminating server listening on port %d"
|
127
|
-
% self.port
|
128
|
-
)
|
129
|
-
if self.proc2 is None:
|
130
|
-
print(
|
131
|
-
"[\033[0;31mDEBUG\033[0;0m] Failed terminating server listening on port %d"
|
132
|
-
% self.port
|
133
|
-
)
|
134
|
-
|
135
|
-
if self.proc.returncode is not None:
|
136
|
-
if self.debug:
|
137
|
-
print(
|
138
|
-
"[\033[0;31mDEBUG\033[0;0m] Server listening on port {port} finished running already with exit {ret}".format( # noqa
|
139
|
-
port=self.pd_port, ret=self.proc.returncode
|
140
|
-
)
|
141
|
-
)
|
142
|
-
elif self.proc2.returncode is not None:
|
143
|
-
if self.debug:
|
144
|
-
print(
|
145
|
-
"[\033[0;31mDEBUG\033[0;0m] Server listening on port {port} finished running already with exit {ret}".format( # noqa
|
146
|
-
port=self.port, ret=self.proc2.returncode
|
147
|
-
)
|
148
|
-
)
|
149
|
-
else:
|
150
|
-
os.kill(self.proc.pid, signal.SIGKILL)
|
151
|
-
self.proc.wait()
|
152
|
-
os.kill(self.proc2.pid, signal.SIGKILL)
|
153
|
-
self.proc2.wait()
|
154
|
-
if self.debug:
|
155
|
-
print(
|
156
|
-
"[\033[0;33mDEBUG\033[0;0m] Server listening on %d was stopped."
|
157
|
-
% self.port
|
158
|
-
)
|
159
|
-
if self.tmpfolder is not None:
|
160
|
-
self.tmpfolder.cleanup()
|
161
|
-
self.tmpfolder = None
|
162
|
-
|
163
|
-
|
164
|
-
def start_tikvd(tikvd: TiKVd):
|
165
|
-
tikvd.start()
|
166
|
-
|
167
|
-
endpoint = "127.0.0.1:{port}".format(port=tikvd.port)
|
168
|
-
retries = 0
|
169
|
-
while True:
|
170
|
-
if retries > 100:
|
171
|
-
break
|
172
|
-
|
173
|
-
try:
|
174
|
-
connection = TransactionClient.connect(endpoint)
|
175
|
-
txn = connection.begin(pessimistic=True)
|
176
|
-
txn.rollback()
|
177
|
-
except: # noqa
|
178
|
-
retries += 1
|
179
|
-
time.sleep(0.1)
|
180
|
-
|
181
|
-
|
182
|
-
@pytest.fixture(scope="session")
|
183
|
-
def tikvd():
|
184
|
-
if os.environ.get("TESTING_TIKV_LOCAL", None):
|
185
|
-
yield "localhost", "XX", "2379"
|
186
|
-
return
|
187
|
-
|
188
|
-
if not os.path.isfile("tikv-server"):
|
189
|
-
version = "v5.3.1"
|
190
|
-
arch = platform.machine()
|
191
|
-
if arch == "x86_64":
|
192
|
-
arch = "amd64"
|
193
|
-
system = platform.system().lower()
|
194
|
-
|
195
|
-
resp = requests.get(
|
196
|
-
f"https://tiup-mirrors.pingcap.com/tikv-{version}-{system}-{arch}.tar.gz"
|
197
|
-
)
|
198
|
-
|
199
|
-
zipfile = tarfile.open(fileobj=BytesIO(resp.content), mode="r:gz")
|
200
|
-
|
201
|
-
zipfile.extract(f"tikv-server")
|
202
|
-
os.chmod("tikv-server", 755)
|
203
|
-
|
204
|
-
if not os.path.isfile("pd-server"):
|
205
|
-
version = "v5.3.1"
|
206
|
-
arch = platform.machine()
|
207
|
-
if arch == "x86_64":
|
208
|
-
arch = "amd64"
|
209
|
-
system = platform.system().lower()
|
210
|
-
|
211
|
-
resp = requests.get(
|
212
|
-
f"https://tiup-mirrors.pingcap.com/pd-{version}-{system}-{arch}.tar.gz"
|
213
|
-
)
|
214
|
-
|
215
|
-
zipfile = tarfile.open(fileobj=BytesIO(resp.content), mode="r:gz")
|
216
|
-
|
217
|
-
zipfile.extract(f"pd-server")
|
218
|
-
os.chmod("pd-server", 755)
|
219
|
-
|
220
|
-
server = TiKVd(debug=True)
|
221
|
-
server.tikv_bin_name = "tikv-server"
|
222
|
-
server.pd_bin_name = "pd-server"
|
223
|
-
server.path = os.getcwd()
|
224
|
-
|
225
|
-
start_tikvd(server)
|
226
|
-
print("Started TiKVd")
|
227
|
-
|
228
|
-
for i in range(100):
|
229
|
-
resp = requests.get(f"http://{server.host}:{server.pd_port}/pd/api/v1/stores")
|
230
|
-
if (
|
231
|
-
resp.status_code == 200
|
232
|
-
and resp.json()["stores"][0]["store"]["state_name"] == "Up"
|
233
|
-
):
|
234
|
-
break
|
235
|
-
print(resp.status_code)
|
236
|
-
print(resp.json())
|
237
|
-
time.sleep(1)
|
238
|
-
|
239
|
-
yield server.host, server.port, server.pd_port
|
240
|
-
server.stop()
|
nucliadb/tests/unit/__init__.py
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
@@ -1,172 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
20
|
-
import asyncio
|
21
|
-
import time
|
22
|
-
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
23
|
-
|
24
|
-
import kubernetes_asyncio.client.models.v1_container_status # type: ignore
|
25
|
-
import kubernetes_asyncio.client.models.v1_object_meta # type: ignore
|
26
|
-
import kubernetes_asyncio.client.models.v1_pod # type: ignore
|
27
|
-
import kubernetes_asyncio.client.models.v1_pod_status # type: ignore
|
28
|
-
import pytest
|
29
|
-
|
30
|
-
from nucliadb.common.cluster import manager
|
31
|
-
from nucliadb.common.cluster.discovery.k8s import EventType, KubernetesDiscovery
|
32
|
-
from nucliadb.common.cluster.discovery.types import IndexNodeMetadata
|
33
|
-
from nucliadb.common.cluster.settings import Settings
|
34
|
-
from nucliadb_protos import nodewriter_pb2
|
35
|
-
|
36
|
-
pytestmark = pytest.mark.asyncio
|
37
|
-
|
38
|
-
|
39
|
-
@pytest.fixture()
|
40
|
-
def writer_stub():
|
41
|
-
writer_stub = MagicMock()
|
42
|
-
writer_stub.GetMetadata = AsyncMock(
|
43
|
-
return_value=nodewriter_pb2.NodeMetadata(
|
44
|
-
node_id="node_id", shard_count=1, available_disk=10, total_disk=10
|
45
|
-
)
|
46
|
-
)
|
47
|
-
with (
|
48
|
-
patch(
|
49
|
-
"nucliadb.common.cluster.discovery.base.nodewriter_pb2_grpc.NodeWriterStub",
|
50
|
-
return_value=writer_stub,
|
51
|
-
),
|
52
|
-
patch(
|
53
|
-
"nucliadb.common.cluster.discovery.base.replication_pb2_grpc.ReplicationServiceStub",
|
54
|
-
return_value=writer_stub,
|
55
|
-
),
|
56
|
-
patch("nucliadb.common.cluster.discovery.base.get_traced_grpc_channel"),
|
57
|
-
):
|
58
|
-
yield writer_stub
|
59
|
-
|
60
|
-
|
61
|
-
@pytest.fixture()
|
62
|
-
def k8s_discovery(writer_stub):
|
63
|
-
disc = KubernetesDiscovery(Settings())
|
64
|
-
disc.node_heartbeat_interval = 0.1
|
65
|
-
manager.INDEX_NODES.clear()
|
66
|
-
yield disc
|
67
|
-
manager.INDEX_NODES.clear()
|
68
|
-
|
69
|
-
|
70
|
-
def create_k8s_event(
|
71
|
-
type_: str = "ADDED",
|
72
|
-
name: str = "node-0",
|
73
|
-
container_name="reader",
|
74
|
-
ready: bool = True,
|
75
|
-
) -> EventType:
|
76
|
-
return {
|
77
|
-
"type": type_,
|
78
|
-
"object": kubernetes_asyncio.client.models.v1_pod.V1Pod(
|
79
|
-
status=kubernetes_asyncio.client.models.v1_pod_status.V1PodStatus(
|
80
|
-
container_statuses=[
|
81
|
-
kubernetes_asyncio.client.models.v1_container_status.V1ContainerStatus(
|
82
|
-
name=container_name,
|
83
|
-
ready=ready,
|
84
|
-
image="image",
|
85
|
-
image_id="image_id",
|
86
|
-
restart_count=0,
|
87
|
-
)
|
88
|
-
],
|
89
|
-
pod_ip="1.2.3.4",
|
90
|
-
),
|
91
|
-
metadata=kubernetes_asyncio.client.models.v1_object_meta.V1ObjectMeta(
|
92
|
-
name=name, labels={"readReplica": "false"}
|
93
|
-
),
|
94
|
-
),
|
95
|
-
}
|
96
|
-
|
97
|
-
|
98
|
-
async def test_get_node_metadata(k8s_discovery: KubernetesDiscovery, writer_stub):
|
99
|
-
assert await k8s_discovery.get_node_metadata(
|
100
|
-
"node-0", "1.1.1.1", read_replica=False
|
101
|
-
) == IndexNodeMetadata(
|
102
|
-
node_id="node_id",
|
103
|
-
shard_count=1,
|
104
|
-
name="node_id",
|
105
|
-
address="1.1.1.1",
|
106
|
-
updated_at=ANY,
|
107
|
-
available_disk=10,
|
108
|
-
)
|
109
|
-
|
110
|
-
writer_stub.GetMetadata.assert_awaited_once()
|
111
|
-
|
112
|
-
# should be cached now
|
113
|
-
await k8s_discovery.get_node_metadata("node-0", "1.1.1.1", read_replica=False)
|
114
|
-
|
115
|
-
assert len(writer_stub.GetMetadata.mock_calls) == 1
|
116
|
-
|
117
|
-
|
118
|
-
async def test_update_node(k8s_discovery: KubernetesDiscovery):
|
119
|
-
await k8s_discovery.update_node(create_k8s_event())
|
120
|
-
assert len(manager.INDEX_NODES) == 1
|
121
|
-
|
122
|
-
# update again
|
123
|
-
await k8s_discovery.update_node(create_k8s_event())
|
124
|
-
assert len(manager.INDEX_NODES) == 1
|
125
|
-
|
126
|
-
|
127
|
-
async def test_remove_node(k8s_discovery: KubernetesDiscovery):
|
128
|
-
await k8s_discovery.update_node(create_k8s_event())
|
129
|
-
|
130
|
-
assert len(manager.INDEX_NODES) == 1
|
131
|
-
|
132
|
-
await k8s_discovery.update_node(create_k8s_event(ready=False))
|
133
|
-
|
134
|
-
assert len(manager.INDEX_NODES) == 0
|
135
|
-
|
136
|
-
|
137
|
-
async def test_update_node_data_cache(k8s_discovery: KubernetesDiscovery, writer_stub):
|
138
|
-
await k8s_discovery.update_node(create_k8s_event())
|
139
|
-
|
140
|
-
task = asyncio.create_task(k8s_discovery.update_node_data_cache())
|
141
|
-
|
142
|
-
await asyncio.sleep(k8s_discovery.node_heartbeat_interval + 0.1)
|
143
|
-
|
144
|
-
task.cancel()
|
145
|
-
|
146
|
-
assert len(writer_stub.GetMetadata.mock_calls) == 2
|
147
|
-
|
148
|
-
|
149
|
-
async def test_remove_stale_nodes(k8s_discovery: KubernetesDiscovery):
|
150
|
-
manager.add_index_node(
|
151
|
-
id="node_id",
|
152
|
-
address="1.1.1.1",
|
153
|
-
shard_count=1,
|
154
|
-
available_disk=100,
|
155
|
-
)
|
156
|
-
nmd = IndexNodeMetadata(
|
157
|
-
node_id="node_id",
|
158
|
-
shard_count=1,
|
159
|
-
available_disk=100,
|
160
|
-
name="node_id",
|
161
|
-
address="1.1.1.1",
|
162
|
-
updated_at=time.time() - k8s_discovery.node_heartbeat_interval * 3,
|
163
|
-
)
|
164
|
-
k8s_discovery.node_id_cache["node_id"] = nmd
|
165
|
-
|
166
|
-
assert len(k8s_discovery.node_id_cache) == 1
|
167
|
-
assert len(manager.get_index_nodes()) == 1
|
168
|
-
|
169
|
-
k8s_discovery._maybe_remove_stale_node("node_id")
|
170
|
-
|
171
|
-
assert len(k8s_discovery.node_id_cache) == 0
|
172
|
-
assert len(manager.get_index_nodes()) == 0
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
@@ -1,114 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
20
|
-
import os
|
21
|
-
import tempfile
|
22
|
-
from unittest.mock import AsyncMock, MagicMock, patch
|
23
|
-
|
24
|
-
import pytest
|
25
|
-
from nucliadb_protos.noderesources_pb2 import ShardId
|
26
|
-
from nucliadb_protos.utils_pb2 import RelationNode
|
27
|
-
|
28
|
-
from nucliadb.common.cluster.settings import Settings
|
29
|
-
from nucliadb.common.cluster.standalone import service
|
30
|
-
from nucliadb_protos import nodereader_pb2, standalone_pb2
|
31
|
-
|
32
|
-
pytestmark = pytest.mark.asyncio
|
33
|
-
|
34
|
-
|
35
|
-
@pytest.fixture
|
36
|
-
def cluster_settings():
|
37
|
-
settings = Settings()
|
38
|
-
with (
|
39
|
-
patch("nucliadb.common.cluster.standalone.service.cluster_settings", settings),
|
40
|
-
tempfile.TemporaryDirectory() as tmpdir,
|
41
|
-
):
|
42
|
-
settings.data_path = tmpdir
|
43
|
-
os.makedirs(os.path.join(tmpdir, "shards"))
|
44
|
-
yield settings
|
45
|
-
|
46
|
-
|
47
|
-
@pytest.fixture
|
48
|
-
def self_node(cluster_settings):
|
49
|
-
self_node = MagicMock(id="id", address="address", shard_count=0, available_disk=10)
|
50
|
-
self_node.reader = AsyncMock()
|
51
|
-
self_node.writer = AsyncMock()
|
52
|
-
self_node.reader.Search.return_value = nodereader_pb2.SearchResponse()
|
53
|
-
|
54
|
-
relation_edges = nodereader_pb2.EdgeList()
|
55
|
-
edge = nodereader_pb2.RelationEdge(
|
56
|
-
edge_type=RelationNode.NodeType.ENTITY, property="foo"
|
57
|
-
)
|
58
|
-
relation_edges.list.append(edge)
|
59
|
-
self_node.reader.RelationEdges.return_value = relation_edges
|
60
|
-
|
61
|
-
with patch("nucliadb.common.cluster.standalone.service.get_self") as mock_get_self:
|
62
|
-
mock_get_self.return_value = self_node
|
63
|
-
yield self_node
|
64
|
-
|
65
|
-
|
66
|
-
@pytest.fixture
|
67
|
-
def servicer(self_node):
|
68
|
-
yield service.StandaloneClusterServiceServicer()
|
69
|
-
|
70
|
-
|
71
|
-
async def test_node_action(
|
72
|
-
servicer: service.StandaloneClusterServiceServicer,
|
73
|
-
self_node,
|
74
|
-
cluster_settings,
|
75
|
-
):
|
76
|
-
resp = await servicer.NodeAction(
|
77
|
-
standalone_pb2.NodeActionRequest(
|
78
|
-
service="reader",
|
79
|
-
action="Search",
|
80
|
-
payload=nodereader_pb2.SearchRequest(body="test").SerializeToString(),
|
81
|
-
),
|
82
|
-
None,
|
83
|
-
)
|
84
|
-
assert resp == standalone_pb2.NodeActionResponse(
|
85
|
-
payload=self_node.reader.Search.return_value.SerializeToString()
|
86
|
-
)
|
87
|
-
|
88
|
-
|
89
|
-
async def test_reader_node_action_relations(
|
90
|
-
servicer: service.StandaloneClusterServiceServicer,
|
91
|
-
self_node,
|
92
|
-
cluster_settings,
|
93
|
-
):
|
94
|
-
resp = await servicer.NodeAction(
|
95
|
-
standalone_pb2.NodeActionRequest(
|
96
|
-
service="reader",
|
97
|
-
action="RelationEdges",
|
98
|
-
payload=ShardId(id="test").SerializeToString(),
|
99
|
-
),
|
100
|
-
None,
|
101
|
-
)
|
102
|
-
assert resp == standalone_pb2.NodeActionResponse(
|
103
|
-
payload=self_node.reader.RelationEdges.return_value.SerializeToString()
|
104
|
-
)
|
105
|
-
|
106
|
-
|
107
|
-
async def test_node_info(
|
108
|
-
servicer: service.StandaloneClusterServiceServicer, self_node, cluster_settings
|
109
|
-
):
|
110
|
-
resp = await servicer.NodeInfo(standalone_pb2.NodeInfoRequest(), None)
|
111
|
-
assert resp.id == self_node.id
|
112
|
-
assert resp.address == self_node.address
|
113
|
-
assert resp.shard_count == self_node.shard_count
|
114
|
-
assert resp.available_disk > 0
|
@@ -1,61 +0,0 @@
|
|
1
|
-
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
-
#
|
3
|
-
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
-
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
-
#
|
6
|
-
# AGPL:
|
7
|
-
# This program is free software: you can redistribute it and/or modify
|
8
|
-
# it under the terms of the GNU Affero General Public License as
|
9
|
-
# published by the Free Software Foundation, either version 3 of the
|
10
|
-
# License, or (at your option) any later version.
|
11
|
-
#
|
12
|
-
# This program is distributed in the hope that it will be useful,
|
13
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
-
# GNU Affero General Public License for more details.
|
16
|
-
#
|
17
|
-
# You should have received a copy of the GNU Affero General Public License
|
18
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
-
#
|
20
|
-
import os
|
21
|
-
import tempfile
|
22
|
-
from unittest.mock import patch
|
23
|
-
|
24
|
-
import pytest
|
25
|
-
|
26
|
-
from nucliadb.common.cluster.settings import Settings
|
27
|
-
from nucliadb.common.cluster.standalone import utils
|
28
|
-
|
29
|
-
|
30
|
-
@pytest.fixture
|
31
|
-
def cluster_settings():
|
32
|
-
settings = Settings()
|
33
|
-
with (
|
34
|
-
patch("nucliadb.common.cluster.standalone.utils.cluster_settings", settings),
|
35
|
-
tempfile.TemporaryDirectory() as tmpdir,
|
36
|
-
):
|
37
|
-
settings.data_path = tmpdir
|
38
|
-
yield settings
|
39
|
-
|
40
|
-
|
41
|
-
def test_get_standalone_node_id(cluster_settings: Settings):
|
42
|
-
assert utils.get_standalone_node_id()
|
43
|
-
assert os.path.exists(os.path.join(cluster_settings.data_path, "node.key"))
|
44
|
-
|
45
|
-
|
46
|
-
def test_get_self(cluster_settings: Settings):
|
47
|
-
os.makedirs(os.path.join(cluster_settings.data_path, "shards", "1"))
|
48
|
-
|
49
|
-
assert utils.get_self().shard_count == 1
|
50
|
-
|
51
|
-
|
52
|
-
def test_get_self_k8s_host(cluster_settings: Settings, monkeypatch):
|
53
|
-
monkeypatch.setenv("NUCLIADB_SERVICE_HOST", "host")
|
54
|
-
monkeypatch.setenv("HOSTNAME", "nucliadb-0")
|
55
|
-
|
56
|
-
with (
|
57
|
-
patch("nucliadb.common.cluster.standalone.grpc_node_binding.NodeWriter"),
|
58
|
-
patch("nucliadb.common.cluster.standalone.grpc_node_binding.NodeReader"),
|
59
|
-
):
|
60
|
-
# patch because loading settings validates address now
|
61
|
-
assert utils.get_self().address == "nucliadb-0.nucliadb"
|