by-qa 0.4.2__tar.gz → 0.4.3.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/.env.example +17 -0
- by_qa-0.4.3.dev1/CONTEXT.md +54 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/PKG-INFO +4 -4
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/pyproject.toml +4 -4
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/config.py +59 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/api/knowledge_entity_schemas.py +72 -2
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/api/routes.py +24 -22
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/api/schemas.py +2 -1
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/build_status.py +39 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/events.py +127 -2
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/infrastructure/runtime.py +173 -15
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/repositories/knowledge_build_acceptance_repository.py +376 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/repositories/knowledge_build_batch_repository.py +218 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/repositories/knowledge_build_task_repository.py +1122 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_fs_entry_repository.py +47 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_item_chunk_repository.py +27 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_semantic_processing_task_repository.py +28 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/repositories/processing_task_query_repository.py +123 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/document_update_service.py +36 -27
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/file_build_background_runner.py +308 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/file_build_execution_service.py +509 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/file_build_models.py +172 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/file_build_mutation_service.py +54 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/file_build_processing_service.py +417 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/file_build_terminal_event_service.py +71 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_base_service.py +166 -19
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_task_worker.py +6 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_item_ingestion_service.py +160 -1
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/services/semantic_task_mutation_service.py +73 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/sql/038_knowledge_build_batch.sql +49 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/sql/039_knowledge_build_task_background_extension.sql +22 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/sql/040_knowledge_build_task_legacy_backfill.sql.tpl +80 -0
- by_qa-0.4.3.dev1/src/by_qa/knowledge_base/sql/041_knowledge_build_task_active_constraints.sql +124 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/main.py +17 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/uv.lock +7 -7
- by_qa-0.4.2/src/by_qa/knowledge_base/repositories/knowledge_build_task_repository.py +0 -153
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/.github/workflows/ci.yml +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/.github/workflows/release.yml +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/.gitignore +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/.pre-commit-config.yaml +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/AGENTS.md +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/CLAUDE.md +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/Dockerfile +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/LICENSE +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/Makefile +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/README.md +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/README_zh.md +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/docker/minio/init/init-minio.sh +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/docker/opengauss/custom/Dockerfile +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/docker/opengauss/init/init-opengauss.sh +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/docker-compose.kb-stack.yml +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/eval/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/eval/cli.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/eval/judge.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/eval/models.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/eval/runner.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/pylintrc +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/pytest.ini +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/core/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/core/exceptions.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/core/framework_client.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/core/logger.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/core/model_config.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/api/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/api/metadata_schemas.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/dsl/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/dsl/compiler.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/dsl/errors.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/dsl/validator.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/infrastructure/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/infrastructure/database.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/infrastructure/object_storage.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/infrastructure/storage.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/infrastructure/storage_s3.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/metadata_types.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/file_metadata_value_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_base_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_entity_asset_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_entity_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_fetch_cache_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_file_reference_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_file_update_timeline_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_item_search_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/knowledge_semantic_processing_batch_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/metadata_search_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/repositories/retrieval_projection_repository.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/bootstrap_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/cache_file_lock.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/embedding_query_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/entry_metadata.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/errors.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/file_metadata_query_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/file_metadata_update_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_document_metadata.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_background_runner.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_discovery.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_enrichment.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_intelligence.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_processing_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_synonym_resolution.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_fetch_cache_cleanup_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/knowledge_item_search_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/markdown_front_matter.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/markdown_reference_resolver.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/markdown_reference_rewriter.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/markdown_update_summary_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/metadata_search_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/services/zip_batch_import_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/000_extensions.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/001_knowledge_base.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/002_knowledge_fs_entry.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/003_knowledge_chunk.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/004_retrieval_projection.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/005_knowledge_fetch_cache_index.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/006_knowledge_build_task.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/009_knowledge_fs_entry_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/010_knowledge_chunk_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/011_retrieval_projection_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/012_knowledge_fetch_cache_index_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/013_knowledge_build_task_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/014_embedding_table.sql.tpl +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/015_metadata_property_def.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/016_metadata_property_def_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/017_file_metadata_value.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/018_file_metadata_value_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/019_knowledge_fs_entry_virtual_path.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/020_knowledge_fs_entry_virtual_path_indexes.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/021_knowledge_fs_entry_virtual_path_backfill.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/022_knowledge_fs_entry_top_level_sibling_unique.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/023_metadata_property_def_name_unique.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/024_knowledge_base_name_unique.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/025_metadata_value_self_contained_migration.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/026_knowledge_file_reference.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/027_knowledge_file_update_timeline.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/028_knowledge_fs_entry_checksum_index.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/029_knowledge_semantic_processing_task.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/030_knowledge_file_reference_semantic_extension.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/031_knowledge_file_document_kind_backfill.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/032_knowledge_schema_migration.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/033_knowledge_entity.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/034_knowledge_entity_embedding_table.sql.tpl +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/035_knowledge_entity_discovery_schema.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/036_knowledge_entity_embedding_full_only.sql.tpl +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_base/sql/037_knowledge_semantic_processing_directory_scope.sql +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_build/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_build/runtime.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_build/services/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_build/services/document_chunking_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_build/services/heading_patterns.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_common/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_common/exceptions.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_common/kb_path_utils.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_common/markdown_reference.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_common/schemas.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/knowledge_common/text_segmentation.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/answer_synthesizer.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/multi_hop_react.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/multi_hop_summarizer.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/query_decomposer.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/single_hop_react.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/standalone_question_rewriter.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/agents/subanswer_aggregator.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/base_engine.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/config.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/context.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/context_manager.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/event_filter.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/exceptions.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/fallback_messages.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/messages.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/middleware/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/middleware/tool_call_guard.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/models.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/operation_registry.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/prompt_fragments.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/reducers.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/common/state.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/engine.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/graph.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/nodes/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/nodes/retrieve.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/state.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/fast/types.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/engine.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/graph.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/nodes/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/nodes/final_answer.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/nodes/router.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/nodes/state_cleanup.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/state.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/engines/instant/types.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/services/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/services/checkpointer_factory.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/services/llm_service.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/services/opengauss_checkpointer.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/services/session_history.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/tools/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/tools/dsl_guide.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/tools/knowledge_tools.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/tools/operations/__init__.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/tools/operations/base.py +0 -0
- {by_qa-0.4.2 → by_qa-0.4.3.dev1}/src/by_qa/qa/tools/operations/knowledge_search.py +0 -0
|
@@ -69,6 +69,23 @@ KNOWLEDGE_ENTITY_REAPER_SECONDS=30
|
|
|
69
69
|
KNOWLEDGE_ENTITY_WORKER_STATUS_LOG_SECONDS=60
|
|
70
70
|
# Maximum wait for active tasks during graceful service shutdown.
|
|
71
71
|
KNOWLEDGE_ENTITY_SHUTDOWN_GRACE_SECONDS=60
|
|
72
|
+
|
|
73
|
+
# Independent durable File Build background processing. This pool does not
|
|
74
|
+
# share its process-local concurrency limit with KnowledgeEntity work.
|
|
75
|
+
KNOWLEDGE_BUILD_WORKER_ENABLED=true
|
|
76
|
+
# Leave blank to generate hostname:pid:random-suffix automatically.
|
|
77
|
+
KNOWLEDGE_BUILD_WORKER_ID=
|
|
78
|
+
# Maximum concurrent file builds per worker process.
|
|
79
|
+
KNOWLEDGE_BUILD_WORKER_CONCURRENCY=16
|
|
80
|
+
KNOWLEDGE_BUILD_WORKER_POLL_SECONDS=3
|
|
81
|
+
# Timeout and lease loss are terminal failures and are not retried.
|
|
82
|
+
KNOWLEDGE_BUILD_TASK_TIMEOUT_SECONDS=1200
|
|
83
|
+
KNOWLEDGE_BUILD_LEASE_SECONDS=180
|
|
84
|
+
# Must be shorter than KNOWLEDGE_BUILD_LEASE_SECONDS.
|
|
85
|
+
KNOWLEDGE_BUILD_HEARTBEAT_SECONDS=30
|
|
86
|
+
KNOWLEDGE_BUILD_REAPER_SECONDS=30
|
|
87
|
+
KNOWLEDGE_BUILD_WORKER_STATUS_LOG_SECONDS=60
|
|
88
|
+
KNOWLEDGE_BUILD_SHUTDOWN_GRACE_SECONDS=60
|
|
72
89
|
# Optional module:attribute provider implementing KnowledgeEventPublisher.
|
|
73
90
|
# It receives semantic Discovery/Enrich events and file/directory mutation events.
|
|
74
91
|
BY_QA_EVENT_PUBLISHER_PROVIDER=
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# by-qa Domain Language
|
|
2
|
+
|
|
3
|
+
by-qa manages knowledge files and the derived content that makes those files
|
|
4
|
+
searchable and usable by question-answering flows.
|
|
5
|
+
|
|
6
|
+
## Knowledge construction
|
|
7
|
+
|
|
8
|
+
**Knowledge File**:
|
|
9
|
+
A stable file identity inside a knowledge base, independent of its current name
|
|
10
|
+
or path.
|
|
11
|
+
_Avoid_: File path, document path
|
|
12
|
+
|
|
13
|
+
**File Build**:
|
|
14
|
+
The complete derivation of Markdown, chunks, embeddings, and retrieval data from
|
|
15
|
+
one Knowledge File. Its only model dependency is the embedding model.
|
|
16
|
+
_Avoid_: Embedding task, Markdown conversion
|
|
17
|
+
|
|
18
|
+
**Build Batch**:
|
|
19
|
+
One accepted File Build request and the newly created File Build Tasks that it
|
|
20
|
+
owns. Files matched to existing tasks are reported as reuse hits but do not
|
|
21
|
+
belong to the new batch.
|
|
22
|
+
_Avoid_: Parent task, job
|
|
23
|
+
|
|
24
|
+
**File Build Task**:
|
|
25
|
+
One durable attempt to build one Knowledge File from a captured content checksum
|
|
26
|
+
and Build Profile.
|
|
27
|
+
_Avoid_: Batch item, file path task
|
|
28
|
+
|
|
29
|
+
**Reuse Hit**:
|
|
30
|
+
An acceptance outcome in which an equivalent active, successful, or unsupported
|
|
31
|
+
File Build Task already exists, so the current Build Batch creates no task for
|
|
32
|
+
that file.
|
|
33
|
+
_Avoid_: Reused batch task, shared task
|
|
34
|
+
|
|
35
|
+
**Build Profile**:
|
|
36
|
+
The readable, canonical description of the parser, chunking, embedding, and
|
|
37
|
+
retrieval protocol that determines File Build output.
|
|
38
|
+
_Avoid_: Build hash, model version
|
|
39
|
+
|
|
40
|
+
**Built Content**:
|
|
41
|
+
Derived data whose latest task succeeded with the Knowledge File's current
|
|
42
|
+
checksum and whose Markdown, chunks, embeddings, and retrieval data are still
|
|
43
|
+
present.
|
|
44
|
+
_Avoid_: Latest task, protocol freshness
|
|
45
|
+
|
|
46
|
+
**Protocol Freshness**:
|
|
47
|
+
Whether Built Content was produced by the current Build Profile. A profile
|
|
48
|
+
change does not trigger File Build by itself.
|
|
49
|
+
_Avoid_: Content freshness, file freshness
|
|
50
|
+
|
|
51
|
+
**Inline File Build**:
|
|
52
|
+
A File Build owned and synchronously completed by an Entity processing task,
|
|
53
|
+
without joining an externally requested Build Batch.
|
|
54
|
+
_Avoid_: Priority build, nested batch
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: by-qa
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.3.dev1
|
|
4
4
|
Summary: Open-source knowledge-base ingestion and retrieval service.
|
|
5
5
|
Author: AI Assistant
|
|
6
6
|
License: MIT
|
|
@@ -22,7 +22,7 @@ Requires-Dist: python-dotenv>=1.0.0
|
|
|
22
22
|
Provides-Extra: all
|
|
23
23
|
Requires-Dist: aioboto3>=13.0.0; extra == 'all'
|
|
24
24
|
Requires-Dist: aiosqlite>=0.20.0; extra == 'all'
|
|
25
|
-
Requires-Dist: by-framework==
|
|
25
|
+
Requires-Dist: by-framework==v0.2.2.dev15; extra == 'all'
|
|
26
26
|
Requires-Dist: fastapi>=0.115.0; extra == 'all'
|
|
27
27
|
Requires-Dist: jieba>=0.42.1; extra == 'all'
|
|
28
28
|
Requires-Dist: json-repair==0.52.2; extra == 'all'
|
|
@@ -61,7 +61,7 @@ Requires-Dist: requests>=2.32.0; extra == 'eval'
|
|
|
61
61
|
Requires-Dist: tqdm>=4.67.0; extra == 'eval'
|
|
62
62
|
Provides-Extra: knowledge
|
|
63
63
|
Requires-Dist: aioboto3>=13.0.0; extra == 'knowledge'
|
|
64
|
-
Requires-Dist: by-framework==
|
|
64
|
+
Requires-Dist: by-framework==v0.2.2.dev15; extra == 'knowledge'
|
|
65
65
|
Requires-Dist: fastapi>=0.115.0; extra == 'knowledge'
|
|
66
66
|
Requires-Dist: jieba>=0.42.1; extra == 'knowledge'
|
|
67
67
|
Requires-Dist: json-repair==0.52.2; extra == 'knowledge'
|
|
@@ -81,7 +81,7 @@ Requires-Dist: uvicorn[standard]>=0.32.0; extra == 'knowledge'
|
|
|
81
81
|
Requires-Dist: xlrd>=2.0.0; extra == 'knowledge'
|
|
82
82
|
Provides-Extra: qa
|
|
83
83
|
Requires-Dist: aiosqlite>=0.20.0; extra == 'qa'
|
|
84
|
-
Requires-Dist: by-framework==
|
|
84
|
+
Requires-Dist: by-framework==v0.2.2.dev15; extra == 'qa'
|
|
85
85
|
Requires-Dist: json-repair==0.52.2; extra == 'qa'
|
|
86
86
|
Requires-Dist: langchain-core==1.3.3; extra == 'qa'
|
|
87
87
|
Requires-Dist: langchain-openai>=0.2.0; extra == 'qa'
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "by-qa"
|
|
7
|
-
version = "0.4.
|
|
7
|
+
version = "0.4.3.dev1"
|
|
8
8
|
description = "Open-source knowledge-base ingestion and retrieval service."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -49,7 +49,7 @@ knowledge = [
|
|
|
49
49
|
"python-pptx>=1.0.2",
|
|
50
50
|
"xlrd>=2.0.0",
|
|
51
51
|
"jieba>=0.42.1",
|
|
52
|
-
"by-framework==
|
|
52
|
+
"by-framework==v0.2.2.dev15",
|
|
53
53
|
"json_repair==0.52.2",
|
|
54
54
|
"lxml==6.1.0",
|
|
55
55
|
"urllib3==2.7.0",
|
|
@@ -64,7 +64,7 @@ qa = [
|
|
|
64
64
|
"langgraph-checkpoint-sqlite>=2.0.0",
|
|
65
65
|
"psycopg[binary]>=3.2.0",
|
|
66
66
|
"aiosqlite>=0.20.0",
|
|
67
|
-
"by-framework==
|
|
67
|
+
"by-framework==v0.2.2.dev15",
|
|
68
68
|
"json_repair==0.52.2",
|
|
69
69
|
"langsmith==0.8.0",
|
|
70
70
|
"lxml==6.1.0",
|
|
@@ -91,7 +91,7 @@ all = [
|
|
|
91
91
|
"langgraph-checkpoint-postgres>=2.0.0",
|
|
92
92
|
"langgraph-checkpoint-sqlite>=2.0.0",
|
|
93
93
|
"aiosqlite>=0.20.0",
|
|
94
|
-
"by-framework==
|
|
94
|
+
"by-framework==v0.2.2.dev15",
|
|
95
95
|
"json_repair==0.52.2",
|
|
96
96
|
"jieba>=0.42.1",
|
|
97
97
|
"langsmith==0.8.0",
|
|
@@ -225,6 +225,54 @@ class Settings(BaseSettings):
|
|
|
225
225
|
ge=0,
|
|
226
226
|
alias="KNOWLEDGE_ENTITY_SHUTDOWN_GRACE_SECONDS",
|
|
227
227
|
)
|
|
228
|
+
knowledge_build_worker_enabled: bool = Field(
|
|
229
|
+
default=True,
|
|
230
|
+
alias="KNOWLEDGE_BUILD_WORKER_ENABLED",
|
|
231
|
+
)
|
|
232
|
+
knowledge_build_worker_id: str = Field(
|
|
233
|
+
default="",
|
|
234
|
+
alias="KNOWLEDGE_BUILD_WORKER_ID",
|
|
235
|
+
)
|
|
236
|
+
knowledge_build_worker_concurrency: int = Field(
|
|
237
|
+
default=16,
|
|
238
|
+
ge=1,
|
|
239
|
+
alias="KNOWLEDGE_BUILD_WORKER_CONCURRENCY",
|
|
240
|
+
)
|
|
241
|
+
knowledge_build_worker_poll_seconds: float = Field(
|
|
242
|
+
default=3.0,
|
|
243
|
+
gt=0,
|
|
244
|
+
alias="KNOWLEDGE_BUILD_WORKER_POLL_SECONDS",
|
|
245
|
+
)
|
|
246
|
+
knowledge_build_task_timeout_seconds: float = Field(
|
|
247
|
+
default=1200.0,
|
|
248
|
+
gt=0,
|
|
249
|
+
alias="KNOWLEDGE_BUILD_TASK_TIMEOUT_SECONDS",
|
|
250
|
+
)
|
|
251
|
+
knowledge_build_lease_seconds: int = Field(
|
|
252
|
+
default=180,
|
|
253
|
+
ge=2,
|
|
254
|
+
alias="KNOWLEDGE_BUILD_LEASE_SECONDS",
|
|
255
|
+
)
|
|
256
|
+
knowledge_build_heartbeat_seconds: float = Field(
|
|
257
|
+
default=30.0,
|
|
258
|
+
gt=0,
|
|
259
|
+
alias="KNOWLEDGE_BUILD_HEARTBEAT_SECONDS",
|
|
260
|
+
)
|
|
261
|
+
knowledge_build_reaper_seconds: float = Field(
|
|
262
|
+
default=30.0,
|
|
263
|
+
gt=0,
|
|
264
|
+
alias="KNOWLEDGE_BUILD_REAPER_SECONDS",
|
|
265
|
+
)
|
|
266
|
+
knowledge_build_worker_status_log_seconds: float = Field(
|
|
267
|
+
default=60.0,
|
|
268
|
+
gt=0,
|
|
269
|
+
alias="KNOWLEDGE_BUILD_WORKER_STATUS_LOG_SECONDS",
|
|
270
|
+
)
|
|
271
|
+
knowledge_build_shutdown_grace_seconds: float = Field(
|
|
272
|
+
default=60.0,
|
|
273
|
+
ge=0,
|
|
274
|
+
alias="KNOWLEDGE_BUILD_SHUTDOWN_GRACE_SECONDS",
|
|
275
|
+
)
|
|
228
276
|
entity_embedding_index_enabled: bool = Field(
|
|
229
277
|
default=True,
|
|
230
278
|
alias="ENTITY_EMBEDDING_INDEX_ENABLED",
|
|
@@ -298,6 +346,17 @@ class Settings(BaseSettings):
|
|
|
298
346
|
)
|
|
299
347
|
return value
|
|
300
348
|
|
|
349
|
+
@field_validator("knowledge_build_heartbeat_seconds")
|
|
350
|
+
@classmethod
|
|
351
|
+
def _validate_knowledge_build_heartbeat_seconds(cls, value: float, info) -> float:
|
|
352
|
+
lease_seconds = info.data.get("knowledge_build_lease_seconds", 180)
|
|
353
|
+
if value >= lease_seconds:
|
|
354
|
+
raise ValueError(
|
|
355
|
+
"KNOWLEDGE_BUILD_HEARTBEAT_SECONDS must be less than "
|
|
356
|
+
"KNOWLEDGE_BUILD_LEASE_SECONDS"
|
|
357
|
+
)
|
|
358
|
+
return value
|
|
359
|
+
|
|
301
360
|
@field_validator(
|
|
302
361
|
"llm_standard_extra_body", "llm_lightweight_extra_body", mode="before"
|
|
303
362
|
)
|
|
@@ -6,7 +6,14 @@ from datetime import datetime
|
|
|
6
6
|
from enum import StrEnum
|
|
7
7
|
from typing import Any, Protocol
|
|
8
8
|
|
|
9
|
-
from pydantic import
|
|
9
|
+
from pydantic import (
|
|
10
|
+
AliasChoices,
|
|
11
|
+
BaseModel,
|
|
12
|
+
ConfigDict,
|
|
13
|
+
Field,
|
|
14
|
+
field_validator,
|
|
15
|
+
model_validator,
|
|
16
|
+
)
|
|
10
17
|
|
|
11
18
|
|
|
12
19
|
class _ApiModel(BaseModel):
|
|
@@ -54,6 +61,7 @@ class ProcessingEligibility(StrEnum):
|
|
|
54
61
|
class ProcessingTaskType(StrEnum):
|
|
55
62
|
ENTITY_DISCOVERY = "ENTITY_DISCOVERY"
|
|
56
63
|
DOCUMENT_ENRICH = "DOCUMENT_ENRICH"
|
|
64
|
+
FILE_BUILD = "FILE_BUILD"
|
|
57
65
|
|
|
58
66
|
|
|
59
67
|
class ProcessingTaskStatus(StrEnum):
|
|
@@ -63,6 +71,7 @@ class ProcessingTaskStatus(StrEnum):
|
|
|
63
71
|
FAILED = "FAILED"
|
|
64
72
|
CANCELLED = "CANCELLED"
|
|
65
73
|
SKIPPED = "SKIPPED"
|
|
74
|
+
UNSUPPORTED = "UNSUPPORTED"
|
|
66
75
|
|
|
67
76
|
|
|
68
77
|
class ProcessingScope(StrEnum):
|
|
@@ -72,6 +81,7 @@ class ProcessingScope(StrEnum):
|
|
|
72
81
|
|
|
73
82
|
|
|
74
83
|
class ProcessingBatchStatus(StrEnum):
|
|
84
|
+
PENDING = "PENDING"
|
|
75
85
|
PROCESSING = "PROCESSING"
|
|
76
86
|
COMPLETED = "COMPLETED"
|
|
77
87
|
|
|
@@ -202,6 +212,16 @@ class ProcessingTaskStatusRequest(_ApiModel):
|
|
|
202
212
|
min_length=1,
|
|
203
213
|
validation_alias=AliasChoices("knCode", "kb_code"),
|
|
204
214
|
)
|
|
215
|
+
task_id: int | None = Field(
|
|
216
|
+
default=None,
|
|
217
|
+
gt=0,
|
|
218
|
+
validation_alias=AliasChoices("taskId", "task_id"),
|
|
219
|
+
)
|
|
220
|
+
file_id: int | None = Field(
|
|
221
|
+
default=None,
|
|
222
|
+
gt=0,
|
|
223
|
+
validation_alias=AliasChoices("fileId", "file_id"),
|
|
224
|
+
)
|
|
205
225
|
file_path: str | None = Field(
|
|
206
226
|
default=None,
|
|
207
227
|
validation_alias=AliasChoices("filePath", "file_path"),
|
|
@@ -251,6 +271,12 @@ class ProcessingTaskStatusRequest(_ApiModel):
|
|
|
251
271
|
raise ValueError("statusList must not contain duplicate values")
|
|
252
272
|
return value
|
|
253
273
|
|
|
274
|
+
@model_validator(mode="after")
|
|
275
|
+
def validate_exact_task_query(self) -> "ProcessingTaskStatusRequest":
|
|
276
|
+
if self.task_id is not None and self.task_type is None:
|
|
277
|
+
raise ValueError("taskType is required when taskId is provided")
|
|
278
|
+
return self
|
|
279
|
+
|
|
254
280
|
|
|
255
281
|
class ProcessingBatchStatusRequest(_ApiModel):
|
|
256
282
|
"""Query one durable batch and its per-file progress."""
|
|
@@ -368,7 +394,32 @@ class ProcessingTaskItem(_ApiModel):
|
|
|
368
394
|
current_stage: str | None = Field(default=None, serialization_alias="currentStage")
|
|
369
395
|
progress: int | None = Field(default=None, ge=0, le=100)
|
|
370
396
|
file_id: str | None = Field(serialization_alias="fileId")
|
|
371
|
-
file_path: str = Field(serialization_alias="filePath")
|
|
397
|
+
file_path: str | None = Field(default=None, serialization_alias="filePath")
|
|
398
|
+
file_path_snapshot: str | None = Field(
|
|
399
|
+
default=None, serialization_alias="filePathSnapshot"
|
|
400
|
+
)
|
|
401
|
+
origin: str | None = None
|
|
402
|
+
execution_mode: str | None = Field(
|
|
403
|
+
default=None, serialization_alias="executionMode"
|
|
404
|
+
)
|
|
405
|
+
parent_semantic_task_id: str | None = Field(
|
|
406
|
+
default=None, serialization_alias="parentSemanticTaskId"
|
|
407
|
+
)
|
|
408
|
+
input_checksum: str | None = Field(
|
|
409
|
+
default=None, serialization_alias="inputChecksum"
|
|
410
|
+
)
|
|
411
|
+
input_is_deleted: bool | None = Field(
|
|
412
|
+
default=None, serialization_alias="inputIsDeleted"
|
|
413
|
+
)
|
|
414
|
+
build_profile: dict[str, Any] | None = Field(
|
|
415
|
+
default=None, serialization_alias="buildProfile"
|
|
416
|
+
)
|
|
417
|
+
build_profile_hash: str | None = Field(
|
|
418
|
+
default=None, serialization_alias="buildProfileHash"
|
|
419
|
+
)
|
|
420
|
+
outcome_uncertain: bool | None = Field(
|
|
421
|
+
default=None, serialization_alias="outcomeUncertain"
|
|
422
|
+
)
|
|
372
423
|
index_version: str | None = Field(default=None, serialization_alias="indexVersion")
|
|
373
424
|
created_at: datetime = Field(serialization_alias="createdAt")
|
|
374
425
|
started_at: datetime | None = Field(default=None, serialization_alias="startedAt")
|
|
@@ -393,6 +444,7 @@ class ProcessingBatchStatusResult(_ApiModel):
|
|
|
393
444
|
kb_code: str = Field(serialization_alias="knCode")
|
|
394
445
|
task_type: ProcessingTaskType = Field(serialization_alias="taskType")
|
|
395
446
|
scope: ProcessingScope
|
|
447
|
+
target_path: str | None = Field(default=None, serialization_alias="targetPath")
|
|
396
448
|
status: ProcessingBatchStatus
|
|
397
449
|
version: int = Field(ge=0)
|
|
398
450
|
total_count: int = Field(ge=0, serialization_alias="totalCount")
|
|
@@ -402,6 +454,24 @@ class ProcessingBatchStatusResult(_ApiModel):
|
|
|
402
454
|
succeeded_count: int = Field(ge=0, serialization_alias="succeededCount")
|
|
403
455
|
failed_count: int = Field(ge=0, serialization_alias="failedCount")
|
|
404
456
|
skipped_count: int = Field(ge=0, serialization_alias="skippedCount")
|
|
457
|
+
unsupported_count: int = Field(
|
|
458
|
+
default=0, ge=0, serialization_alias="unsupportedCount"
|
|
459
|
+
)
|
|
460
|
+
candidate_count: int | None = Field(
|
|
461
|
+
default=None, ge=0, serialization_alias="candidateCount"
|
|
462
|
+
)
|
|
463
|
+
eligible_count: int | None = Field(
|
|
464
|
+
default=None, ge=0, serialization_alias="eligibleCount"
|
|
465
|
+
)
|
|
466
|
+
accepted_count: int | None = Field(
|
|
467
|
+
default=None, ge=0, serialization_alias="acceptedCount"
|
|
468
|
+
)
|
|
469
|
+
reused_count: int | None = Field(
|
|
470
|
+
default=None, ge=0, serialization_alias="reusedCount"
|
|
471
|
+
)
|
|
472
|
+
acceptance_skipped_count: int | None = Field(
|
|
473
|
+
default=None, ge=0, serialization_alias="acceptanceSkippedCount"
|
|
474
|
+
)
|
|
405
475
|
progress: int = Field(ge=0, le=100)
|
|
406
476
|
created_at: datetime = Field(serialization_alias="createdAt")
|
|
407
477
|
completed_at: datetime | None = Field(
|
|
@@ -251,15 +251,6 @@ def register_routes(
|
|
|
251
251
|
):
|
|
252
252
|
"""Register knowledge base API routes on the FastAPI app."""
|
|
253
253
|
|
|
254
|
-
async def _run_file_to_markdown_index_task(service, request, *, build_task_id: int):
|
|
255
|
-
"""Resolve heavy dependencies inside the background task itself."""
|
|
256
|
-
chunking_service = await _resolve_maybe_async(get_document_chunking_service)
|
|
257
|
-
await service.execute_file_to_markdown_index_task(
|
|
258
|
-
request,
|
|
259
|
-
document_chunking_service=chunking_service,
|
|
260
|
-
build_task_id=build_task_id,
|
|
261
|
-
)
|
|
262
|
-
|
|
263
254
|
async def _get_knowledge_entity_service() -> KnowledgeEntityProcessingService:
|
|
264
255
|
if get_knowledge_entity_processing_service is None:
|
|
265
256
|
raise KnowledgeBaseConfigurationError(
|
|
@@ -1116,7 +1107,7 @@ def register_routes(
|
|
|
1116
1107
|
|
|
1117
1108
|
@app.post("/api/v1/fileToMarkdownIndex")
|
|
1118
1109
|
async def file_to_markdown_index(
|
|
1119
|
-
|
|
1110
|
+
body: dict[str, Any] = Body(...),
|
|
1120
1111
|
):
|
|
1121
1112
|
logger.info(
|
|
1122
1113
|
"file_to_markdown_index request received: body_keys=%s",
|
|
@@ -1132,13 +1123,7 @@ def register_routes(
|
|
|
1132
1123
|
|
|
1133
1124
|
try:
|
|
1134
1125
|
service = await get_knowledge_item_ingestion_service()
|
|
1135
|
-
|
|
1136
|
-
background_tasks.add_task(
|
|
1137
|
-
_run_file_to_markdown_index_task,
|
|
1138
|
-
service,
|
|
1139
|
-
request,
|
|
1140
|
-
build_task_id=build_task_id,
|
|
1141
|
-
)
|
|
1126
|
+
result = await service.accept_file_to_markdown_index(request)
|
|
1142
1127
|
except KnowledgeBaseConfigurationError as exc:
|
|
1143
1128
|
logger.warning("file_to_markdown_index configuration failed: error=%s", exc)
|
|
1144
1129
|
return _documented_error_response(
|
|
@@ -1159,7 +1144,10 @@ def register_routes(
|
|
|
1159
1144
|
request.kb_code,
|
|
1160
1145
|
request.file_path,
|
|
1161
1146
|
)
|
|
1162
|
-
return _documented_success_response(
|
|
1147
|
+
return _documented_success_response(
|
|
1148
|
+
result_object=result,
|
|
1149
|
+
result_msg="accepted",
|
|
1150
|
+
)
|
|
1163
1151
|
|
|
1164
1152
|
@app.post("/api/v1/fileBuildStatus")
|
|
1165
1153
|
async def file_build_status(body: dict[str, Any] = Body(...)):
|
|
@@ -1915,8 +1903,15 @@ def register_routes(
|
|
|
1915
1903
|
request.batch_id,
|
|
1916
1904
|
)
|
|
1917
1905
|
try:
|
|
1918
|
-
|
|
1919
|
-
|
|
1906
|
+
ingestion_service = await get_knowledge_item_ingestion_service()
|
|
1907
|
+
build_service = getattr(
|
|
1908
|
+
ingestion_service, "file_build_processing_service", None
|
|
1909
|
+
)
|
|
1910
|
+
if build_service is not None:
|
|
1911
|
+
result = await build_service.get_unified_processing_task_status(request)
|
|
1912
|
+
else:
|
|
1913
|
+
service = await _get_knowledge_entity_service()
|
|
1914
|
+
result = await service.get_processing_task_status(request)
|
|
1920
1915
|
except KnowledgeBaseConfigurationError as exc:
|
|
1921
1916
|
return _documented_error_response(
|
|
1922
1917
|
result_msg=str(exc),
|
|
@@ -1956,8 +1951,15 @@ def register_routes(
|
|
|
1956
1951
|
request.batch_id,
|
|
1957
1952
|
)
|
|
1958
1953
|
try:
|
|
1959
|
-
|
|
1960
|
-
|
|
1954
|
+
result = None
|
|
1955
|
+
if request.batch_id.startswith("fb-"):
|
|
1956
|
+
ingestion_service = await get_knowledge_item_ingestion_service()
|
|
1957
|
+
result = await ingestion_service.file_build_processing_service.get_processing_batch_status(
|
|
1958
|
+
request
|
|
1959
|
+
)
|
|
1960
|
+
if result is None:
|
|
1961
|
+
service = await _get_knowledge_entity_service()
|
|
1962
|
+
result = await service.get_processing_batch_status(request)
|
|
1961
1963
|
except KnowledgeBaseConfigurationError as exc:
|
|
1962
1964
|
return _documented_error_response(
|
|
1963
1965
|
result_msg=str(exc),
|
|
@@ -240,7 +240,7 @@ class MoveKnowledgeItemsResponse(BaseModel):
|
|
|
240
240
|
|
|
241
241
|
|
|
242
242
|
class FileToMarkdownIndexRequest(BaseModel):
|
|
243
|
-
"""Request body for
|
|
243
|
+
"""Request body for accepting a file or directory build batch."""
|
|
244
244
|
|
|
245
245
|
model_config = ConfigDict(populate_by_name=True)
|
|
246
246
|
|
|
@@ -252,6 +252,7 @@ class FileToMarkdownIndexRequest(BaseModel):
|
|
|
252
252
|
min_length=1,
|
|
253
253
|
validation_alias=AliasChoices("filePath", "file_path"),
|
|
254
254
|
)
|
|
255
|
+
force: bool = False
|
|
255
256
|
|
|
256
257
|
|
|
257
258
|
class FileBuildStatusRequest(BaseModel):
|
|
@@ -7,6 +7,7 @@ from typing import Final
|
|
|
7
7
|
BUILD_STATUS_COMPLETE: Final[str] = "complete"
|
|
8
8
|
BUILD_STATUS_FAILED: Final[str] = "failed"
|
|
9
9
|
BUILD_STATUS_RUNNING: Final[str] = "running"
|
|
10
|
+
BUILD_STATUS_SKIPPED: Final[str] = "skipped"
|
|
10
11
|
BUILD_STATUS_UNSUPPORTED: Final[str] = "unsupported"
|
|
11
12
|
|
|
12
13
|
BUILD_STEP_MARKDOWN: Final[str] = "markdown"
|
|
@@ -30,6 +31,11 @@ STATUS_DICT: Final[list[dict[str, str]]] = [
|
|
|
30
31
|
"standDisplayValue": "构建中",
|
|
31
32
|
"standDisplayValueEn": BUILD_STATUS_RUNNING,
|
|
32
33
|
},
|
|
34
|
+
{
|
|
35
|
+
"standCode": BUILD_STATUS_SKIPPED,
|
|
36
|
+
"standDisplayValue": "已跳过",
|
|
37
|
+
"standDisplayValueEn": BUILD_STATUS_SKIPPED,
|
|
38
|
+
},
|
|
33
39
|
{
|
|
34
40
|
"standCode": BUILD_STATUS_UNSUPPORTED,
|
|
35
41
|
"standDisplayValue": "不支持构建",
|
|
@@ -59,3 +65,36 @@ STEP_DICT: Final[list[dict[str, str]]] = [
|
|
|
59
65
|
"standDisplayValueEn": BUILD_STEP_COMPLETE,
|
|
60
66
|
},
|
|
61
67
|
]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def legacy_build_status(status: str | None) -> str | None:
|
|
71
|
+
"""Map the durable lifecycle onto the legacy status vocabulary."""
|
|
72
|
+
normalized = (status or "").lower()
|
|
73
|
+
if normalized in {"pending", "running"}:
|
|
74
|
+
return BUILD_STATUS_RUNNING
|
|
75
|
+
if normalized in {"succeeded", "complete"}:
|
|
76
|
+
return BUILD_STATUS_COMPLETE
|
|
77
|
+
if normalized in {
|
|
78
|
+
BUILD_STATUS_FAILED,
|
|
79
|
+
BUILD_STATUS_SKIPPED,
|
|
80
|
+
BUILD_STATUS_UNSUPPORTED,
|
|
81
|
+
}:
|
|
82
|
+
return normalized
|
|
83
|
+
return status
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def legacy_build_step(
|
|
87
|
+
*, status: str | None, current_step: str | None, current_stage: str | None
|
|
88
|
+
) -> str | None:
|
|
89
|
+
"""Map new milestones onto the four legacy build steps."""
|
|
90
|
+
if legacy_build_status(status) == BUILD_STATUS_COMPLETE:
|
|
91
|
+
return BUILD_STEP_COMPLETE
|
|
92
|
+
normalized_stage = (current_stage or "").lower()
|
|
93
|
+
stage_steps = {
|
|
94
|
+
"accepted": BUILD_STEP_MARKDOWN,
|
|
95
|
+
"extracting": BUILD_STEP_MARKDOWN,
|
|
96
|
+
"chunking": BUILD_STEP_CHUNKING,
|
|
97
|
+
"embedding": BUILD_STEP_VECTORIZING,
|
|
98
|
+
"committing": BUILD_STEP_VECTORIZING,
|
|
99
|
+
}
|
|
100
|
+
return stage_steps.get(normalized_stage, current_step)
|
|
@@ -46,6 +46,7 @@ class SemanticEventType(StrEnum):
|
|
|
46
46
|
|
|
47
47
|
class BuildEventType(StrEnum):
|
|
48
48
|
FILE_COMPLETED = "build.file.completed"
|
|
49
|
+
BATCH_COMPLETED = "build.batch.completed"
|
|
49
50
|
|
|
50
51
|
|
|
51
52
|
class _StrictEventModel(BaseModel):
|
|
@@ -198,6 +199,46 @@ class BuildFileCompletedPayload(_StrictEventModel):
|
|
|
198
199
|
return self
|
|
199
200
|
|
|
200
201
|
|
|
202
|
+
class FileBuildTerminalPayload(_StrictEventModel):
|
|
203
|
+
batch_id: str = Field(alias="batchId")
|
|
204
|
+
task_id: str = Field(alias="taskId")
|
|
205
|
+
task_type: Literal["FILE_BUILD"] = Field(default="FILE_BUILD", alias="taskType")
|
|
206
|
+
file_id: str = Field(alias="fileId")
|
|
207
|
+
file_path_snapshot: str = Field(alias="filePathSnapshot")
|
|
208
|
+
status: Literal["SUCCEEDED", "FAILED", "SKIPPED", "UNSUPPORTED"]
|
|
209
|
+
stage: Literal["ACCEPTED", "EXTRACTING", "CHUNKING", "EMBEDDING", "COMMITTING"]
|
|
210
|
+
progress: Literal[100] = 100
|
|
211
|
+
result: BuildFileCompletedResult | None
|
|
212
|
+
error: BuildProcessingError | None
|
|
213
|
+
|
|
214
|
+
@model_validator(mode="after")
|
|
215
|
+
def _validate_terminal_shape(self) -> FileBuildTerminalPayload:
|
|
216
|
+
if self.status == "SUCCEEDED":
|
|
217
|
+
if self.result is None or self.error is not None:
|
|
218
|
+
raise ValueError("successful file build requires result only")
|
|
219
|
+
elif self.result is not None or self.error is None:
|
|
220
|
+
raise ValueError("non-successful file build requires error only")
|
|
221
|
+
return self
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class FileBuildBatchTerminalPayload(_StrictEventModel):
|
|
225
|
+
batch_id: str = Field(alias="batchId")
|
|
226
|
+
task_type: Literal["FILE_BUILD"] = Field(default="FILE_BUILD", alias="taskType")
|
|
227
|
+
scope: Literal["SINGLE_FILE", "DIRECTORY"]
|
|
228
|
+
target_path: str = Field(alias="targetPath")
|
|
229
|
+
candidate_count: int = Field(ge=0, alias="candidateCount")
|
|
230
|
+
eligible_count: int = Field(ge=0, alias="eligibleCount")
|
|
231
|
+
accepted_count: int = Field(ge=0, alias="acceptedCount")
|
|
232
|
+
reused_count: int = Field(ge=0, alias="reusedCount")
|
|
233
|
+
acceptance_skipped_count: int = Field(ge=0, alias="acceptanceSkippedCount")
|
|
234
|
+
total_count: int = Field(ge=0, alias="totalCount")
|
|
235
|
+
completed_count: int = Field(ge=0, alias="completedCount")
|
|
236
|
+
succeeded_count: int = Field(ge=0, alias="succeededCount")
|
|
237
|
+
failed_count: int = Field(ge=0, alias="failedCount")
|
|
238
|
+
skipped_count: int = Field(ge=0, alias="skippedCount")
|
|
239
|
+
unsupported_count: int = Field(ge=0, alias="unsupportedCount")
|
|
240
|
+
|
|
241
|
+
|
|
201
242
|
class _KnowledgeEventBase(_StrictEventModel):
|
|
202
243
|
event_id: str = Field(alias="eventId")
|
|
203
244
|
event_version: Literal[1] = Field(default=EVENT_VERSION, alias="eventVersion")
|
|
@@ -282,11 +323,29 @@ class EnrichBatchCompletedEvent(_KnowledgeEventBase):
|
|
|
282
323
|
payload: SemanticBatchCompletedPayload
|
|
283
324
|
|
|
284
325
|
|
|
285
|
-
class BuildFileCompletedEvent(
|
|
326
|
+
class BuildFileCompletedEvent(_StrictEventModel):
|
|
327
|
+
event_id: str = Field(alias="eventId")
|
|
328
|
+
event_version: Literal[1, 2] = Field(default=1, alias="eventVersion")
|
|
329
|
+
kb_code: str = Field(alias="knCode")
|
|
330
|
+
occurred_at: datetime = Field(alias="occurredAt")
|
|
286
331
|
event_type: Literal["build.file.completed"] = Field(
|
|
287
332
|
default=BuildEventType.FILE_COMPLETED.value, alias="eventType"
|
|
288
333
|
)
|
|
289
|
-
payload: BuildFileCompletedPayload
|
|
334
|
+
payload: BuildFileCompletedPayload | FileBuildTerminalPayload
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class _KnowledgeEventV2Base(_StrictEventModel):
|
|
338
|
+
event_id: str = Field(alias="eventId")
|
|
339
|
+
event_version: Literal[2] = Field(default=2, alias="eventVersion")
|
|
340
|
+
kb_code: str = Field(alias="knCode")
|
|
341
|
+
occurred_at: datetime = Field(alias="occurredAt")
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class FileBuildBatchTerminalEvent(_KnowledgeEventV2Base):
|
|
345
|
+
event_type: Literal["build.batch.completed"] = Field(
|
|
346
|
+
default=BuildEventType.BATCH_COMPLETED.value, alias="eventType"
|
|
347
|
+
)
|
|
348
|
+
payload: FileBuildBatchTerminalPayload
|
|
290
349
|
|
|
291
350
|
|
|
292
351
|
_KnowledgeEventUnion: TypeAlias = (
|
|
@@ -302,6 +361,7 @@ _KnowledgeEventUnion: TypeAlias = (
|
|
|
302
361
|
| EnrichFileCompletedEvent
|
|
303
362
|
| EnrichBatchCompletedEvent
|
|
304
363
|
| BuildFileCompletedEvent
|
|
364
|
+
| FileBuildBatchTerminalEvent
|
|
305
365
|
)
|
|
306
366
|
KnowledgeEvent: TypeAlias = Annotated[
|
|
307
367
|
_KnowledgeEventUnion, Field(discriminator="event_type")
|
|
@@ -485,6 +545,71 @@ def build_file_completed_event(
|
|
|
485
545
|
)
|
|
486
546
|
|
|
487
547
|
|
|
548
|
+
def build_file_build_terminal_event(
|
|
549
|
+
task: Mapping[str, Any],
|
|
550
|
+
) -> BuildFileCompletedEvent:
|
|
551
|
+
"""Build the eventVersion 2 terminal event for an external Build task."""
|
|
552
|
+
status = str(task["status"]).upper()
|
|
553
|
+
result_payload = normalize_json_mapping(task.get("result_payload"))
|
|
554
|
+
result = None
|
|
555
|
+
error = None
|
|
556
|
+
if status == "SUCCEEDED":
|
|
557
|
+
result = BuildFileCompletedResult(
|
|
558
|
+
chunk_count=int((result_payload or {}).get("chunkCount") or 0),
|
|
559
|
+
line_count=int((result_payload or {}).get("lineCount") or 0),
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
error = BuildProcessingError(
|
|
563
|
+
code=str(task.get("error_code") or "BUILD_FAILED"),
|
|
564
|
+
message=str(task.get("error_message") or "file build did not succeed"),
|
|
565
|
+
)
|
|
566
|
+
return BuildFileCompletedEvent(
|
|
567
|
+
**_event_fields(
|
|
568
|
+
kb_code=str(task.get("kb_code") or task["knowledge_base_id"]),
|
|
569
|
+
occurred_at=task.get("finished_at"),
|
|
570
|
+
),
|
|
571
|
+
event_version=2,
|
|
572
|
+
payload=FileBuildTerminalPayload(
|
|
573
|
+
batch_id=str(task["batch_id"]),
|
|
574
|
+
task_id=str(task["kid"]),
|
|
575
|
+
file_id=str(task["fs_entry_id"]),
|
|
576
|
+
file_path_snapshot=str(task["file_path_snapshot"]),
|
|
577
|
+
status=status,
|
|
578
|
+
stage=str(task["current_stage"]).upper(),
|
|
579
|
+
result=result,
|
|
580
|
+
error=error,
|
|
581
|
+
),
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def build_file_build_batch_terminal_event(
|
|
586
|
+
batch: Mapping[str, Any], counts: Mapping[str, int]
|
|
587
|
+
) -> FileBuildBatchTerminalEvent:
|
|
588
|
+
"""Build the eventVersion 2 aggregate event for a completed Build batch."""
|
|
589
|
+
return FileBuildBatchTerminalEvent(
|
|
590
|
+
**_event_fields(
|
|
591
|
+
kb_code=str(batch.get("kb_code") or batch["knowledge_base_id"]),
|
|
592
|
+
occurred_at=batch.get("completed_at"),
|
|
593
|
+
),
|
|
594
|
+
payload=FileBuildBatchTerminalPayload(
|
|
595
|
+
batch_id=str(batch["batch_id"]),
|
|
596
|
+
scope=str(batch["scope"]).upper(),
|
|
597
|
+
target_path=str(batch["target_path_snapshot"]),
|
|
598
|
+
candidate_count=int(batch["candidate_count"]),
|
|
599
|
+
eligible_count=int(batch["eligible_count"]),
|
|
600
|
+
accepted_count=int(batch["accepted_count"]),
|
|
601
|
+
reused_count=int(batch["reused_count"]),
|
|
602
|
+
acceptance_skipped_count=int(batch["acceptance_skipped_count"]),
|
|
603
|
+
total_count=int(batch["accepted_count"]),
|
|
604
|
+
completed_count=int(batch["completed_count"]),
|
|
605
|
+
succeeded_count=int(counts.get("succeeded", 0)),
|
|
606
|
+
failed_count=int(counts.get("failed", 0)),
|
|
607
|
+
skipped_count=int(counts.get("skipped", 0)),
|
|
608
|
+
unsupported_count=int(counts.get("unsupported", 0)),
|
|
609
|
+
),
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
|
|
488
613
|
def _semantic_event_classes(task_type: str):
|
|
489
614
|
if task_type == "ENTITY_DISCOVERY":
|
|
490
615
|
return DiscoveryFileCompletedEvent, DiscoveryBatchCompletedEvent
|