synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/__init__.py +24 -0
- synapse_sdk/cli/__init__.py +308 -5
- synapse_sdk/cli/alias/utils.py +1 -1
- synapse_sdk/cli/code_server.py +687 -0
- synapse_sdk/cli/config.py +440 -0
- synapse_sdk/cli/devtools.py +90 -0
- synapse_sdk/cli/plugin/publish.py +23 -15
- synapse_sdk/clients/agent/__init__.py +9 -3
- synapse_sdk/clients/agent/container.py +133 -0
- synapse_sdk/clients/agent/core.py +19 -0
- synapse_sdk/clients/agent/ray.py +298 -9
- synapse_sdk/clients/backend/__init__.py +28 -12
- synapse_sdk/clients/backend/annotation.py +9 -1
- synapse_sdk/clients/backend/core.py +31 -4
- synapse_sdk/clients/backend/data_collection.py +186 -0
- synapse_sdk/clients/backend/hitl.py +1 -1
- synapse_sdk/clients/backend/integration.py +4 -3
- synapse_sdk/clients/backend/ml.py +1 -1
- synapse_sdk/clients/backend/models.py +35 -1
- synapse_sdk/clients/base.py +309 -36
- synapse_sdk/clients/ray/serve.py +2 -0
- synapse_sdk/devtools/__init__.py +0 -0
- synapse_sdk/devtools/config.py +94 -0
- synapse_sdk/devtools/docs/.gitignore +20 -0
- synapse_sdk/devtools/docs/README.md +41 -0
- synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md +12 -0
- synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md +44 -0
- synapse_sdk/devtools/docs/blog/2021-08-01-mdx-blog-post.mdx +24 -0
- synapse_sdk/devtools/docs/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg +0 -0
- synapse_sdk/devtools/docs/blog/2021-08-26-welcome/index.md +29 -0
- synapse_sdk/devtools/docs/blog/authors.yml +25 -0
- synapse_sdk/devtools/docs/blog/tags.yml +19 -0
- synapse_sdk/devtools/docs/docs/api/clients/agent.md +43 -0
- synapse_sdk/devtools/docs/docs/api/clients/annotation-mixin.md +378 -0
- synapse_sdk/devtools/docs/docs/api/clients/backend.md +420 -0
- synapse_sdk/devtools/docs/docs/api/clients/base.md +257 -0
- synapse_sdk/devtools/docs/docs/api/clients/core-mixin.md +477 -0
- synapse_sdk/devtools/docs/docs/api/clients/data-collection-mixin.md +422 -0
- synapse_sdk/devtools/docs/docs/api/clients/hitl-mixin.md +554 -0
- synapse_sdk/devtools/docs/docs/api/clients/index.md +391 -0
- synapse_sdk/devtools/docs/docs/api/clients/integration-mixin.md +571 -0
- synapse_sdk/devtools/docs/docs/api/clients/ml-mixin.md +578 -0
- synapse_sdk/devtools/docs/docs/api/clients/ray.md +342 -0
- synapse_sdk/devtools/docs/docs/api/index.md +52 -0
- synapse_sdk/devtools/docs/docs/api/plugins/categories.md +43 -0
- synapse_sdk/devtools/docs/docs/api/plugins/models.md +114 -0
- synapse_sdk/devtools/docs/docs/api/plugins/utils.md +328 -0
- synapse_sdk/devtools/docs/docs/categories.md +0 -0
- synapse_sdk/devtools/docs/docs/cli-usage.md +280 -0
- synapse_sdk/devtools/docs/docs/concepts/index.md +38 -0
- synapse_sdk/devtools/docs/docs/configuration.md +83 -0
- synapse_sdk/devtools/docs/docs/contributing.md +306 -0
- synapse_sdk/devtools/docs/docs/examples/index.md +29 -0
- synapse_sdk/devtools/docs/docs/faq.md +179 -0
- synapse_sdk/devtools/docs/docs/features/converters/index.md +455 -0
- synapse_sdk/devtools/docs/docs/features/index.md +24 -0
- synapse_sdk/devtools/docs/docs/features/utils/file.md +415 -0
- synapse_sdk/devtools/docs/docs/features/utils/network.md +378 -0
- synapse_sdk/devtools/docs/docs/features/utils/storage.md +57 -0
- synapse_sdk/devtools/docs/docs/features/utils/types.md +51 -0
- synapse_sdk/devtools/docs/docs/installation.md +94 -0
- synapse_sdk/devtools/docs/docs/introduction.md +47 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/neural-net-plugins/train-action-overview.md +814 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-action.md +948 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-overview.md +544 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-template.md +766 -0
- synapse_sdk/devtools/docs/docs/plugins/export-plugins.md +1092 -0
- synapse_sdk/devtools/docs/docs/plugins/plugins.md +852 -0
- synapse_sdk/devtools/docs/docs/quickstart.md +78 -0
- synapse_sdk/devtools/docs/docs/troubleshooting.md +519 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/_category_.json +8 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/congratulations.md +23 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-blog-post.md +34 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-document.md +57 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-page.md +43 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/deploy-your-site.md +31 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/markdown-features.mdx +152 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/_category_.json +7 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/img/docsVersionDropdown.png +0 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/img/localeDropdown.png +0 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/manage-docs-versions.md +55 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/translate-your-site.md +88 -0
- synapse_sdk/devtools/docs/docusaurus.config.ts +148 -0
- synapse_sdk/devtools/docs/i18n/ko/code.json +325 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/agent.md +43 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/annotation-mixin.md +289 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/backend.md +420 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/base.md +257 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/core-mixin.md +417 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/data-collection-mixin.md +356 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/hitl-mixin.md +192 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/index.md +391 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/integration-mixin.md +479 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ml-mixin.md +284 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ray.md +342 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/index.md +52 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/plugins/models.md +114 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/categories.md +0 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/cli-usage.md +280 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/concepts/index.md +38 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/configuration.md +83 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/contributing.md +306 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/examples/index.md +29 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/faq.md +179 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/converters/index.md +30 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/index.md +24 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/file.md +415 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/network.md +378 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/storage.md +60 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/types.md +51 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/installation.md +94 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/introduction.md +47 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/neural-net-plugins/train-action-overview.md +815 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-action.md +948 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-overview.md +544 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-template.md +766 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md +1092 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md +117 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/quickstart.md +78 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/troubleshooting.md +519 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current.json +34 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/footer.json +42 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/navbar.json +18 -0
- synapse_sdk/devtools/docs/package-lock.json +18784 -0
- synapse_sdk/devtools/docs/package.json +48 -0
- synapse_sdk/devtools/docs/sidebars.ts +122 -0
- synapse_sdk/devtools/docs/src/components/HomepageFeatures/index.tsx +71 -0
- synapse_sdk/devtools/docs/src/components/HomepageFeatures/styles.module.css +11 -0
- synapse_sdk/devtools/docs/src/css/custom.css +30 -0
- synapse_sdk/devtools/docs/src/pages/index.module.css +23 -0
- synapse_sdk/devtools/docs/src/pages/index.tsx +21 -0
- synapse_sdk/devtools/docs/src/pages/markdown-page.md +7 -0
- synapse_sdk/devtools/docs/static/.nojekyll +0 -0
- synapse_sdk/devtools/docs/static/img/docusaurus-social-card.jpg +0 -0
- synapse_sdk/devtools/docs/static/img/docusaurus.png +0 -0
- synapse_sdk/devtools/docs/static/img/favicon.ico +0 -0
- synapse_sdk/devtools/docs/static/img/logo.png +0 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_mountain.svg +171 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_react.svg +170 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_tree.svg +40 -0
- synapse_sdk/devtools/docs/tsconfig.json +8 -0
- synapse_sdk/devtools/server.py +41 -0
- synapse_sdk/devtools/streamlit_app/__init__.py +5 -0
- synapse_sdk/devtools/streamlit_app/app.py +128 -0
- synapse_sdk/devtools/streamlit_app/services/__init__.py +11 -0
- synapse_sdk/devtools/streamlit_app/services/job_service.py +233 -0
- synapse_sdk/devtools/streamlit_app/services/plugin_service.py +236 -0
- synapse_sdk/devtools/streamlit_app/services/serve_service.py +95 -0
- synapse_sdk/devtools/streamlit_app/ui/__init__.py +15 -0
- synapse_sdk/devtools/streamlit_app/ui/config_tab.py +76 -0
- synapse_sdk/devtools/streamlit_app/ui/deployment_tab.py +66 -0
- synapse_sdk/devtools/streamlit_app/ui/http_tab.py +125 -0
- synapse_sdk/devtools/streamlit_app/ui/jobs_tab.py +573 -0
- synapse_sdk/devtools/streamlit_app/ui/serve_tab.py +346 -0
- synapse_sdk/devtools/streamlit_app/ui/status_bar.py +118 -0
- synapse_sdk/devtools/streamlit_app/utils/__init__.py +40 -0
- synapse_sdk/devtools/streamlit_app/utils/json_viewer.py +197 -0
- synapse_sdk/devtools/streamlit_app/utils/log_formatter.py +38 -0
- synapse_sdk/devtools/streamlit_app/utils/styles.py +241 -0
- synapse_sdk/devtools/streamlit_app/utils/ui_components.py +289 -0
- synapse_sdk/devtools/streamlit_app.py +10 -0
- synapse_sdk/loggers.py +65 -7
- synapse_sdk/plugins/README.md +1340 -0
- synapse_sdk/plugins/categories/base.py +73 -11
- synapse_sdk/plugins/categories/data_validation/actions/validation.py +72 -0
- synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py +33 -5
- synapse_sdk/plugins/categories/export/actions/__init__.py +3 -0
- synapse_sdk/plugins/categories/export/actions/export/__init__.py +28 -0
- synapse_sdk/plugins/categories/export/actions/export/action.py +165 -0
- synapse_sdk/plugins/categories/export/actions/export/enums.py +113 -0
- synapse_sdk/plugins/categories/export/actions/export/exceptions.py +53 -0
- synapse_sdk/plugins/categories/export/actions/export/models.py +74 -0
- synapse_sdk/plugins/categories/export/actions/export/run.py +195 -0
- synapse_sdk/plugins/categories/export/actions/{export.py → export/utils.py} +47 -82
- synapse_sdk/plugins/categories/export/templates/config.yaml +19 -1
- synapse_sdk/plugins/categories/export/templates/plugin/__init__.py +390 -0
- synapse_sdk/plugins/categories/export/templates/plugin/export.py +153 -129
- synapse_sdk/plugins/categories/neural_net/actions/deployment.py +9 -62
- synapse_sdk/plugins/categories/neural_net/actions/train.py +1062 -32
- synapse_sdk/plugins/categories/neural_net/actions/tune.py +534 -0
- synapse_sdk/plugins/categories/neural_net/templates/config.yaml +27 -5
- synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py +26 -10
- synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +4 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/__init__.py +3 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/action.py +10 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/__init__.py +28 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/action.py +145 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/enums.py +269 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/exceptions.py +14 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/factory.py +76 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/models.py +97 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/orchestrator.py +250 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/run.py +64 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/__init__.py +17 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/annotation.py +287 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py +170 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py +83 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py +87 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py +127 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/validation.py +143 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py +966 -0
- synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml +19 -0
- synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py +40 -0
- synapse_sdk/plugins/categories/upload/actions/upload/__init__.py +19 -0
- synapse_sdk/plugins/categories/upload/actions/upload/action.py +232 -0
- synapse_sdk/plugins/categories/upload/actions/upload/context.py +185 -0
- synapse_sdk/plugins/categories/upload/actions/upload/enums.py +471 -0
- synapse_sdk/plugins/categories/upload/actions/upload/exceptions.py +36 -0
- synapse_sdk/plugins/categories/upload/actions/upload/factory.py +138 -0
- synapse_sdk/plugins/categories/upload/actions/upload/models.py +203 -0
- synapse_sdk/plugins/categories/upload/actions/upload/orchestrator.py +183 -0
- synapse_sdk/plugins/categories/upload/actions/upload/registry.py +113 -0
- synapse_sdk/plugins/categories/upload/actions/upload/run.py +179 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py +107 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py +62 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py +63 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py +84 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py +82 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py +235 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py +203 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py +97 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py +71 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py +82 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py +39 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py +29 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py +258 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py +281 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/excel.py +174 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/none.py +16 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/sync.py +84 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py +60 -0
- synapse_sdk/plugins/categories/upload/actions/upload/utils.py +250 -0
- synapse_sdk/plugins/categories/upload/templates/README.md +470 -0
- synapse_sdk/plugins/categories/upload/templates/config.yaml +29 -2
- synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py +294 -0
- synapse_sdk/plugins/categories/upload/templates/plugin/upload.py +88 -30
- synapse_sdk/plugins/models.py +122 -16
- synapse_sdk/plugins/templates/plugin-config-schema.json +406 -0
- synapse_sdk/plugins/templates/schema.json +491 -0
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/requirements.txt +1 -1
- synapse_sdk/plugins/utils/__init__.py +46 -0
- synapse_sdk/plugins/utils/actions.py +119 -0
- synapse_sdk/plugins/utils/config.py +203 -0
- synapse_sdk/plugins/{utils.py → utils/legacy.py} +26 -46
- synapse_sdk/plugins/utils/ray_gcs.py +66 -0
- synapse_sdk/plugins/utils/registry.py +58 -0
- synapse_sdk/shared/__init__.py +25 -0
- synapse_sdk/shared/enums.py +93 -0
- synapse_sdk/utils/converters/__init__.py +240 -0
- synapse_sdk/utils/converters/coco/__init__.py +0 -0
- synapse_sdk/utils/converters/coco/from_dm.py +322 -0
- synapse_sdk/utils/converters/coco/to_dm.py +215 -0
- synapse_sdk/utils/converters/dm/__init__.py +56 -0
- synapse_sdk/utils/converters/dm/from_v1.py +627 -0
- synapse_sdk/utils/converters/dm/to_v1.py +367 -0
- synapse_sdk/utils/converters/pascal/__init__.py +0 -0
- synapse_sdk/utils/converters/pascal/from_dm.py +244 -0
- synapse_sdk/utils/converters/pascal/to_dm.py +214 -0
- synapse_sdk/utils/converters/yolo/__init__.py +0 -0
- synapse_sdk/utils/converters/yolo/from_dm.py +384 -0
- synapse_sdk/utils/converters/yolo/to_dm.py +267 -0
- synapse_sdk/utils/dataset.py +46 -0
- synapse_sdk/utils/encryption.py +158 -0
- synapse_sdk/utils/file/__init__.py +39 -0
- synapse_sdk/utils/file/archive.py +32 -0
- synapse_sdk/utils/file/checksum.py +56 -0
- synapse_sdk/utils/file/chunking.py +31 -0
- synapse_sdk/utils/file/download.py +385 -0
- synapse_sdk/utils/file/encoding.py +40 -0
- synapse_sdk/utils/file/io.py +22 -0
- synapse_sdk/utils/file/video/__init__.py +29 -0
- synapse_sdk/utils/file/video/transcode.py +307 -0
- synapse_sdk/utils/{file.py → file.py.backup} +84 -2
- synapse_sdk/utils/http.py +138 -0
- synapse_sdk/utils/network.py +293 -0
- synapse_sdk/utils/storage/__init__.py +36 -2
- synapse_sdk/utils/storage/providers/__init__.py +141 -0
- synapse_sdk/utils/storage/providers/file_system.py +134 -0
- synapse_sdk/utils/storage/providers/http.py +190 -0
- synapse_sdk/utils/storage/providers/s3.py +54 -6
- synapse_sdk/utils/storage/providers/sftp.py +31 -0
- synapse_sdk/utils/storage/registry.py +6 -0
- synapse_sdk-2025.11.7.dist-info/METADATA +122 -0
- synapse_sdk-2025.11.7.dist-info/RECORD +386 -0
- {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/WHEEL +1 -1
- synapse_sdk/clients/backend/dataset.py +0 -102
- synapse_sdk/plugins/categories/upload/actions/upload.py +0 -293
- synapse_sdk-1.0.0a35.dist-info/METADATA +0 -47
- synapse_sdk-1.0.0a35.dist-info/RECORD +0 -137
- {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/licenses/LICENSE +0 -0
- {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
from synapse_sdk.shared.enums import Context
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class UploadStatus(str, Enum):
|
|
7
|
+
"""Upload processing status enumeration.
|
|
8
|
+
|
|
9
|
+
Defines the possible states for upload operations, data files, and data units
|
|
10
|
+
throughout the upload process.
|
|
11
|
+
|
|
12
|
+
Attributes:
|
|
13
|
+
SUCCESS: Upload completed successfully
|
|
14
|
+
FAILED: Upload failed with errors
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
SUCCESS = 'success'
|
|
18
|
+
FAILED = 'failed'
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class LogCode(str, Enum):
|
|
22
|
+
"""Type-safe logging codes for upload operations.
|
|
23
|
+
|
|
24
|
+
Enumeration of all possible log events during upload processing. Each code
|
|
25
|
+
corresponds to a specific event or error state with predefined message
|
|
26
|
+
templates and log levels.
|
|
27
|
+
|
|
28
|
+
The codes are organized by category:
|
|
29
|
+
- Validation codes (VALIDATION_FAILED, STORAGE_VALIDATION_FAILED, etc.)
|
|
30
|
+
- File processing codes (NO_FILES_FOUND, FILES_DISCOVERED, etc.)
|
|
31
|
+
- Excel processing codes (EXCEL_SECURITY_VIOLATION, EXCEL_PARSING_ERROR, etc.)
|
|
32
|
+
- Progress tracking codes (UPLOADING_DATA_FILES, GENERATING_DATA_UNITS, etc.)
|
|
33
|
+
|
|
34
|
+
Each code maps to a configuration in LOG_MESSAGES with message template
|
|
35
|
+
and appropriate log level.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
STORAGE_VALIDATION_FAILED = 'STORAGE_VALIDATION_FAILED'
|
|
39
|
+
COLLECTION_VALIDATION_FAILED = 'COLLECTION_VALIDATION_FAILED'
|
|
40
|
+
PROJECT_VALIDATION_FAILED = 'PROJECT_VALIDATION_FAILED'
|
|
41
|
+
VALIDATION_FAILED = 'VALIDATION_FAILED'
|
|
42
|
+
NO_FILES_FOUND = 'NO_FILES_FOUND'
|
|
43
|
+
NO_FILES_UPLOADED = 'NO_FILES_UPLOADED'
|
|
44
|
+
NO_DATA_UNITS_GENERATED = 'NO_DATA_UNITS_GENERATED'
|
|
45
|
+
NO_TYPE_DIRECTORIES = 'NO_TYPE_DIRECTORIES'
|
|
46
|
+
EXCEL_SECURITY_VIOLATION = 'EXCEL_SECURITY_VIOLATION'
|
|
47
|
+
EXCEL_PARSING_ERROR = 'EXCEL_PARSING_ERROR'
|
|
48
|
+
EXCEL_METADATA_LOADED = 'EXCEL_METADATA_LOADED'
|
|
49
|
+
UPLOADING_DATA_FILES = 'UPLOADING_DATA_FILES'
|
|
50
|
+
GENERATING_DATA_UNITS = 'GENERATING_DATA_UNITS'
|
|
51
|
+
IMPORT_COMPLETED = 'IMPORT_COMPLETED'
|
|
52
|
+
TYPE_DIRECTORIES_FOUND = 'TYPE_DIRECTORIES_FOUND'
|
|
53
|
+
TYPE_STRUCTURE_DETECTED = 'TYPE_STRUCTURE_DETECTED'
|
|
54
|
+
FILES_DISCOVERED = 'FILES_DISCOVERED'
|
|
55
|
+
NO_FILES_FOUND_WARNING = 'NO_FILES_FOUND_WARNING'
|
|
56
|
+
FILE_UPLOAD_FAILED = 'FILE_UPLOAD_FAILED'
|
|
57
|
+
DATA_UNIT_BATCH_FAILED = 'DATA_UNIT_BATCH_FAILED'
|
|
58
|
+
FILENAME_TOO_LONG = 'FILENAME_TOO_LONG'
|
|
59
|
+
MISSING_REQUIRED_FILES = 'MISSING_REQUIRED_FILES'
|
|
60
|
+
EXCEL_FILE_NOT_FOUND = 'EXCEL_FILE_NOT_FOUND'
|
|
61
|
+
EXCEL_FILE_VALIDATION_STARTED = 'EXCEL_FILE_VALIDATION_STARTED'
|
|
62
|
+
EXCEL_WORKBOOK_LOADED = 'EXCEL_WORKBOOK_LOADED'
|
|
63
|
+
FILE_ORGANIZATION_STARTED = 'FILE_ORGANIZATION_STARTED'
|
|
64
|
+
BATCH_PROCESSING_STARTED = 'BATCH_PROCESSING_STARTED'
|
|
65
|
+
EXCEL_SECURITY_VALIDATION_STARTED = 'EXCEL_SECURITY_VALIDATION_STARTED'
|
|
66
|
+
EXCEL_MEMORY_ESTIMATION = 'EXCEL_MEMORY_ESTIMATION'
|
|
67
|
+
EXCEL_FILE_NOT_FOUND_PATH = 'EXCEL_FILE_NOT_FOUND_PATH'
|
|
68
|
+
EXCEL_SECURITY_VALIDATION_FAILED = 'EXCEL_SECURITY_VALIDATION_FAILED'
|
|
69
|
+
EXCEL_PARSING_FAILED = 'EXCEL_PARSING_FAILED'
|
|
70
|
+
EXCEL_INVALID_FILE_FORMAT = 'EXCEL_INVALID_FILE_FORMAT'
|
|
71
|
+
EXCEL_FILE_TOO_LARGE = 'EXCEL_FILE_TOO_LARGE'
|
|
72
|
+
EXCEL_FILE_ACCESS_ERROR = 'EXCEL_FILE_ACCESS_ERROR'
|
|
73
|
+
EXCEL_UNEXPECTED_ERROR = 'EXCEL_UNEXPECTED_ERROR'
|
|
74
|
+
# Excel path resolution codes (from HEAD)
|
|
75
|
+
EXCEL_PATH_RESOLVED_STORAGE = 'EXCEL_PATH_RESOLVED_STORAGE'
|
|
76
|
+
EXCEL_PATH_RESOLUTION_FAILED = 'EXCEL_PATH_RESOLUTION_FAILED'
|
|
77
|
+
EXCEL_PATH_RESOLUTION_ERROR = 'EXCEL_PATH_RESOLUTION_ERROR'
|
|
78
|
+
# Asset path codes
|
|
79
|
+
ASSET_PATH_ACCESS_ERROR = 'ASSET_PATH_ACCESS_ERROR'
|
|
80
|
+
ASSET_PATH_NOT_FOUND = 'ASSET_PATH_NOT_FOUND'
|
|
81
|
+
# Step lifecycle codes
|
|
82
|
+
STEP_STARTING = 'STEP_STARTING'
|
|
83
|
+
STEP_COMPLETED = 'STEP_COMPLETED'
|
|
84
|
+
STEP_SKIPPED = 'STEP_SKIPPED'
|
|
85
|
+
STEP_ERROR = 'STEP_ERROR'
|
|
86
|
+
# Rollback codes
|
|
87
|
+
ROLLBACK_INITIALIZATION = 'ROLLBACK_INITIALIZATION'
|
|
88
|
+
ROLLBACK_DATA_UNIT_GENERATION = 'ROLLBACK_DATA_UNIT_GENERATION'
|
|
89
|
+
ROLLBACK_FILE_VALIDATION = 'ROLLBACK_FILE_VALIDATION'
|
|
90
|
+
ROLLBACK_FILE_UPLOADS = 'ROLLBACK_FILE_UPLOADS'
|
|
91
|
+
ROLLBACK_COLLECTION_ANALYSIS = 'ROLLBACK_COLLECTION_ANALYSIS'
|
|
92
|
+
ROLLBACK_FILE_ORGANIZATION = 'ROLLBACK_FILE_ORGANIZATION'
|
|
93
|
+
ROLLBACK_CLEANUP = 'ROLLBACK_CLEANUP'
|
|
94
|
+
# Metadata processing codes
|
|
95
|
+
NO_METADATA_STRATEGY = 'NO_METADATA_STRATEGY'
|
|
96
|
+
METADATA_FILE_ATTRIBUTE_PROCESSING = 'METADATA_FILE_ATTRIBUTE_PROCESSING'
|
|
97
|
+
METADATA_TEMP_FILE_CLEANUP = 'METADATA_TEMP_FILE_CLEANUP'
|
|
98
|
+
METADATA_TEMP_FILE_CLEANUP_FAILED = 'METADATA_TEMP_FILE_CLEANUP_FAILED'
|
|
99
|
+
METADATA_BASE64_DECODED = 'METADATA_BASE64_DECODED'
|
|
100
|
+
METADATA_BASE64_DECODE_FAILED = 'METADATA_BASE64_DECODE_FAILED'
|
|
101
|
+
# Multi-path mode codes
|
|
102
|
+
MULTI_PATH_MODE_ENABLED = 'MULTI_PATH_MODE_ENABLED'
|
|
103
|
+
OPTIONAL_SPEC_SKIPPED = 'OPTIONAL_SPEC_SKIPPED'
|
|
104
|
+
DISCOVERING_FILES_FOR_ASSET = 'DISCOVERING_FILES_FOR_ASSET'
|
|
105
|
+
NO_FILES_FOUND_FOR_ASSET = 'NO_FILES_FOUND_FOR_ASSET'
|
|
106
|
+
FILES_FOUND_FOR_ASSET = 'FILES_FOUND_FOR_ASSET'
|
|
107
|
+
ORGANIZING_FILES_MULTI_PATH = 'ORGANIZING_FILES_MULTI_PATH'
|
|
108
|
+
TYPE_DIRECTORIES_MULTI_PATH = 'TYPE_DIRECTORIES_MULTI_PATH'
|
|
109
|
+
DATA_UNITS_CREATED_FROM_FILES = 'DATA_UNITS_CREATED_FROM_FILES'
|
|
110
|
+
# Cleanup codes
|
|
111
|
+
CLEANUP_WARNING = 'CLEANUP_WARNING'
|
|
112
|
+
CLEANUP_TEMP_DIR_SUCCESS = 'CLEANUP_TEMP_DIR_SUCCESS'
|
|
113
|
+
CLEANUP_TEMP_DIR_FAILED = 'CLEANUP_TEMP_DIR_FAILED'
|
|
114
|
+
# Workflow error codes
|
|
115
|
+
UPLOAD_WORKFLOW_FAILED = 'UPLOAD_WORKFLOW_FAILED'
|
|
116
|
+
UNKNOWN_LOG_CODE = 'UNKNOWN_LOG_CODE'
|
|
117
|
+
# Orchestrator workflow codes
|
|
118
|
+
WORKFLOW_STARTING = 'WORKFLOW_STARTING'
|
|
119
|
+
WORKFLOW_COMPLETED = 'WORKFLOW_COMPLETED'
|
|
120
|
+
WORKFLOW_FAILED = 'WORKFLOW_FAILED'
|
|
121
|
+
STEP_FAILED = 'STEP_FAILED'
|
|
122
|
+
STEP_EXCEPTION = 'STEP_EXCEPTION'
|
|
123
|
+
STEP_TRACEBACK = 'STEP_TRACEBACK'
|
|
124
|
+
ROLLBACK_STARTING = 'ROLLBACK_STARTING'
|
|
125
|
+
ROLLBACK_COMPLETED = 'ROLLBACK_COMPLETED'
|
|
126
|
+
STEP_ROLLBACK = 'STEP_ROLLBACK'
|
|
127
|
+
ROLLBACK_ERROR = 'ROLLBACK_ERROR'
|
|
128
|
+
# Extension filtering codes
|
|
129
|
+
FILES_FILTERED_BY_EXTENSION = 'FILES_FILTERED_BY_EXTENSION'
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
LOG_MESSAGES = {
|
|
133
|
+
LogCode.STORAGE_VALIDATION_FAILED: {
|
|
134
|
+
'message': 'Storage validation failed.',
|
|
135
|
+
'level': Context.DANGER,
|
|
136
|
+
},
|
|
137
|
+
LogCode.COLLECTION_VALIDATION_FAILED: {
|
|
138
|
+
'message': 'Collection validation failed.',
|
|
139
|
+
'level': Context.DANGER,
|
|
140
|
+
},
|
|
141
|
+
LogCode.PROJECT_VALIDATION_FAILED: {
|
|
142
|
+
'message': 'Project validation failed.',
|
|
143
|
+
'level': Context.DANGER,
|
|
144
|
+
},
|
|
145
|
+
LogCode.VALIDATION_FAILED: {
|
|
146
|
+
'message': 'Validation failed.',
|
|
147
|
+
'level': Context.DANGER,
|
|
148
|
+
},
|
|
149
|
+
LogCode.NO_FILES_FOUND: {
|
|
150
|
+
'message': 'Files not found on the path.',
|
|
151
|
+
'level': Context.WARNING,
|
|
152
|
+
},
|
|
153
|
+
LogCode.NO_FILES_UPLOADED: {
|
|
154
|
+
'message': 'No files were uploaded.',
|
|
155
|
+
'level': Context.WARNING,
|
|
156
|
+
},
|
|
157
|
+
LogCode.NO_DATA_UNITS_GENERATED: {
|
|
158
|
+
'message': 'No data units were generated.',
|
|
159
|
+
'level': Context.WARNING,
|
|
160
|
+
},
|
|
161
|
+
LogCode.NO_TYPE_DIRECTORIES: {
|
|
162
|
+
'message': 'No type-based directory structure found.',
|
|
163
|
+
'level': Context.INFO,
|
|
164
|
+
},
|
|
165
|
+
LogCode.EXCEL_SECURITY_VIOLATION: {
|
|
166
|
+
'message': 'Excel security validation failed: {}',
|
|
167
|
+
'level': Context.DANGER,
|
|
168
|
+
},
|
|
169
|
+
LogCode.EXCEL_PARSING_ERROR: {
|
|
170
|
+
'message': 'Excel parsing failed: {}',
|
|
171
|
+
'level': Context.DANGER,
|
|
172
|
+
},
|
|
173
|
+
LogCode.EXCEL_METADATA_LOADED: {
|
|
174
|
+
'message': 'Excel metadata loaded for {} files',
|
|
175
|
+
'level': None,
|
|
176
|
+
},
|
|
177
|
+
LogCode.UPLOADING_DATA_FILES: {
|
|
178
|
+
'message': 'Uploading data files...',
|
|
179
|
+
'level': None,
|
|
180
|
+
},
|
|
181
|
+
LogCode.GENERATING_DATA_UNITS: {
|
|
182
|
+
'message': 'Generating data units...',
|
|
183
|
+
'level': None,
|
|
184
|
+
},
|
|
185
|
+
LogCode.IMPORT_COMPLETED: {
|
|
186
|
+
'message': 'Import completed.',
|
|
187
|
+
'level': None,
|
|
188
|
+
},
|
|
189
|
+
LogCode.TYPE_DIRECTORIES_FOUND: {
|
|
190
|
+
'message': 'Found type directories: {}',
|
|
191
|
+
'level': None,
|
|
192
|
+
},
|
|
193
|
+
LogCode.TYPE_STRUCTURE_DETECTED: {
|
|
194
|
+
'message': 'Detected type-based directory structure',
|
|
195
|
+
'level': None,
|
|
196
|
+
},
|
|
197
|
+
LogCode.FILES_DISCOVERED: {
|
|
198
|
+
'message': 'Discovered {} files',
|
|
199
|
+
'level': None,
|
|
200
|
+
},
|
|
201
|
+
LogCode.NO_FILES_FOUND_WARNING: {
|
|
202
|
+
'message': 'No files found.',
|
|
203
|
+
'level': Context.WARNING,
|
|
204
|
+
},
|
|
205
|
+
LogCode.FILE_UPLOAD_FAILED: {
|
|
206
|
+
'message': 'Failed to upload file: {}',
|
|
207
|
+
'level': Context.DANGER,
|
|
208
|
+
},
|
|
209
|
+
LogCode.DATA_UNIT_BATCH_FAILED: {
|
|
210
|
+
'message': 'Failed to create data units batch: {}',
|
|
211
|
+
'level': Context.DANGER,
|
|
212
|
+
},
|
|
213
|
+
LogCode.FILENAME_TOO_LONG: {
|
|
214
|
+
'message': 'Skipping file with overly long name: {}...',
|
|
215
|
+
'level': Context.WARNING,
|
|
216
|
+
},
|
|
217
|
+
LogCode.MISSING_REQUIRED_FILES: {
|
|
218
|
+
'message': '{} missing required files: {}',
|
|
219
|
+
'level': Context.WARNING,
|
|
220
|
+
},
|
|
221
|
+
LogCode.EXCEL_FILE_NOT_FOUND: {
|
|
222
|
+
'message': 'Excel metadata file not found: {}',
|
|
223
|
+
'level': Context.WARNING,
|
|
224
|
+
},
|
|
225
|
+
LogCode.EXCEL_FILE_VALIDATION_STARTED: {
|
|
226
|
+
'message': 'Excel file validation started',
|
|
227
|
+
'level': Context.INFO,
|
|
228
|
+
},
|
|
229
|
+
LogCode.EXCEL_WORKBOOK_LOADED: {
|
|
230
|
+
'message': 'Excel workbook loaded successfully',
|
|
231
|
+
'level': Context.INFO,
|
|
232
|
+
},
|
|
233
|
+
LogCode.FILE_ORGANIZATION_STARTED: {
|
|
234
|
+
'message': 'File organization started',
|
|
235
|
+
'level': Context.INFO,
|
|
236
|
+
},
|
|
237
|
+
LogCode.BATCH_PROCESSING_STARTED: {
|
|
238
|
+
'message': 'Batch processing started: {} batches of {} items each',
|
|
239
|
+
'level': Context.INFO,
|
|
240
|
+
},
|
|
241
|
+
LogCode.EXCEL_SECURITY_VALIDATION_STARTED: {
|
|
242
|
+
'message': 'Excel security validation started for file size: {} bytes',
|
|
243
|
+
'level': Context.INFO,
|
|
244
|
+
},
|
|
245
|
+
LogCode.EXCEL_MEMORY_ESTIMATION: {
|
|
246
|
+
'message': 'Excel memory estimation: {} bytes (file) * 3 = {} bytes (estimated)',
|
|
247
|
+
'level': Context.INFO,
|
|
248
|
+
},
|
|
249
|
+
LogCode.EXCEL_FILE_NOT_FOUND_PATH: {
|
|
250
|
+
'message': 'Excel metadata file not found',
|
|
251
|
+
'level': Context.WARNING,
|
|
252
|
+
},
|
|
253
|
+
LogCode.EXCEL_SECURITY_VALIDATION_FAILED: {
|
|
254
|
+
'message': 'Excel security validation failed: {}',
|
|
255
|
+
'level': Context.DANGER,
|
|
256
|
+
},
|
|
257
|
+
LogCode.EXCEL_PARSING_FAILED: {
|
|
258
|
+
'message': 'Excel parsing failed: {}',
|
|
259
|
+
'level': Context.DANGER,
|
|
260
|
+
},
|
|
261
|
+
LogCode.EXCEL_INVALID_FILE_FORMAT: {
|
|
262
|
+
'message': 'Invalid Excel file format: {}',
|
|
263
|
+
'level': Context.DANGER,
|
|
264
|
+
},
|
|
265
|
+
LogCode.EXCEL_FILE_TOO_LARGE: {
|
|
266
|
+
'message': 'Excel file too large to process (memory limit exceeded)',
|
|
267
|
+
'level': Context.DANGER,
|
|
268
|
+
},
|
|
269
|
+
LogCode.EXCEL_FILE_ACCESS_ERROR: {
|
|
270
|
+
'message': 'File access error reading excel metadata: {}',
|
|
271
|
+
'level': Context.DANGER,
|
|
272
|
+
},
|
|
273
|
+
LogCode.EXCEL_UNEXPECTED_ERROR: {
|
|
274
|
+
'message': 'Unexpected error reading excel metadata: {}',
|
|
275
|
+
'level': Context.DANGER,
|
|
276
|
+
},
|
|
277
|
+
# Excel path resolution messages (from HEAD)
|
|
278
|
+
LogCode.EXCEL_PATH_RESOLVED_STORAGE: {
|
|
279
|
+
'message': 'Resolved Excel metadata path relative to storage: {}',
|
|
280
|
+
'level': Context.INFO,
|
|
281
|
+
},
|
|
282
|
+
LogCode.EXCEL_PATH_RESOLUTION_FAILED: {
|
|
283
|
+
'message': 'Storage path resolution failed ({}): {} - trying other strategies',
|
|
284
|
+
'level': Context.INFO,
|
|
285
|
+
},
|
|
286
|
+
LogCode.EXCEL_PATH_RESOLUTION_ERROR: {
|
|
287
|
+
'message': 'Unexpected error resolving storage path ({}): {} - trying other strategies',
|
|
288
|
+
'level': Context.WARNING,
|
|
289
|
+
},
|
|
290
|
+
# Asset path messages
|
|
291
|
+
LogCode.ASSET_PATH_ACCESS_ERROR: {
|
|
292
|
+
'message': 'Error accessing path for {}: {}',
|
|
293
|
+
'level': Context.WARNING,
|
|
294
|
+
},
|
|
295
|
+
LogCode.ASSET_PATH_NOT_FOUND: {
|
|
296
|
+
'message': 'Path does not exist for {}: {}',
|
|
297
|
+
'level': Context.WARNING,
|
|
298
|
+
},
|
|
299
|
+
# Step lifecycle messages
|
|
300
|
+
LogCode.STEP_STARTING: {
|
|
301
|
+
'message': 'Starting step: {}',
|
|
302
|
+
'level': Context.INFO,
|
|
303
|
+
},
|
|
304
|
+
LogCode.STEP_COMPLETED: {
|
|
305
|
+
'message': 'Completed step: {}',
|
|
306
|
+
'level': Context.INFO,
|
|
307
|
+
},
|
|
308
|
+
LogCode.STEP_SKIPPED: {
|
|
309
|
+
'message': 'Skipped step: {}',
|
|
310
|
+
'level': Context.INFO,
|
|
311
|
+
},
|
|
312
|
+
LogCode.STEP_ERROR: {
|
|
313
|
+
'message': 'Error in step {}: {}',
|
|
314
|
+
'level': Context.DANGER,
|
|
315
|
+
},
|
|
316
|
+
# Rollback messages
|
|
317
|
+
LogCode.ROLLBACK_INITIALIZATION: {
|
|
318
|
+
'message': 'Rolling back initialization step',
|
|
319
|
+
'level': Context.INFO,
|
|
320
|
+
},
|
|
321
|
+
LogCode.ROLLBACK_DATA_UNIT_GENERATION: {
|
|
322
|
+
'message': 'Rolled back data unit generation',
|
|
323
|
+
'level': Context.INFO,
|
|
324
|
+
},
|
|
325
|
+
LogCode.ROLLBACK_FILE_VALIDATION: {
|
|
326
|
+
'message': 'Rolled back file validation',
|
|
327
|
+
'level': Context.INFO,
|
|
328
|
+
},
|
|
329
|
+
LogCode.ROLLBACK_FILE_UPLOADS: {
|
|
330
|
+
'message': 'Rolled back file uploads',
|
|
331
|
+
'level': Context.INFO,
|
|
332
|
+
},
|
|
333
|
+
LogCode.ROLLBACK_COLLECTION_ANALYSIS: {
|
|
334
|
+
'message': 'Rolled back collection analysis',
|
|
335
|
+
'level': Context.INFO,
|
|
336
|
+
},
|
|
337
|
+
LogCode.ROLLBACK_FILE_ORGANIZATION: {
|
|
338
|
+
'message': 'Rolled back file organization',
|
|
339
|
+
'level': Context.INFO,
|
|
340
|
+
},
|
|
341
|
+
LogCode.ROLLBACK_CLEANUP: {
|
|
342
|
+
'message': 'Cleanup step rollback - no action needed',
|
|
343
|
+
'level': Context.INFO,
|
|
344
|
+
},
|
|
345
|
+
# Metadata processing messages
|
|
346
|
+
LogCode.NO_METADATA_STRATEGY: {
|
|
347
|
+
'message': 'No metadata strategy configured - skipping metadata processing',
|
|
348
|
+
'level': Context.INFO,
|
|
349
|
+
},
|
|
350
|
+
LogCode.METADATA_FILE_ATTRIBUTE_PROCESSING: {
|
|
351
|
+
'message': 'Processing metadata for file attribute: {}',
|
|
352
|
+
'level': Context.INFO,
|
|
353
|
+
},
|
|
354
|
+
LogCode.METADATA_TEMP_FILE_CLEANUP: {
|
|
355
|
+
'message': 'Cleaned up temporary Excel file: {}',
|
|
356
|
+
'level': Context.INFO,
|
|
357
|
+
},
|
|
358
|
+
LogCode.METADATA_TEMP_FILE_CLEANUP_FAILED: {
|
|
359
|
+
'message': 'Failed to clean up temporary file {}: {}',
|
|
360
|
+
'level': Context.WARNING,
|
|
361
|
+
},
|
|
362
|
+
LogCode.METADATA_BASE64_DECODED: {
|
|
363
|
+
'message': 'Decoded base64 Excel metadata to temporary file: {}',
|
|
364
|
+
'level': Context.INFO,
|
|
365
|
+
},
|
|
366
|
+
LogCode.METADATA_BASE64_DECODE_FAILED: {
|
|
367
|
+
'message': 'Failed to decode base64 Excel metadata: {}',
|
|
368
|
+
'level': Context.DANGER,
|
|
369
|
+
},
|
|
370
|
+
# Multi-path mode messages
|
|
371
|
+
LogCode.MULTI_PATH_MODE_ENABLED: {
|
|
372
|
+
'message': 'Using multi-path mode with {} asset configurations',
|
|
373
|
+
'level': Context.INFO,
|
|
374
|
+
},
|
|
375
|
+
LogCode.OPTIONAL_SPEC_SKIPPED: {
|
|
376
|
+
'message': 'Skipping optional spec {}: no asset path configured',
|
|
377
|
+
'level': Context.INFO,
|
|
378
|
+
},
|
|
379
|
+
LogCode.DISCOVERING_FILES_FOR_ASSET: {
|
|
380
|
+
'message': 'Discovering files for {} at {} (recursive={})',
|
|
381
|
+
'level': Context.INFO,
|
|
382
|
+
},
|
|
383
|
+
LogCode.NO_FILES_FOUND_FOR_ASSET: {
|
|
384
|
+
'message': 'No files found for {}',
|
|
385
|
+
'level': Context.WARNING,
|
|
386
|
+
},
|
|
387
|
+
LogCode.FILES_FOUND_FOR_ASSET: {
|
|
388
|
+
'message': 'Found {} files for {}',
|
|
389
|
+
'level': Context.INFO,
|
|
390
|
+
},
|
|
391
|
+
LogCode.ORGANIZING_FILES_MULTI_PATH: {
|
|
392
|
+
'message': 'Organizing {} files across {} specs',
|
|
393
|
+
'level': Context.INFO,
|
|
394
|
+
},
|
|
395
|
+
LogCode.TYPE_DIRECTORIES_MULTI_PATH: {
|
|
396
|
+
'message': 'Type directories: {}',
|
|
397
|
+
'level': Context.INFO,
|
|
398
|
+
},
|
|
399
|
+
LogCode.DATA_UNITS_CREATED_FROM_FILES: {
|
|
400
|
+
'message': 'Created {} data units from {} files',
|
|
401
|
+
'level': Context.INFO,
|
|
402
|
+
},
|
|
403
|
+
# Cleanup messages
|
|
404
|
+
LogCode.CLEANUP_WARNING: {
|
|
405
|
+
'message': 'Cleanup warning: {}',
|
|
406
|
+
'level': Context.WARNING,
|
|
407
|
+
},
|
|
408
|
+
LogCode.CLEANUP_TEMP_DIR_SUCCESS: {
|
|
409
|
+
'message': 'Cleaned up temporary directory: {}',
|
|
410
|
+
'level': Context.INFO,
|
|
411
|
+
},
|
|
412
|
+
LogCode.CLEANUP_TEMP_DIR_FAILED: {
|
|
413
|
+
'message': 'Failed to cleanup temporary directory: {}',
|
|
414
|
+
'level': Context.WARNING,
|
|
415
|
+
},
|
|
416
|
+
# Workflow error messages
|
|
417
|
+
LogCode.UPLOAD_WORKFLOW_FAILED: {
|
|
418
|
+
'message': 'Upload workflow failed: {}',
|
|
419
|
+
'level': Context.DANGER,
|
|
420
|
+
},
|
|
421
|
+
LogCode.UNKNOWN_LOG_CODE: {
|
|
422
|
+
'message': 'Unknown log code: {}',
|
|
423
|
+
'level': Context.WARNING,
|
|
424
|
+
},
|
|
425
|
+
# Orchestrator workflow messages
|
|
426
|
+
LogCode.WORKFLOW_STARTING: {
|
|
427
|
+
'message': 'Starting upload workflow with {} steps: {}',
|
|
428
|
+
'level': Context.INFO,
|
|
429
|
+
},
|
|
430
|
+
LogCode.WORKFLOW_COMPLETED: {
|
|
431
|
+
'message': 'Upload workflow completed successfully',
|
|
432
|
+
'level': Context.INFO,
|
|
433
|
+
},
|
|
434
|
+
LogCode.WORKFLOW_FAILED: {
|
|
435
|
+
'message': 'Upload workflow failed: {}',
|
|
436
|
+
'level': Context.DANGER,
|
|
437
|
+
},
|
|
438
|
+
LogCode.STEP_FAILED: {
|
|
439
|
+
'message': "Step '{}' failed: {}",
|
|
440
|
+
'level': Context.DANGER,
|
|
441
|
+
},
|
|
442
|
+
LogCode.STEP_EXCEPTION: {
|
|
443
|
+
'message': "Exception in step '{}': {}",
|
|
444
|
+
'level': Context.DANGER,
|
|
445
|
+
},
|
|
446
|
+
LogCode.STEP_TRACEBACK: {
|
|
447
|
+
'message': 'Traceback: {}',
|
|
448
|
+
'level': Context.DANGER,
|
|
449
|
+
},
|
|
450
|
+
LogCode.ROLLBACK_STARTING: {
|
|
451
|
+
'message': 'Starting rollback of {} executed steps',
|
|
452
|
+
'level': Context.WARNING,
|
|
453
|
+
},
|
|
454
|
+
LogCode.ROLLBACK_COMPLETED: {
|
|
455
|
+
'message': 'Rollback completed',
|
|
456
|
+
'level': Context.INFO,
|
|
457
|
+
},
|
|
458
|
+
LogCode.STEP_ROLLBACK: {
|
|
459
|
+
'message': 'Rolling back step: {}',
|
|
460
|
+
'level': Context.INFO,
|
|
461
|
+
},
|
|
462
|
+
LogCode.ROLLBACK_ERROR: {
|
|
463
|
+
'message': "Error rolling back step '{}': {}",
|
|
464
|
+
'level': Context.WARNING,
|
|
465
|
+
},
|
|
466
|
+
# Extension filtering messages
|
|
467
|
+
LogCode.FILES_FILTERED_BY_EXTENSION: {
|
|
468
|
+
'message': 'Filtered {} {} files with unavailable extensions: {} (allowed: {})',
|
|
469
|
+
'level': Context.WARNING,
|
|
470
|
+
},
|
|
471
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
class ExcelSecurityError(Exception):
|
|
2
|
+
"""Exception raised when Excel file security validation fails.
|
|
3
|
+
|
|
4
|
+
This exception is raised when an Excel file violates security constraints
|
|
5
|
+
such as file size limits, memory usage limits, or contains potentially
|
|
6
|
+
dangerous content.
|
|
7
|
+
|
|
8
|
+
Used during Excel metadata processing to enforce security policies
|
|
9
|
+
and prevent processing of files that could pose security risks.
|
|
10
|
+
|
|
11
|
+
Example:
|
|
12
|
+
>>> if file_size > max_size:
|
|
13
|
+
... raise ExcelSecurityError(f"File size {file_size} exceeds limit {max_size}")
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ExcelParsingError(Exception):
|
|
20
|
+
"""Exception raised when Excel file parsing encounters errors.
|
|
21
|
+
|
|
22
|
+
This exception is raised when an Excel file cannot be parsed due to
|
|
23
|
+
format issues, corruption, or other parsing-related problems that
|
|
24
|
+
prevent successful metadata extraction.
|
|
25
|
+
|
|
26
|
+
Used during Excel metadata loading to distinguish parsing errors
|
|
27
|
+
from security violations or other types of errors.
|
|
28
|
+
|
|
29
|
+
Example:
|
|
30
|
+
>>> try:
|
|
31
|
+
... workbook = load_workbook(excel_file)
|
|
32
|
+
... except InvalidFileException as e:
|
|
33
|
+
... raise ExcelParsingError(f"Failed to parse Excel file: {e}")
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
pass
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
from typing import Any, Dict
|
|
2
|
+
|
|
3
|
+
from .strategies.base import (
|
|
4
|
+
DataUnitStrategy,
|
|
5
|
+
FileDiscoveryStrategy,
|
|
6
|
+
MetadataStrategy,
|
|
7
|
+
UploadStrategy,
|
|
8
|
+
ValidationStrategy,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class StrategyFactory:
|
|
13
|
+
"""Factory for creating strategy instances based on configuration."""
|
|
14
|
+
|
|
15
|
+
def __init__(self):
|
|
16
|
+
self._validation_strategies = {}
|
|
17
|
+
self._file_discovery_strategies = {}
|
|
18
|
+
self._metadata_strategies = {}
|
|
19
|
+
self._upload_strategies = {}
|
|
20
|
+
self._data_unit_strategies = {}
|
|
21
|
+
|
|
22
|
+
def register_validation_strategy(self, name: str, strategy_class: type) -> None:
|
|
23
|
+
"""Register a validation strategy class."""
|
|
24
|
+
self._validation_strategies[name] = strategy_class
|
|
25
|
+
|
|
26
|
+
def register_file_discovery_strategy(self, name: str, strategy_class: type) -> None:
|
|
27
|
+
"""Register a file discovery strategy class."""
|
|
28
|
+
self._file_discovery_strategies[name] = strategy_class
|
|
29
|
+
|
|
30
|
+
def register_metadata_strategy(self, name: str, strategy_class: type) -> None:
|
|
31
|
+
"""Register a metadata strategy class."""
|
|
32
|
+
self._metadata_strategies[name] = strategy_class
|
|
33
|
+
|
|
34
|
+
def register_upload_strategy(self, name: str, strategy_class: type) -> None:
|
|
35
|
+
"""Register an upload strategy class."""
|
|
36
|
+
self._upload_strategies[name] = strategy_class
|
|
37
|
+
|
|
38
|
+
def register_data_unit_strategy(self, name: str, strategy_class: type) -> None:
|
|
39
|
+
"""Register a data unit strategy class."""
|
|
40
|
+
self._data_unit_strategies[name] = strategy_class
|
|
41
|
+
|
|
42
|
+
def create_validation_strategy(self, params: Dict[str, Any], context=None) -> ValidationStrategy:
|
|
43
|
+
"""Create validation strategy based on parameters."""
|
|
44
|
+
strategy_name = params.get('validation_strategy', 'default')
|
|
45
|
+
|
|
46
|
+
if strategy_name not in self._validation_strategies:
|
|
47
|
+
# Import default strategy if not registered
|
|
48
|
+
from .strategies.validation.default import DefaultValidationStrategy
|
|
49
|
+
|
|
50
|
+
self.register_validation_strategy('default', DefaultValidationStrategy)
|
|
51
|
+
strategy_name = 'default'
|
|
52
|
+
|
|
53
|
+
strategy_class = self._validation_strategies[strategy_name]
|
|
54
|
+
return strategy_class()
|
|
55
|
+
|
|
56
|
+
def create_file_discovery_strategy(self, params: Dict[str, Any], context=None) -> FileDiscoveryStrategy:
|
|
57
|
+
"""Create file discovery strategy based on parameters."""
|
|
58
|
+
is_recursive = params.get('is_recursive', True)
|
|
59
|
+
strategy_name = 'recursive' if is_recursive else 'flat'
|
|
60
|
+
|
|
61
|
+
if strategy_name not in self._file_discovery_strategies:
|
|
62
|
+
# Import default strategies if not registered
|
|
63
|
+
if strategy_name == 'recursive':
|
|
64
|
+
from .strategies.file_discovery.recursive import RecursiveFileDiscoveryStrategy
|
|
65
|
+
|
|
66
|
+
self.register_file_discovery_strategy('recursive', RecursiveFileDiscoveryStrategy)
|
|
67
|
+
else:
|
|
68
|
+
from .strategies.file_discovery.flat import FlatFileDiscoveryStrategy
|
|
69
|
+
|
|
70
|
+
self.register_file_discovery_strategy('flat', FlatFileDiscoveryStrategy)
|
|
71
|
+
|
|
72
|
+
strategy_class = self._file_discovery_strategies[strategy_name]
|
|
73
|
+
return strategy_class()
|
|
74
|
+
|
|
75
|
+
def create_metadata_strategy(self, params: Dict[str, Any], context=None) -> MetadataStrategy:
|
|
76
|
+
"""Create metadata strategy based on parameters."""
|
|
77
|
+
# Always use Excel strategy for metadata processing
|
|
78
|
+
# It will handle both specified paths and default meta.xlsx/meta.xls files
|
|
79
|
+
strategy_name = 'excel'
|
|
80
|
+
|
|
81
|
+
if strategy_name not in self._metadata_strategies:
|
|
82
|
+
from .strategies.metadata.excel import ExcelMetadataStrategy
|
|
83
|
+
|
|
84
|
+
self.register_metadata_strategy('excel', ExcelMetadataStrategy)
|
|
85
|
+
|
|
86
|
+
strategy_class = self._metadata_strategies[strategy_name]
|
|
87
|
+
return strategy_class()
|
|
88
|
+
|
|
89
|
+
def create_upload_strategy(self, params: Dict[str, Any], context=None) -> UploadStrategy:
|
|
90
|
+
"""Create upload strategy (always uses synchronous upload for guaranteed ordering)."""
|
|
91
|
+
if context is None:
|
|
92
|
+
raise ValueError('Upload strategies require context parameter')
|
|
93
|
+
|
|
94
|
+
# Always use sync upload to guarantee file order
|
|
95
|
+
# This is critical for video frame extraction and PDF page extraction
|
|
96
|
+
strategy_name = 'sync'
|
|
97
|
+
|
|
98
|
+
if strategy_name not in self._upload_strategies:
|
|
99
|
+
from .strategies.upload.sync import SyncUploadStrategy
|
|
100
|
+
|
|
101
|
+
self.register_upload_strategy('sync', SyncUploadStrategy)
|
|
102
|
+
|
|
103
|
+
strategy_class = self._upload_strategies[strategy_name]
|
|
104
|
+
# Upload strategies always need context for client access
|
|
105
|
+
return strategy_class(context)
|
|
106
|
+
|
|
107
|
+
def create_data_unit_strategy(self, params: Dict[str, Any], context=None) -> DataUnitStrategy:
|
|
108
|
+
"""Create data unit strategy based on parameters."""
|
|
109
|
+
if context is None:
|
|
110
|
+
raise ValueError('Data unit strategies require context parameter')
|
|
111
|
+
|
|
112
|
+
batch_size = params.get('creating_data_unit_batch_size', 1)
|
|
113
|
+
strategy_name = 'batch' if batch_size > 1 else 'single'
|
|
114
|
+
|
|
115
|
+
if strategy_name not in self._data_unit_strategies:
|
|
116
|
+
# Import default strategies if not registered
|
|
117
|
+
if strategy_name == 'batch':
|
|
118
|
+
from .strategies.data_unit.batch import BatchDataUnitStrategy
|
|
119
|
+
|
|
120
|
+
self.register_data_unit_strategy('batch', BatchDataUnitStrategy)
|
|
121
|
+
else:
|
|
122
|
+
from .strategies.data_unit.single import SingleDataUnitStrategy
|
|
123
|
+
|
|
124
|
+
self.register_data_unit_strategy('single', SingleDataUnitStrategy)
|
|
125
|
+
|
|
126
|
+
strategy_class = self._data_unit_strategies[strategy_name]
|
|
127
|
+
# Data unit strategies always need context for client access
|
|
128
|
+
return strategy_class(context)
|
|
129
|
+
|
|
130
|
+
def get_available_strategies(self) -> Dict[str, list]:
|
|
131
|
+
"""Get all available strategy types and their registered names."""
|
|
132
|
+
return {
|
|
133
|
+
'validation': list(self._validation_strategies.keys()),
|
|
134
|
+
'file_discovery': list(self._file_discovery_strategies.keys()),
|
|
135
|
+
'metadata': list(self._metadata_strategies.keys()),
|
|
136
|
+
'upload': list(self._upload_strategies.keys()),
|
|
137
|
+
'data_unit': list(self._data_unit_strategies.keys()),
|
|
138
|
+
}
|