synapse-sdk 1.0.0a13__py3-none-any.whl → 2025.11.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/__init__.py +24 -0
- synapse_sdk/cli/__init__.py +310 -5
- synapse_sdk/cli/alias/__init__.py +22 -0
- synapse_sdk/cli/alias/create.py +36 -0
- synapse_sdk/cli/alias/dataclass.py +31 -0
- synapse_sdk/cli/alias/default.py +16 -0
- synapse_sdk/cli/alias/delete.py +15 -0
- synapse_sdk/cli/alias/list.py +19 -0
- synapse_sdk/cli/alias/read.py +15 -0
- synapse_sdk/cli/alias/update.py +17 -0
- synapse_sdk/cli/alias/utils.py +61 -0
- synapse_sdk/cli/code_server.py +687 -0
- synapse_sdk/cli/config.py +440 -0
- synapse_sdk/cli/devtools.py +90 -0
- synapse_sdk/cli/plugin/__init__.py +33 -0
- synapse_sdk/cli/{create_plugin.py → plugin/create.py} +2 -2
- synapse_sdk/cli/plugin/publish.py +45 -0
- synapse_sdk/{plugins/cli → cli/plugin}/run.py +12 -5
- synapse_sdk/clients/agent/__init__.py +9 -3
- synapse_sdk/clients/agent/container.py +133 -0
- synapse_sdk/clients/agent/core.py +19 -0
- synapse_sdk/clients/agent/ray.py +298 -9
- synapse_sdk/clients/backend/__init__.py +41 -12
- synapse_sdk/clients/backend/annotation.py +13 -5
- synapse_sdk/clients/backend/core.py +59 -0
- synapse_sdk/clients/backend/data_collection.py +186 -0
- synapse_sdk/clients/backend/hitl.py +17 -0
- synapse_sdk/clients/backend/integration.py +19 -4
- synapse_sdk/clients/backend/ml.py +10 -7
- synapse_sdk/clients/backend/models.py +78 -0
- synapse_sdk/clients/base.py +381 -34
- synapse_sdk/clients/ray/serve.py +2 -0
- synapse_sdk/clients/validators/collections.py +31 -0
- synapse_sdk/devtools/config.py +94 -0
- synapse_sdk/devtools/docs/.gitignore +20 -0
- synapse_sdk/devtools/docs/README.md +41 -0
- synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md +12 -0
- synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md +44 -0
- synapse_sdk/devtools/docs/blog/2021-08-01-mdx-blog-post.mdx +24 -0
- synapse_sdk/devtools/docs/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg +0 -0
- synapse_sdk/devtools/docs/blog/2021-08-26-welcome/index.md +29 -0
- synapse_sdk/devtools/docs/blog/authors.yml +25 -0
- synapse_sdk/devtools/docs/blog/tags.yml +19 -0
- synapse_sdk/devtools/docs/docs/api/clients/agent.md +43 -0
- synapse_sdk/devtools/docs/docs/api/clients/annotation-mixin.md +378 -0
- synapse_sdk/devtools/docs/docs/api/clients/backend.md +420 -0
- synapse_sdk/devtools/docs/docs/api/clients/base.md +257 -0
- synapse_sdk/devtools/docs/docs/api/clients/core-mixin.md +477 -0
- synapse_sdk/devtools/docs/docs/api/clients/data-collection-mixin.md +422 -0
- synapse_sdk/devtools/docs/docs/api/clients/hitl-mixin.md +554 -0
- synapse_sdk/devtools/docs/docs/api/clients/index.md +391 -0
- synapse_sdk/devtools/docs/docs/api/clients/integration-mixin.md +571 -0
- synapse_sdk/devtools/docs/docs/api/clients/ml-mixin.md +578 -0
- synapse_sdk/devtools/docs/docs/api/clients/ray.md +342 -0
- synapse_sdk/devtools/docs/docs/api/index.md +52 -0
- synapse_sdk/devtools/docs/docs/api/plugins/categories.md +43 -0
- synapse_sdk/devtools/docs/docs/api/plugins/models.md +114 -0
- synapse_sdk/devtools/docs/docs/api/plugins/utils.md +328 -0
- synapse_sdk/devtools/docs/docs/categories.md +0 -0
- synapse_sdk/devtools/docs/docs/cli-usage.md +280 -0
- synapse_sdk/devtools/docs/docs/concepts/index.md +38 -0
- synapse_sdk/devtools/docs/docs/configuration.md +83 -0
- synapse_sdk/devtools/docs/docs/contributing.md +306 -0
- synapse_sdk/devtools/docs/docs/examples/index.md +29 -0
- synapse_sdk/devtools/docs/docs/faq.md +179 -0
- synapse_sdk/devtools/docs/docs/features/converters/index.md +455 -0
- synapse_sdk/devtools/docs/docs/features/index.md +24 -0
- synapse_sdk/devtools/docs/docs/features/utils/file.md +415 -0
- synapse_sdk/devtools/docs/docs/features/utils/network.md +378 -0
- synapse_sdk/devtools/docs/docs/features/utils/storage.md +57 -0
- synapse_sdk/devtools/docs/docs/features/utils/types.md +51 -0
- synapse_sdk/devtools/docs/docs/installation.md +94 -0
- synapse_sdk/devtools/docs/docs/introduction.md +47 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/neural-net-plugins/train-action-overview.md +814 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-action.md +948 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-overview.md +544 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-template.md +766 -0
- synapse_sdk/devtools/docs/docs/plugins/export-plugins.md +1092 -0
- synapse_sdk/devtools/docs/docs/plugins/plugins.md +852 -0
- synapse_sdk/devtools/docs/docs/quickstart.md +78 -0
- synapse_sdk/devtools/docs/docs/troubleshooting.md +519 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/_category_.json +8 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/congratulations.md +23 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-blog-post.md +34 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-document.md +57 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-page.md +43 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/deploy-your-site.md +31 -0
- synapse_sdk/devtools/docs/docs/tutorial-basics/markdown-features.mdx +152 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/_category_.json +7 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/img/docsVersionDropdown.png +0 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/img/localeDropdown.png +0 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/manage-docs-versions.md +55 -0
- synapse_sdk/devtools/docs/docs/tutorial-extras/translate-your-site.md +88 -0
- synapse_sdk/devtools/docs/docusaurus.config.ts +148 -0
- synapse_sdk/devtools/docs/i18n/ko/code.json +325 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/agent.md +43 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/annotation-mixin.md +289 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/backend.md +420 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/base.md +257 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/core-mixin.md +417 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/data-collection-mixin.md +356 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/hitl-mixin.md +192 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/index.md +391 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/integration-mixin.md +479 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ml-mixin.md +284 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ray.md +342 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/index.md +52 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/plugins/models.md +114 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/categories.md +0 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/cli-usage.md +280 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/concepts/index.md +38 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/configuration.md +83 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/contributing.md +306 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/examples/index.md +29 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/faq.md +179 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/converters/index.md +30 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/index.md +24 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/file.md +415 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/network.md +378 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/storage.md +60 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/types.md +51 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/installation.md +94 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/introduction.md +47 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/neural-net-plugins/train-action-overview.md +815 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-action.md +948 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-overview.md +544 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-template.md +766 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md +1092 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md +117 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/quickstart.md +78 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/troubleshooting.md +519 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current.json +34 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/footer.json +42 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/navbar.json +18 -0
- synapse_sdk/devtools/docs/package-lock.json +18784 -0
- synapse_sdk/devtools/docs/package.json +48 -0
- synapse_sdk/devtools/docs/sidebars.ts +122 -0
- synapse_sdk/devtools/docs/src/components/HomepageFeatures/index.tsx +71 -0
- synapse_sdk/devtools/docs/src/components/HomepageFeatures/styles.module.css +11 -0
- synapse_sdk/devtools/docs/src/css/custom.css +30 -0
- synapse_sdk/devtools/docs/src/pages/index.module.css +23 -0
- synapse_sdk/devtools/docs/src/pages/index.tsx +21 -0
- synapse_sdk/devtools/docs/src/pages/markdown-page.md +7 -0
- synapse_sdk/devtools/docs/static/.nojekyll +0 -0
- synapse_sdk/devtools/docs/static/img/docusaurus-social-card.jpg +0 -0
- synapse_sdk/devtools/docs/static/img/docusaurus.png +0 -0
- synapse_sdk/devtools/docs/static/img/favicon.ico +0 -0
- synapse_sdk/devtools/docs/static/img/logo.png +0 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_mountain.svg +171 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_react.svg +170 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_tree.svg +40 -0
- synapse_sdk/devtools/docs/tsconfig.json +8 -0
- synapse_sdk/devtools/server.py +41 -0
- synapse_sdk/devtools/streamlit_app/__init__.py +5 -0
- synapse_sdk/devtools/streamlit_app/app.py +128 -0
- synapse_sdk/devtools/streamlit_app/services/__init__.py +11 -0
- synapse_sdk/devtools/streamlit_app/services/job_service.py +233 -0
- synapse_sdk/devtools/streamlit_app/services/plugin_service.py +236 -0
- synapse_sdk/devtools/streamlit_app/services/serve_service.py +95 -0
- synapse_sdk/devtools/streamlit_app/ui/__init__.py +15 -0
- synapse_sdk/devtools/streamlit_app/ui/config_tab.py +76 -0
- synapse_sdk/devtools/streamlit_app/ui/deployment_tab.py +66 -0
- synapse_sdk/devtools/streamlit_app/ui/http_tab.py +125 -0
- synapse_sdk/devtools/streamlit_app/ui/jobs_tab.py +573 -0
- synapse_sdk/devtools/streamlit_app/ui/serve_tab.py +346 -0
- synapse_sdk/devtools/streamlit_app/ui/status_bar.py +118 -0
- synapse_sdk/devtools/streamlit_app/utils/__init__.py +40 -0
- synapse_sdk/devtools/streamlit_app/utils/json_viewer.py +197 -0
- synapse_sdk/devtools/streamlit_app/utils/log_formatter.py +38 -0
- synapse_sdk/devtools/streamlit_app/utils/styles.py +241 -0
- synapse_sdk/devtools/streamlit_app/utils/ui_components.py +289 -0
- synapse_sdk/devtools/streamlit_app.py +10 -0
- synapse_sdk/loggers.py +74 -9
- synapse_sdk/plugins/README.md +1340 -0
- synapse_sdk/plugins/__init__.py +0 -13
- synapse_sdk/plugins/categories/base.py +145 -30
- synapse_sdk/plugins/categories/data_validation/actions/validation.py +72 -0
- synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py +33 -5
- synapse_sdk/plugins/categories/export/actions/__init__.py +3 -0
- synapse_sdk/plugins/categories/export/actions/export/__init__.py +28 -0
- synapse_sdk/plugins/categories/export/actions/export/action.py +165 -0
- synapse_sdk/plugins/categories/export/actions/export/enums.py +113 -0
- synapse_sdk/plugins/categories/export/actions/export/exceptions.py +53 -0
- synapse_sdk/plugins/categories/export/actions/export/models.py +74 -0
- synapse_sdk/plugins/categories/export/actions/export/run.py +195 -0
- synapse_sdk/plugins/categories/export/actions/export/utils.py +187 -0
- synapse_sdk/plugins/categories/export/templates/config.yaml +21 -0
- synapse_sdk/plugins/categories/export/templates/plugin/__init__.py +390 -0
- synapse_sdk/plugins/categories/export/templates/plugin/export.py +160 -0
- synapse_sdk/plugins/categories/neural_net/actions/deployment.py +29 -14
- synapse_sdk/plugins/categories/neural_net/actions/inference.py +13 -1
- synapse_sdk/plugins/categories/neural_net/actions/train.py +1084 -38
- synapse_sdk/plugins/categories/neural_net/actions/tune.py +534 -0
- synapse_sdk/plugins/categories/neural_net/base/__init__.py +0 -0
- synapse_sdk/plugins/categories/neural_net/base/inference.py +37 -0
- synapse_sdk/plugins/categories/neural_net/templates/config.yaml +30 -5
- synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py +26 -10
- synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +4 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/__init__.py +3 -0
- synapse_sdk/plugins/categories/{export/actions/export.py → pre_annotation/actions/pre_annotation/action.py} +4 -4
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/__init__.py +28 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/action.py +145 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/enums.py +269 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/exceptions.py +14 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/factory.py +76 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/models.py +97 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/orchestrator.py +250 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/run.py +64 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/__init__.py +17 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/annotation.py +287 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py +170 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py +83 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py +87 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py +127 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/validation.py +143 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py +966 -0
- synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml +19 -0
- synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py +40 -0
- synapse_sdk/plugins/categories/smart_tool/templates/config.yaml +5 -2
- synapse_sdk/plugins/categories/upload/__init__.py +0 -0
- synapse_sdk/plugins/categories/upload/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/upload/actions/upload/__init__.py +19 -0
- synapse_sdk/plugins/categories/upload/actions/upload/action.py +232 -0
- synapse_sdk/plugins/categories/upload/actions/upload/context.py +185 -0
- synapse_sdk/plugins/categories/upload/actions/upload/enums.py +471 -0
- synapse_sdk/plugins/categories/upload/actions/upload/exceptions.py +36 -0
- synapse_sdk/plugins/categories/upload/actions/upload/factory.py +138 -0
- synapse_sdk/plugins/categories/upload/actions/upload/models.py +203 -0
- synapse_sdk/plugins/categories/upload/actions/upload/orchestrator.py +183 -0
- synapse_sdk/plugins/categories/upload/actions/upload/registry.py +113 -0
- synapse_sdk/plugins/categories/upload/actions/upload/run.py +179 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py +107 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py +62 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py +63 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py +84 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py +82 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py +235 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py +203 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py +97 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py +71 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py +82 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py +39 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py +29 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py +258 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py +281 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/excel.py +174 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/none.py +16 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/sync.py +84 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py +60 -0
- synapse_sdk/plugins/categories/upload/actions/upload/utils.py +250 -0
- synapse_sdk/plugins/categories/upload/templates/README.md +470 -0
- synapse_sdk/plugins/categories/upload/templates/config.yaml +33 -0
- synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py +294 -0
- synapse_sdk/plugins/categories/upload/templates/plugin/upload.py +102 -0
- synapse_sdk/plugins/enums.py +3 -1
- synapse_sdk/plugins/models.py +140 -16
- synapse_sdk/plugins/templates/plugin-config-schema.json +406 -0
- synapse_sdk/plugins/templates/schema.json +491 -0
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/config.yaml +1 -0
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/requirements.txt +1 -1
- synapse_sdk/plugins/utils/__init__.py +46 -0
- synapse_sdk/plugins/utils/actions.py +119 -0
- synapse_sdk/plugins/utils/config.py +203 -0
- synapse_sdk/plugins/utils/legacy.py +95 -0
- synapse_sdk/plugins/utils/ray_gcs.py +66 -0
- synapse_sdk/plugins/utils/registry.py +58 -0
- synapse_sdk/shared/__init__.py +25 -0
- synapse_sdk/shared/enums.py +93 -0
- synapse_sdk/types.py +19 -0
- synapse_sdk/utils/converters/__init__.py +240 -0
- synapse_sdk/utils/converters/coco/__init__.py +0 -0
- synapse_sdk/utils/converters/coco/from_dm.py +322 -0
- synapse_sdk/utils/converters/coco/to_dm.py +215 -0
- synapse_sdk/utils/converters/dm/__init__.py +56 -0
- synapse_sdk/utils/converters/dm/from_v1.py +627 -0
- synapse_sdk/utils/converters/dm/to_v1.py +367 -0
- synapse_sdk/utils/converters/pascal/__init__.py +0 -0
- synapse_sdk/utils/converters/pascal/from_dm.py +244 -0
- synapse_sdk/utils/converters/pascal/to_dm.py +214 -0
- synapse_sdk/utils/converters/yolo/__init__.py +0 -0
- synapse_sdk/utils/converters/yolo/from_dm.py +384 -0
- synapse_sdk/utils/converters/yolo/to_dm.py +267 -0
- synapse_sdk/utils/dataset.py +46 -0
- synapse_sdk/utils/encryption.py +158 -0
- synapse_sdk/utils/file/__init__.py +39 -0
- synapse_sdk/utils/file/archive.py +32 -0
- synapse_sdk/utils/file/checksum.py +56 -0
- synapse_sdk/utils/file/chunking.py +31 -0
- synapse_sdk/utils/file/download.py +385 -0
- synapse_sdk/utils/file/encoding.py +40 -0
- synapse_sdk/utils/file/io.py +22 -0
- synapse_sdk/utils/file/video/__init__.py +29 -0
- synapse_sdk/utils/file/video/transcode.py +307 -0
- synapse_sdk/utils/file.py.backup +301 -0
- synapse_sdk/utils/http.py +138 -0
- synapse_sdk/utils/network.py +309 -0
- synapse_sdk/utils/storage/__init__.py +72 -0
- synapse_sdk/utils/storage/providers/__init__.py +183 -0
- synapse_sdk/utils/storage/providers/file_system.py +134 -0
- synapse_sdk/utils/storage/providers/gcp.py +13 -0
- synapse_sdk/utils/storage/providers/http.py +190 -0
- synapse_sdk/utils/storage/providers/s3.py +91 -0
- synapse_sdk/utils/storage/providers/sftp.py +47 -0
- synapse_sdk/utils/storage/registry.py +17 -0
- synapse_sdk-2025.11.7.dist-info/METADATA +122 -0
- synapse_sdk-2025.11.7.dist-info/RECORD +386 -0
- {synapse_sdk-1.0.0a13.dist-info → synapse_sdk-2025.11.7.dist-info}/WHEEL +1 -1
- synapse_sdk/clients/backend/dataset.py +0 -51
- synapse_sdk/plugins/categories/import/actions/import.py +0 -10
- synapse_sdk/plugins/cli/__init__.py +0 -21
- synapse_sdk/plugins/cli/publish.py +0 -37
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/.env +0 -24
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/.env.dist +0 -24
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/main.py +0 -4
- synapse_sdk/plugins/utils.py +0 -50
- synapse_sdk/utils/file.py +0 -87
- synapse_sdk/utils/storage.py +0 -91
- synapse_sdk-1.0.0a13.dist-info/METADATA +0 -43
- synapse_sdk-1.0.0a13.dist-info/RECORD +0 -111
- /synapse_sdk/{plugins/categories/import → clients/validators}/__init__.py +0 -0
- /synapse_sdk/{plugins/categories/import/actions → devtools}/__init__.py +0 -0
- {synapse_sdk-1.0.0a13.dist-info → synapse_sdk-2025.11.7.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-1.0.0a13.dist-info → synapse_sdk-2025.11.7.dist-info/licenses}/LICENSE +0 -0
- {synapse_sdk-1.0.0a13.dist-info → synapse_sdk-2025.11.7.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Orchestrator for coordinating ToTask action workflow using Facade pattern."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
|
|
5
|
+
from synapse_sdk.clients.backend.models import JobStatus
|
|
6
|
+
|
|
7
|
+
from .enums import AnnotationMethod, LogCode
|
|
8
|
+
from .exceptions import CriticalError, PreAnnotationToTaskFailed
|
|
9
|
+
from .factory import ToTaskStrategyFactory
|
|
10
|
+
from .models import ToTaskResult
|
|
11
|
+
from .strategies.base import ToTaskContext
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ToTaskOrchestrator:
|
|
15
|
+
"""Facade that orchestrates the complete ToTask annotation workflow."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, context: ToTaskContext):
|
|
18
|
+
"""Initialize orchestrator with context and strategies.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
context: Shared context for the action execution
|
|
22
|
+
"""
|
|
23
|
+
self.context = context
|
|
24
|
+
self.factory = ToTaskStrategyFactory()
|
|
25
|
+
self.steps_completed = []
|
|
26
|
+
|
|
27
|
+
# Initialize strategies
|
|
28
|
+
self.project_validation = self.factory.create_validation_strategy('project')
|
|
29
|
+
self.task_validation = self.factory.create_validation_strategy('task')
|
|
30
|
+
self.target_spec_validation = self.factory.create_validation_strategy('target_spec')
|
|
31
|
+
self.metrics_strategy = self.factory.create_metrics_strategy()
|
|
32
|
+
|
|
33
|
+
def execute_workflow(self) -> Dict[str, Any]:
|
|
34
|
+
"""Execute the complete ToTask workflow with rollback support.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
Dict containing the workflow result
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
# Step 1: Project and data collection validation
|
|
41
|
+
self._execute_step('project_validation', self._validate_project)
|
|
42
|
+
|
|
43
|
+
# Step 2: Task discovery and validation
|
|
44
|
+
self._execute_step('task_validation', self._validate_tasks)
|
|
45
|
+
|
|
46
|
+
# Step 3: Determine annotation method
|
|
47
|
+
self._execute_step('method_determination', self._determine_annotation_method)
|
|
48
|
+
|
|
49
|
+
# Step 4: Method-specific validation
|
|
50
|
+
self._execute_step('method_validation', self._validate_annotation_method)
|
|
51
|
+
|
|
52
|
+
# Step 5: Initialize processing
|
|
53
|
+
self._execute_step('processing_initialization', self._initialize_processing)
|
|
54
|
+
|
|
55
|
+
# Step 6: Process all tasks
|
|
56
|
+
self._execute_step('task_processing', self._process_all_tasks)
|
|
57
|
+
|
|
58
|
+
# Step 7: Finalize metrics and progress
|
|
59
|
+
self._execute_step('finalization', self._finalize_processing)
|
|
60
|
+
|
|
61
|
+
# Return success result
|
|
62
|
+
result = ToTaskResult(status=JobStatus.SUCCEEDED, message='Pre-annotation to task completed successfully')
|
|
63
|
+
return result.model_dump()
|
|
64
|
+
|
|
65
|
+
except Exception as e:
|
|
66
|
+
self._rollback_completed_steps()
|
|
67
|
+
if isinstance(e, PreAnnotationToTaskFailed):
|
|
68
|
+
raise e
|
|
69
|
+
raise PreAnnotationToTaskFailed(f'Workflow failed at step {len(self.steps_completed)}: {e}')
|
|
70
|
+
|
|
71
|
+
def _execute_step(self, step_name: str, step_func: callable):
|
|
72
|
+
"""Execute a workflow step with error handling and progress tracking.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
step_name: Name of the step for logging
|
|
76
|
+
step_func: Function to execute for this step
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
Result of the step function
|
|
80
|
+
"""
|
|
81
|
+
self.context.logger.log_message_with_code(LogCode.STEP_STARTED, step_name)
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
result = step_func()
|
|
85
|
+
self.steps_completed.append(step_name)
|
|
86
|
+
self.context.logger.log_message_with_code(LogCode.STEP_COMPLETED, step_name)
|
|
87
|
+
return result
|
|
88
|
+
except Exception as e:
|
|
89
|
+
self.context.logger.log_message_with_code(LogCode.STEP_FAILED, step_name, str(e))
|
|
90
|
+
raise
|
|
91
|
+
|
|
92
|
+
def _validate_project(self):
|
|
93
|
+
"""Step 1: Validate project and data collection."""
|
|
94
|
+
result = self.project_validation.validate(self.context)
|
|
95
|
+
if not result['success']:
|
|
96
|
+
error_msg = result.get('error', 'Project validation failed')
|
|
97
|
+
self.context.logger.end_log()
|
|
98
|
+
raise PreAnnotationToTaskFailed(error_msg)
|
|
99
|
+
|
|
100
|
+
def _validate_tasks(self):
|
|
101
|
+
"""Step 2: Discover and validate tasks."""
|
|
102
|
+
result = self.task_validation.validate(self.context)
|
|
103
|
+
if not result['success']:
|
|
104
|
+
error_msg = result.get('error', 'Task validation failed')
|
|
105
|
+
self.context.logger.end_log()
|
|
106
|
+
raise PreAnnotationToTaskFailed(error_msg)
|
|
107
|
+
|
|
108
|
+
def _determine_annotation_method(self):
|
|
109
|
+
"""Step 3: Determine annotation method from parameters."""
|
|
110
|
+
method = self.context.params.get('method')
|
|
111
|
+
if method == AnnotationMethod.FILE:
|
|
112
|
+
self.context.annotation_method = AnnotationMethod.FILE
|
|
113
|
+
elif method == AnnotationMethod.INFERENCE:
|
|
114
|
+
self.context.annotation_method = AnnotationMethod.INFERENCE
|
|
115
|
+
else:
|
|
116
|
+
self.context.logger.log_message_with_code(LogCode.UNSUPPORTED_METHOD, method)
|
|
117
|
+
self.context.logger.end_log()
|
|
118
|
+
raise PreAnnotationToTaskFailed(f'Unsupported annotation method: {method}')
|
|
119
|
+
|
|
120
|
+
def _validate_annotation_method(self):
|
|
121
|
+
"""Step 4: Validate method-specific requirements."""
|
|
122
|
+
if self.context.annotation_method == AnnotationMethod.FILE:
|
|
123
|
+
result = self.target_spec_validation.validate(self.context)
|
|
124
|
+
if not result['success']:
|
|
125
|
+
error_msg = result.get('error', 'Target specification validation failed')
|
|
126
|
+
self.context.logger.end_log()
|
|
127
|
+
raise PreAnnotationToTaskFailed(error_msg)
|
|
128
|
+
|
|
129
|
+
def _initialize_processing(self):
|
|
130
|
+
"""Step 5: Initialize processing metrics and progress."""
|
|
131
|
+
total_tasks = len(self.context.task_ids)
|
|
132
|
+
self.context.update_metrics(0, 0, total_tasks)
|
|
133
|
+
self.metrics_strategy.update_progress(self.context, 0, total_tasks)
|
|
134
|
+
self.context.logger.log_message_with_code(LogCode.ANNOTATING_DATA)
|
|
135
|
+
|
|
136
|
+
def _process_all_tasks(self):
|
|
137
|
+
"""Step 6: Process all tasks using appropriate annotation strategy."""
|
|
138
|
+
annotation_strategy = self.factory.create_annotation_strategy(self.context.annotation_method)
|
|
139
|
+
|
|
140
|
+
total_tasks = len(self.context.task_ids)
|
|
141
|
+
success_count = 0
|
|
142
|
+
failed_count = 0
|
|
143
|
+
current_progress = 0
|
|
144
|
+
|
|
145
|
+
# Get task parameters
|
|
146
|
+
task_params = {
|
|
147
|
+
'fields': 'id,data,data_unit',
|
|
148
|
+
'expand': 'data_unit',
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
# Process each task
|
|
152
|
+
for task_id in self.context.task_ids:
|
|
153
|
+
try:
|
|
154
|
+
# Get task data
|
|
155
|
+
task_response = self.context.client.get_task(task_id, params=task_params)
|
|
156
|
+
if isinstance(task_response, str):
|
|
157
|
+
error_msg = 'Invalid task response'
|
|
158
|
+
self.context.logger.log_annotate_task_event(LogCode.INVALID_TASK_RESPONSE, task_id)
|
|
159
|
+
self.metrics_strategy.record_task_result(self.context, task_id, False, error_msg)
|
|
160
|
+
failed_count += 1
|
|
161
|
+
continue
|
|
162
|
+
|
|
163
|
+
task_data: Dict[str, Any] = task_response
|
|
164
|
+
|
|
165
|
+
# Process task using annotation strategy
|
|
166
|
+
if self.context.annotation_method == AnnotationMethod.FILE:
|
|
167
|
+
target_spec_name = self.context.params.get('target_specification_name')
|
|
168
|
+
result = annotation_strategy.process_task(
|
|
169
|
+
self.context, task_id, task_data, target_specification_name=target_spec_name
|
|
170
|
+
)
|
|
171
|
+
else:
|
|
172
|
+
result = annotation_strategy.process_task(self.context, task_id, task_data)
|
|
173
|
+
|
|
174
|
+
# Record result
|
|
175
|
+
if result['success']:
|
|
176
|
+
success_count += 1
|
|
177
|
+
self.metrics_strategy.record_task_result(self.context, task_id, True)
|
|
178
|
+
else:
|
|
179
|
+
failed_count += 1
|
|
180
|
+
error_msg = result.get('error', 'Unknown error')
|
|
181
|
+
self.metrics_strategy.record_task_result(self.context, task_id, False, error_msg)
|
|
182
|
+
|
|
183
|
+
# Update progress
|
|
184
|
+
current_progress += 1
|
|
185
|
+
self.context.update_metrics(success_count, failed_count, total_tasks)
|
|
186
|
+
self.metrics_strategy.update_progress(self.context, current_progress, total_tasks)
|
|
187
|
+
self.metrics_strategy.update_metrics(self.context, total_tasks, success_count, failed_count)
|
|
188
|
+
|
|
189
|
+
except CriticalError:
|
|
190
|
+
self.context.logger.log_message_with_code(LogCode.CRITICAL_ERROR)
|
|
191
|
+
raise PreAnnotationToTaskFailed('Critical error occurred during task processing')
|
|
192
|
+
|
|
193
|
+
except Exception as e:
|
|
194
|
+
self.context.logger.log_annotate_task_event(LogCode.TASK_PROCESSING_FAILED, task_id, str(e))
|
|
195
|
+
self.metrics_strategy.record_task_result(self.context, task_id, False, str(e))
|
|
196
|
+
failed_count += 1
|
|
197
|
+
current_progress += 1
|
|
198
|
+
self.context.update_metrics(success_count, failed_count, total_tasks)
|
|
199
|
+
self.metrics_strategy.update_progress(self.context, current_progress, total_tasks)
|
|
200
|
+
self.metrics_strategy.update_metrics(self.context, total_tasks, success_count, failed_count)
|
|
201
|
+
|
|
202
|
+
def _finalize_processing(self):
|
|
203
|
+
"""Step 7: Finalize metrics."""
|
|
204
|
+
# Finalize metrics
|
|
205
|
+
self.metrics_strategy.finalize_metrics(self.context)
|
|
206
|
+
|
|
207
|
+
def _rollback_completed_steps(self):
|
|
208
|
+
"""Rollback completed steps in reverse order."""
|
|
209
|
+
for step in reversed(self.steps_completed):
|
|
210
|
+
try:
|
|
211
|
+
rollback_method = getattr(self, f'_rollback_{step}', None)
|
|
212
|
+
if rollback_method:
|
|
213
|
+
rollback_method()
|
|
214
|
+
except Exception as e:
|
|
215
|
+
self.context.logger.log_message_with_code(LogCode.ROLLBACK_FAILED, step, str(e))
|
|
216
|
+
|
|
217
|
+
# Execute any additional rollback actions
|
|
218
|
+
for action in reversed(self.context.rollback_actions):
|
|
219
|
+
try:
|
|
220
|
+
action()
|
|
221
|
+
except Exception as e:
|
|
222
|
+
self.context.logger.log_message_with_code(LogCode.ROLLBACK_ACTION_FAILED, str(e))
|
|
223
|
+
|
|
224
|
+
def _rollback_project_validation(self):
|
|
225
|
+
"""Rollback project validation step."""
|
|
226
|
+
# Clear cached project and data collection data
|
|
227
|
+
self.context.project = None
|
|
228
|
+
self.context.data_collection = None
|
|
229
|
+
|
|
230
|
+
def _rollback_task_validation(self):
|
|
231
|
+
"""Rollback task validation step."""
|
|
232
|
+
# Clear cached task data
|
|
233
|
+
self.context.task_ids = []
|
|
234
|
+
|
|
235
|
+
def _rollback_processing_initialization(self):
|
|
236
|
+
"""Rollback processing initialization step."""
|
|
237
|
+
# Reset metrics
|
|
238
|
+
self.context.update_metrics(0, 0, 0)
|
|
239
|
+
|
|
240
|
+
def _rollback_task_processing(self):
|
|
241
|
+
"""Rollback task processing step."""
|
|
242
|
+
# Clean up any temporary files
|
|
243
|
+
for temp_file in self.context.temp_files:
|
|
244
|
+
try:
|
|
245
|
+
import os
|
|
246
|
+
|
|
247
|
+
if os.path.exists(temp_file):
|
|
248
|
+
os.remove(temp_file)
|
|
249
|
+
except Exception:
|
|
250
|
+
pass # Best effort cleanup
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from typing import Any, Dict, Optional
|
|
4
|
+
|
|
5
|
+
from synapse_sdk.plugins.models import Run
|
|
6
|
+
from synapse_sdk.shared.enums import Context
|
|
7
|
+
|
|
8
|
+
from .enums import LOG_MESSAGES, AnnotateTaskDataStatus, LogCode
|
|
9
|
+
from .models import AnnotateTaskDataLog, AnnotateTaskEventLog, MetricsRecord
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ToTaskRun(Run):
|
|
13
|
+
def log_message_with_code(self, code: LogCode, *args, level: Optional[Context] = None):
|
|
14
|
+
"""Log message using predefined code and optional level override."""
|
|
15
|
+
if code not in LOG_MESSAGES:
|
|
16
|
+
self.log_message(f'Unknown log code: {code}')
|
|
17
|
+
return
|
|
18
|
+
|
|
19
|
+
log_config = LOG_MESSAGES[code]
|
|
20
|
+
message = log_config['message'].format(*args) if args else log_config['message']
|
|
21
|
+
log_level = level or log_config['level']
|
|
22
|
+
|
|
23
|
+
if log_level:
|
|
24
|
+
self.log_message(message, context=log_level.value)
|
|
25
|
+
else:
|
|
26
|
+
self.log_message(message, context=Context.INFO.value)
|
|
27
|
+
|
|
28
|
+
def log_annotate_task_event(self, code: LogCode, *args, level: Optional[Context] = None):
|
|
29
|
+
"""Log annotate task event using predefined code."""
|
|
30
|
+
if code not in LOG_MESSAGES:
|
|
31
|
+
now = datetime.now().isoformat()
|
|
32
|
+
self.log(
|
|
33
|
+
'annotate_task_event',
|
|
34
|
+
AnnotateTaskEventLog(info=f'Unknown log code: {code}', status=Context.DANGER, created=now).model_dump(),
|
|
35
|
+
)
|
|
36
|
+
return
|
|
37
|
+
|
|
38
|
+
log_config = LOG_MESSAGES[code]
|
|
39
|
+
message = log_config['message'].format(*args) if args else log_config['message']
|
|
40
|
+
log_level = level or log_config['level'] or Context.INFO
|
|
41
|
+
|
|
42
|
+
now = datetime.now().isoformat()
|
|
43
|
+
self.log(
|
|
44
|
+
'annotate_task_event',
|
|
45
|
+
AnnotateTaskEventLog(info=message, status=log_level, created=now).model_dump(),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
def log_annotate_task_data(self, task_info: Dict[str, Any], status: AnnotateTaskDataStatus):
|
|
49
|
+
"""Log annotate task data."""
|
|
50
|
+
now = datetime.now().isoformat()
|
|
51
|
+
self.log(
|
|
52
|
+
'annotate_task_data',
|
|
53
|
+
AnnotateTaskDataLog(task_info=json.dumps(task_info), status=status, created=now).model_dump(),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def log_metrics(self, record: MetricsRecord, category: str):
|
|
57
|
+
"""Log FileToTask metrics.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
record (MetricsRecord): The metrics record to log.
|
|
61
|
+
category (str): The category of the metrics.
|
|
62
|
+
"""
|
|
63
|
+
record = MetricsRecord.model_validate(record)
|
|
64
|
+
self.set_metrics(value=record.model_dump(), category=category)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Strategy classes for ToTask action refactoring."""
|
|
2
|
+
|
|
3
|
+
from .base import (
|
|
4
|
+
AnnotationStrategy,
|
|
5
|
+
DataExtractionStrategy,
|
|
6
|
+
MetricsStrategy,
|
|
7
|
+
PreProcessorStrategy,
|
|
8
|
+
ValidationStrategy,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
'AnnotationStrategy',
|
|
13
|
+
'DataExtractionStrategy',
|
|
14
|
+
'MetricsStrategy',
|
|
15
|
+
'PreProcessorStrategy',
|
|
16
|
+
'ValidationStrategy',
|
|
17
|
+
]
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""Annotation strategies for ToTask action."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
|
|
5
|
+
from ..enums import AnnotateTaskDataStatus, LogCode
|
|
6
|
+
from .base import AnnotationStrategy, ToTaskContext
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class FileAnnotationStrategy(AnnotationStrategy):
|
|
10
|
+
"""Strategy for file-based annotation processing."""
|
|
11
|
+
|
|
12
|
+
def process_task(
|
|
13
|
+
self, context: ToTaskContext, task_id: int, task_data: Dict[str, Any], target_specification_name: str, **kwargs
|
|
14
|
+
) -> Dict[str, Any]:
|
|
15
|
+
"""Process a single task for file-based annotation.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
context: Shared context for the action execution
|
|
19
|
+
task_id: The task ID to process
|
|
20
|
+
task_data: The task data dictionary
|
|
21
|
+
target_specification_name: The name of the target specification
|
|
22
|
+
**kwargs: Additional parameters
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
Dict with 'success' boolean and optional 'error' message
|
|
26
|
+
"""
|
|
27
|
+
try:
|
|
28
|
+
client = context.client
|
|
29
|
+
logger = context.logger
|
|
30
|
+
|
|
31
|
+
# Get data unit
|
|
32
|
+
data_unit = task_data.get('data_unit')
|
|
33
|
+
if not data_unit:
|
|
34
|
+
error_msg = 'Task does not have a data unit'
|
|
35
|
+
logger.log_annotate_task_event(LogCode.NO_DATA_UNIT, task_id)
|
|
36
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
37
|
+
return {'success': False, 'error': error_msg}
|
|
38
|
+
|
|
39
|
+
# Get data unit files
|
|
40
|
+
data_unit_files = data_unit.get('files', {})
|
|
41
|
+
if not data_unit_files:
|
|
42
|
+
error_msg = 'Data unit does not have files'
|
|
43
|
+
logger.log_annotate_task_event(LogCode.NO_DATA_UNIT_FILES, task_id)
|
|
44
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
45
|
+
return {'success': False, 'error': error_msg}
|
|
46
|
+
|
|
47
|
+
# Extract primary file URL from task data
|
|
48
|
+
primary_file_url, primary_file_original_name = self._extract_primary_file_url(task_data)
|
|
49
|
+
if not primary_file_url:
|
|
50
|
+
error_msg = 'Primary image URL not found in task data'
|
|
51
|
+
logger.log_annotate_task_event(LogCode.PRIMARY_IMAGE_URL_NOT_FOUND, task_id)
|
|
52
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
53
|
+
return {'success': False, 'error': error_msg}
|
|
54
|
+
|
|
55
|
+
# Get target specification file
|
|
56
|
+
target_file = data_unit_files.get(target_specification_name)
|
|
57
|
+
if not target_file:
|
|
58
|
+
error_msg = 'File specification not found'
|
|
59
|
+
logger.log_annotate_task_event(LogCode.FILE_SPEC_NOT_FOUND, task_id)
|
|
60
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
61
|
+
return {'success': False, 'error': error_msg}
|
|
62
|
+
|
|
63
|
+
# Get target file details
|
|
64
|
+
target_file_url = target_file.get('url')
|
|
65
|
+
target_file_original_name = target_file.get('file_name_original')
|
|
66
|
+
|
|
67
|
+
if not target_file_original_name:
|
|
68
|
+
error_msg = 'File original name not found'
|
|
69
|
+
logger.log_annotate_task_event(LogCode.FILE_ORIGINAL_NAME_NOT_FOUND, task_id)
|
|
70
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
71
|
+
return {'success': False, 'error': error_msg}
|
|
72
|
+
|
|
73
|
+
if not target_file_url:
|
|
74
|
+
error_msg = 'URL not found'
|
|
75
|
+
logger.log_annotate_task_event(LogCode.URL_NOT_FOUND, task_id)
|
|
76
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
77
|
+
return {'success': False, 'error': error_msg}
|
|
78
|
+
|
|
79
|
+
# Fetch and process the data using template
|
|
80
|
+
try:
|
|
81
|
+
# Convert data to task object using action's entrypoint
|
|
82
|
+
annotation_to_task = context.entrypoint(logger)
|
|
83
|
+
converted_data = annotation_to_task.convert_data_from_file(
|
|
84
|
+
primary_file_url, primary_file_original_name, target_file_url, target_file_original_name
|
|
85
|
+
)
|
|
86
|
+
except Exception as e:
|
|
87
|
+
if 'requests' in str(type(e)):
|
|
88
|
+
error_msg = f'Failed to fetch data from URL: {str(e)}'
|
|
89
|
+
logger.log_annotate_task_event(LogCode.FETCH_DATA_FAILED, target_file_url, task_id)
|
|
90
|
+
else:
|
|
91
|
+
error_msg = f'Failed to convert data to task object: {str(e)}'
|
|
92
|
+
logger.log_annotate_task_event(LogCode.CONVERT_DATA_FAILED, str(e), task_id)
|
|
93
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
94
|
+
return {'success': False, 'error': error_msg}
|
|
95
|
+
|
|
96
|
+
# Submit annotation data
|
|
97
|
+
try:
|
|
98
|
+
client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_data})
|
|
99
|
+
logger.log_annotate_task_data(
|
|
100
|
+
{'task_id': task_id, 'target_spec': target_specification_name}, AnnotateTaskDataStatus.SUCCESS
|
|
101
|
+
)
|
|
102
|
+
return {'success': True}
|
|
103
|
+
except Exception as e:
|
|
104
|
+
error_msg = f'Failed to submit annotation data: {str(e)}'
|
|
105
|
+
logger.log_annotate_task_event(LogCode.ANNOTATION_SUBMISSION_FAILED, task_id, str(e))
|
|
106
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
107
|
+
return {'success': False, 'error': error_msg}
|
|
108
|
+
|
|
109
|
+
except Exception as e:
|
|
110
|
+
error_msg = f'Failed to process file annotation for task {task_id}: {str(e)}'
|
|
111
|
+
context.logger.log_annotate_task_event(LogCode.TASK_PROCESSING_FAILED, task_id, str(e))
|
|
112
|
+
context.logger.log_annotate_task_data(
|
|
113
|
+
{'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED
|
|
114
|
+
)
|
|
115
|
+
return {'success': False, 'error': error_msg}
|
|
116
|
+
|
|
117
|
+
def _extract_primary_file_url(self, task_data: Dict[str, Any]) -> tuple:
|
|
118
|
+
"""Extract the primary file URL from task data.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
task_data: The task data dictionary
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
Tuple of (primary_file_url, primary_file_original_name)
|
|
125
|
+
"""
|
|
126
|
+
data_unit = task_data.get('data_unit', {})
|
|
127
|
+
files = data_unit.get('files', {})
|
|
128
|
+
|
|
129
|
+
for file_info in files.values():
|
|
130
|
+
if isinstance(file_info, dict) and file_info.get('is_primary') and file_info.get('url'):
|
|
131
|
+
return file_info['url'], file_info.get('file_name_original')
|
|
132
|
+
|
|
133
|
+
return None, None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class InferenceAnnotationStrategy(AnnotationStrategy):
|
|
137
|
+
"""Strategy for inference-based annotation processing."""
|
|
138
|
+
|
|
139
|
+
def process_task(self, context: ToTaskContext, task_id: int, task_data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
|
|
140
|
+
"""Process a single task for inference-based annotation.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
context: Shared context for the action execution
|
|
144
|
+
task_id: The task ID to process
|
|
145
|
+
task_data: The task data dictionary
|
|
146
|
+
**kwargs: Additional parameters
|
|
147
|
+
|
|
148
|
+
Returns:
|
|
149
|
+
Dict with 'success' boolean and optional 'error' message
|
|
150
|
+
"""
|
|
151
|
+
try:
|
|
152
|
+
client = context.client
|
|
153
|
+
logger = context.logger
|
|
154
|
+
|
|
155
|
+
# Get pre-processor ID from parameters
|
|
156
|
+
pre_processor_id = context.params.get('pre_processor')
|
|
157
|
+
if not pre_processor_id:
|
|
158
|
+
error_msg = 'Pre-processor ID is required for inference annotation method'
|
|
159
|
+
logger.log_annotate_task_event(LogCode.NO_PREPROCESSOR_ID, task_id)
|
|
160
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
161
|
+
return {'success': False, 'error': error_msg}
|
|
162
|
+
|
|
163
|
+
# Get pre-processor info using factory-created strategy
|
|
164
|
+
from ..factory import ToTaskStrategyFactory
|
|
165
|
+
|
|
166
|
+
factory = ToTaskStrategyFactory()
|
|
167
|
+
preprocessor_strategy = factory.create_preprocessor_strategy()
|
|
168
|
+
|
|
169
|
+
pre_processor_info = preprocessor_strategy.get_preprocessor_info(context, pre_processor_id)
|
|
170
|
+
if not pre_processor_info['success']:
|
|
171
|
+
error_msg = pre_processor_info.get('error', 'Failed to get pre-processor info')
|
|
172
|
+
logger.log_annotate_task_event(LogCode.INFERENCE_PREPROCESSOR_FAILED, task_id, error_msg)
|
|
173
|
+
return pre_processor_info
|
|
174
|
+
|
|
175
|
+
pre_processor_code = pre_processor_info['code']
|
|
176
|
+
pre_processor_version = pre_processor_info['version']
|
|
177
|
+
|
|
178
|
+
# Ensure pre-processor is running
|
|
179
|
+
pre_processor_status = preprocessor_strategy.ensure_preprocessor_running(context, pre_processor_code)
|
|
180
|
+
if not pre_processor_status['success']:
|
|
181
|
+
error_msg = pre_processor_status.get('error', 'Failed to ensure pre-processor running')
|
|
182
|
+
logger.log_annotate_task_event(LogCode.INFERENCE_PREPROCESSOR_FAILED, task_id, error_msg)
|
|
183
|
+
return pre_processor_status
|
|
184
|
+
|
|
185
|
+
# Extract primary file URL using factory-created strategy
|
|
186
|
+
extraction_strategy = factory.create_extraction_strategy(context.annotation_method)
|
|
187
|
+
primary_file_url, _ = extraction_strategy.extract_data(context, task_data)
|
|
188
|
+
if not primary_file_url:
|
|
189
|
+
error_msg = 'Primary image URL not found in task data'
|
|
190
|
+
logger.log_annotate_task_event(LogCode.PRIMARY_IMAGE_URL_NOT_FOUND, task_id)
|
|
191
|
+
logger.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
192
|
+
return {'success': False, 'error': error_msg}
|
|
193
|
+
|
|
194
|
+
# Run inference
|
|
195
|
+
inference_result = self._run_inference(
|
|
196
|
+
client,
|
|
197
|
+
pre_processor_code,
|
|
198
|
+
pre_processor_version,
|
|
199
|
+
primary_file_url,
|
|
200
|
+
context.params['agent'],
|
|
201
|
+
context.params['model'],
|
|
202
|
+
pre_processor_params=context.params.get('pre_processor_params', {}),
|
|
203
|
+
)
|
|
204
|
+
if not inference_result['success']:
|
|
205
|
+
error_msg = inference_result.get('error', 'Failed to run inference')
|
|
206
|
+
logger.log_annotate_task_event(LogCode.INFERENCE_PREPROCESSOR_FAILED, task_id, error_msg)
|
|
207
|
+
return inference_result
|
|
208
|
+
|
|
209
|
+
# Convert and submit inference data
|
|
210
|
+
try:
|
|
211
|
+
# This would need to be injected or configured based on the action's entrypoint
|
|
212
|
+
# For now, we'll assume the conversion is done externally
|
|
213
|
+
inference_data = inference_result['data'] # Simplified for refactoring
|
|
214
|
+
|
|
215
|
+
annotation_to_task = context.entrypoint(logger)
|
|
216
|
+
converted_result = annotation_to_task.convert_data_from_inference(inference_data)
|
|
217
|
+
|
|
218
|
+
client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_result})
|
|
219
|
+
logger.log_annotate_task_data(
|
|
220
|
+
{'task_id': task_id, 'pre_processor_id': pre_processor_id}, AnnotateTaskDataStatus.SUCCESS
|
|
221
|
+
)
|
|
222
|
+
return {'success': True, 'pre_processor_id': pre_processor_id}
|
|
223
|
+
|
|
224
|
+
except Exception as e:
|
|
225
|
+
error_msg = f'Failed to convert/submit inference data: {str(e)}'
|
|
226
|
+
logger.log_annotate_task_event(LogCode.INFERENCE_PREPROCESSOR_FAILED, task_id, error_msg)
|
|
227
|
+
return {'success': False, 'error': error_msg}
|
|
228
|
+
|
|
229
|
+
except Exception as e:
|
|
230
|
+
error_msg = f'Failed to process inference for task {task_id}: {str(e)}'
|
|
231
|
+
context.logger.log_message_with_code(LogCode.INFERENCE_PROCESSING_FAILED, task_id, str(e))
|
|
232
|
+
context.logger.log_annotate_task_data(
|
|
233
|
+
{'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED
|
|
234
|
+
)
|
|
235
|
+
return {'success': False, 'error': error_msg}
|
|
236
|
+
|
|
237
|
+
def _run_inference(
|
|
238
|
+
self,
|
|
239
|
+
client: Any,
|
|
240
|
+
pre_processor_code: str,
|
|
241
|
+
pre_processor_version: str,
|
|
242
|
+
primary_file_url: str,
|
|
243
|
+
agent: int,
|
|
244
|
+
model: int,
|
|
245
|
+
pre_processor_params: Dict[str, Any] = {},
|
|
246
|
+
) -> Dict[str, Any]:
|
|
247
|
+
"""Run inference using the pre-processor.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
client: Backend client instance
|
|
251
|
+
pre_processor_code: Pre-processor code
|
|
252
|
+
pre_processor_version: Pre-processor version
|
|
253
|
+
primary_file_url: URL of the primary file to process
|
|
254
|
+
agent: Agent id for inference
|
|
255
|
+
model: Model id for inference
|
|
256
|
+
pre_processor_params: Additional parameters for the pre-processor
|
|
257
|
+
|
|
258
|
+
Returns:
|
|
259
|
+
Dict with inference results or error
|
|
260
|
+
"""
|
|
261
|
+
try:
|
|
262
|
+
if not agent or not model:
|
|
263
|
+
return {'success': False, 'error': 'Parameters not available'}
|
|
264
|
+
|
|
265
|
+
pre_processor_params['image_path'] = primary_file_url
|
|
266
|
+
|
|
267
|
+
inference_payload = {
|
|
268
|
+
'agent': agent,
|
|
269
|
+
'action': 'inference',
|
|
270
|
+
'version': pre_processor_version,
|
|
271
|
+
'params': {
|
|
272
|
+
'model': model,
|
|
273
|
+
'method': 'post',
|
|
274
|
+
'json': pre_processor_params,
|
|
275
|
+
},
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
inference_data = client.run_plugin(pre_processor_code, inference_payload)
|
|
279
|
+
|
|
280
|
+
# Every inference api should return None if failed to inference.
|
|
281
|
+
if inference_data is None:
|
|
282
|
+
return {'success': False, 'error': 'Inference data is None'}
|
|
283
|
+
|
|
284
|
+
return {'success': True, 'data': inference_data}
|
|
285
|
+
|
|
286
|
+
except Exception as e:
|
|
287
|
+
return {'success': False, 'error': f'Failed to run inference: {str(e)}'}
|