synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse-sdk might be problematic. Click here for more details.

Files changed (307) hide show
  1. synapse_sdk/__init__.py +24 -0
  2. synapse_sdk/cli/__init__.py +308 -5
  3. synapse_sdk/cli/alias/utils.py +1 -1
  4. synapse_sdk/cli/code_server.py +687 -0
  5. synapse_sdk/cli/config.py +440 -0
  6. synapse_sdk/cli/devtools.py +90 -0
  7. synapse_sdk/cli/plugin/publish.py +23 -15
  8. synapse_sdk/clients/agent/__init__.py +9 -3
  9. synapse_sdk/clients/agent/container.py +133 -0
  10. synapse_sdk/clients/agent/core.py +19 -0
  11. synapse_sdk/clients/agent/ray.py +298 -9
  12. synapse_sdk/clients/backend/__init__.py +28 -12
  13. synapse_sdk/clients/backend/annotation.py +9 -1
  14. synapse_sdk/clients/backend/core.py +31 -4
  15. synapse_sdk/clients/backend/data_collection.py +186 -0
  16. synapse_sdk/clients/backend/hitl.py +1 -1
  17. synapse_sdk/clients/backend/integration.py +4 -3
  18. synapse_sdk/clients/backend/ml.py +1 -1
  19. synapse_sdk/clients/backend/models.py +35 -1
  20. synapse_sdk/clients/base.py +309 -36
  21. synapse_sdk/clients/ray/serve.py +2 -0
  22. synapse_sdk/devtools/__init__.py +0 -0
  23. synapse_sdk/devtools/config.py +94 -0
  24. synapse_sdk/devtools/docs/.gitignore +20 -0
  25. synapse_sdk/devtools/docs/README.md +41 -0
  26. synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md +12 -0
  27. synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md +44 -0
  28. synapse_sdk/devtools/docs/blog/2021-08-01-mdx-blog-post.mdx +24 -0
  29. synapse_sdk/devtools/docs/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg +0 -0
  30. synapse_sdk/devtools/docs/blog/2021-08-26-welcome/index.md +29 -0
  31. synapse_sdk/devtools/docs/blog/authors.yml +25 -0
  32. synapse_sdk/devtools/docs/blog/tags.yml +19 -0
  33. synapse_sdk/devtools/docs/docs/api/clients/agent.md +43 -0
  34. synapse_sdk/devtools/docs/docs/api/clients/annotation-mixin.md +378 -0
  35. synapse_sdk/devtools/docs/docs/api/clients/backend.md +420 -0
  36. synapse_sdk/devtools/docs/docs/api/clients/base.md +257 -0
  37. synapse_sdk/devtools/docs/docs/api/clients/core-mixin.md +477 -0
  38. synapse_sdk/devtools/docs/docs/api/clients/data-collection-mixin.md +422 -0
  39. synapse_sdk/devtools/docs/docs/api/clients/hitl-mixin.md +554 -0
  40. synapse_sdk/devtools/docs/docs/api/clients/index.md +391 -0
  41. synapse_sdk/devtools/docs/docs/api/clients/integration-mixin.md +571 -0
  42. synapse_sdk/devtools/docs/docs/api/clients/ml-mixin.md +578 -0
  43. synapse_sdk/devtools/docs/docs/api/clients/ray.md +342 -0
  44. synapse_sdk/devtools/docs/docs/api/index.md +52 -0
  45. synapse_sdk/devtools/docs/docs/api/plugins/categories.md +43 -0
  46. synapse_sdk/devtools/docs/docs/api/plugins/models.md +114 -0
  47. synapse_sdk/devtools/docs/docs/api/plugins/utils.md +328 -0
  48. synapse_sdk/devtools/docs/docs/categories.md +0 -0
  49. synapse_sdk/devtools/docs/docs/cli-usage.md +280 -0
  50. synapse_sdk/devtools/docs/docs/concepts/index.md +38 -0
  51. synapse_sdk/devtools/docs/docs/configuration.md +83 -0
  52. synapse_sdk/devtools/docs/docs/contributing.md +306 -0
  53. synapse_sdk/devtools/docs/docs/examples/index.md +29 -0
  54. synapse_sdk/devtools/docs/docs/faq.md +179 -0
  55. synapse_sdk/devtools/docs/docs/features/converters/index.md +455 -0
  56. synapse_sdk/devtools/docs/docs/features/index.md +24 -0
  57. synapse_sdk/devtools/docs/docs/features/utils/file.md +415 -0
  58. synapse_sdk/devtools/docs/docs/features/utils/network.md +378 -0
  59. synapse_sdk/devtools/docs/docs/features/utils/storage.md +57 -0
  60. synapse_sdk/devtools/docs/docs/features/utils/types.md +51 -0
  61. synapse_sdk/devtools/docs/docs/installation.md +94 -0
  62. synapse_sdk/devtools/docs/docs/introduction.md +47 -0
  63. synapse_sdk/devtools/docs/docs/plugins/categories/neural-net-plugins/train-action-overview.md +814 -0
  64. synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
  65. synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
  66. synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
  67. synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
  68. synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-action.md +948 -0
  69. synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-overview.md +544 -0
  70. synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-template.md +766 -0
  71. synapse_sdk/devtools/docs/docs/plugins/export-plugins.md +1092 -0
  72. synapse_sdk/devtools/docs/docs/plugins/plugins.md +852 -0
  73. synapse_sdk/devtools/docs/docs/quickstart.md +78 -0
  74. synapse_sdk/devtools/docs/docs/troubleshooting.md +519 -0
  75. synapse_sdk/devtools/docs/docs/tutorial-basics/_category_.json +8 -0
  76. synapse_sdk/devtools/docs/docs/tutorial-basics/congratulations.md +23 -0
  77. synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-blog-post.md +34 -0
  78. synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-document.md +57 -0
  79. synapse_sdk/devtools/docs/docs/tutorial-basics/create-a-page.md +43 -0
  80. synapse_sdk/devtools/docs/docs/tutorial-basics/deploy-your-site.md +31 -0
  81. synapse_sdk/devtools/docs/docs/tutorial-basics/markdown-features.mdx +152 -0
  82. synapse_sdk/devtools/docs/docs/tutorial-extras/_category_.json +7 -0
  83. synapse_sdk/devtools/docs/docs/tutorial-extras/img/docsVersionDropdown.png +0 -0
  84. synapse_sdk/devtools/docs/docs/tutorial-extras/img/localeDropdown.png +0 -0
  85. synapse_sdk/devtools/docs/docs/tutorial-extras/manage-docs-versions.md +55 -0
  86. synapse_sdk/devtools/docs/docs/tutorial-extras/translate-your-site.md +88 -0
  87. synapse_sdk/devtools/docs/docusaurus.config.ts +148 -0
  88. synapse_sdk/devtools/docs/i18n/ko/code.json +325 -0
  89. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/agent.md +43 -0
  90. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/annotation-mixin.md +289 -0
  91. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/backend.md +420 -0
  92. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/base.md +257 -0
  93. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/core-mixin.md +417 -0
  94. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/data-collection-mixin.md +356 -0
  95. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/hitl-mixin.md +192 -0
  96. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/index.md +391 -0
  97. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/integration-mixin.md +479 -0
  98. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ml-mixin.md +284 -0
  99. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ray.md +342 -0
  100. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/index.md +52 -0
  101. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/plugins/models.md +114 -0
  102. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/categories.md +0 -0
  103. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/cli-usage.md +280 -0
  104. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/concepts/index.md +38 -0
  105. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/configuration.md +83 -0
  106. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/contributing.md +306 -0
  107. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/examples/index.md +29 -0
  108. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/faq.md +179 -0
  109. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/converters/index.md +30 -0
  110. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/index.md +24 -0
  111. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/file.md +415 -0
  112. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/network.md +378 -0
  113. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/storage.md +60 -0
  114. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/types.md +51 -0
  115. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/installation.md +94 -0
  116. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/introduction.md +47 -0
  117. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/neural-net-plugins/train-action-overview.md +815 -0
  118. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
  119. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
  120. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
  121. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
  122. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-action.md +948 -0
  123. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-overview.md +544 -0
  124. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-template.md +766 -0
  125. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md +1092 -0
  126. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md +117 -0
  127. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/quickstart.md +78 -0
  128. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/troubleshooting.md +519 -0
  129. synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current.json +34 -0
  130. synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/footer.json +42 -0
  131. synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/navbar.json +18 -0
  132. synapse_sdk/devtools/docs/package-lock.json +18784 -0
  133. synapse_sdk/devtools/docs/package.json +48 -0
  134. synapse_sdk/devtools/docs/sidebars.ts +122 -0
  135. synapse_sdk/devtools/docs/src/components/HomepageFeatures/index.tsx +71 -0
  136. synapse_sdk/devtools/docs/src/components/HomepageFeatures/styles.module.css +11 -0
  137. synapse_sdk/devtools/docs/src/css/custom.css +30 -0
  138. synapse_sdk/devtools/docs/src/pages/index.module.css +23 -0
  139. synapse_sdk/devtools/docs/src/pages/index.tsx +21 -0
  140. synapse_sdk/devtools/docs/src/pages/markdown-page.md +7 -0
  141. synapse_sdk/devtools/docs/static/.nojekyll +0 -0
  142. synapse_sdk/devtools/docs/static/img/docusaurus-social-card.jpg +0 -0
  143. synapse_sdk/devtools/docs/static/img/docusaurus.png +0 -0
  144. synapse_sdk/devtools/docs/static/img/favicon.ico +0 -0
  145. synapse_sdk/devtools/docs/static/img/logo.png +0 -0
  146. synapse_sdk/devtools/docs/static/img/undraw_docusaurus_mountain.svg +171 -0
  147. synapse_sdk/devtools/docs/static/img/undraw_docusaurus_react.svg +170 -0
  148. synapse_sdk/devtools/docs/static/img/undraw_docusaurus_tree.svg +40 -0
  149. synapse_sdk/devtools/docs/tsconfig.json +8 -0
  150. synapse_sdk/devtools/server.py +41 -0
  151. synapse_sdk/devtools/streamlit_app/__init__.py +5 -0
  152. synapse_sdk/devtools/streamlit_app/app.py +128 -0
  153. synapse_sdk/devtools/streamlit_app/services/__init__.py +11 -0
  154. synapse_sdk/devtools/streamlit_app/services/job_service.py +233 -0
  155. synapse_sdk/devtools/streamlit_app/services/plugin_service.py +236 -0
  156. synapse_sdk/devtools/streamlit_app/services/serve_service.py +95 -0
  157. synapse_sdk/devtools/streamlit_app/ui/__init__.py +15 -0
  158. synapse_sdk/devtools/streamlit_app/ui/config_tab.py +76 -0
  159. synapse_sdk/devtools/streamlit_app/ui/deployment_tab.py +66 -0
  160. synapse_sdk/devtools/streamlit_app/ui/http_tab.py +125 -0
  161. synapse_sdk/devtools/streamlit_app/ui/jobs_tab.py +573 -0
  162. synapse_sdk/devtools/streamlit_app/ui/serve_tab.py +346 -0
  163. synapse_sdk/devtools/streamlit_app/ui/status_bar.py +118 -0
  164. synapse_sdk/devtools/streamlit_app/utils/__init__.py +40 -0
  165. synapse_sdk/devtools/streamlit_app/utils/json_viewer.py +197 -0
  166. synapse_sdk/devtools/streamlit_app/utils/log_formatter.py +38 -0
  167. synapse_sdk/devtools/streamlit_app/utils/styles.py +241 -0
  168. synapse_sdk/devtools/streamlit_app/utils/ui_components.py +289 -0
  169. synapse_sdk/devtools/streamlit_app.py +10 -0
  170. synapse_sdk/loggers.py +65 -7
  171. synapse_sdk/plugins/README.md +1340 -0
  172. synapse_sdk/plugins/categories/base.py +73 -11
  173. synapse_sdk/plugins/categories/data_validation/actions/validation.py +72 -0
  174. synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py +33 -5
  175. synapse_sdk/plugins/categories/export/actions/__init__.py +3 -0
  176. synapse_sdk/plugins/categories/export/actions/export/__init__.py +28 -0
  177. synapse_sdk/plugins/categories/export/actions/export/action.py +165 -0
  178. synapse_sdk/plugins/categories/export/actions/export/enums.py +113 -0
  179. synapse_sdk/plugins/categories/export/actions/export/exceptions.py +53 -0
  180. synapse_sdk/plugins/categories/export/actions/export/models.py +74 -0
  181. synapse_sdk/plugins/categories/export/actions/export/run.py +195 -0
  182. synapse_sdk/plugins/categories/export/actions/{export.py → export/utils.py} +47 -82
  183. synapse_sdk/plugins/categories/export/templates/config.yaml +19 -1
  184. synapse_sdk/plugins/categories/export/templates/plugin/__init__.py +390 -0
  185. synapse_sdk/plugins/categories/export/templates/plugin/export.py +153 -129
  186. synapse_sdk/plugins/categories/neural_net/actions/deployment.py +9 -62
  187. synapse_sdk/plugins/categories/neural_net/actions/train.py +1062 -32
  188. synapse_sdk/plugins/categories/neural_net/actions/tune.py +534 -0
  189. synapse_sdk/plugins/categories/neural_net/templates/config.yaml +27 -5
  190. synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py +26 -10
  191. synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +4 -0
  192. synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/__init__.py +3 -0
  193. synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/action.py +10 -0
  194. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/__init__.py +28 -0
  195. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/action.py +145 -0
  196. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/enums.py +269 -0
  197. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/exceptions.py +14 -0
  198. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/factory.py +76 -0
  199. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/models.py +97 -0
  200. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/orchestrator.py +250 -0
  201. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/run.py +64 -0
  202. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/__init__.py +17 -0
  203. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/annotation.py +287 -0
  204. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py +170 -0
  205. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py +83 -0
  206. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py +87 -0
  207. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py +127 -0
  208. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/validation.py +143 -0
  209. synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py +966 -0
  210. synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml +19 -0
  211. synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py +40 -0
  212. synapse_sdk/plugins/categories/upload/actions/upload/__init__.py +19 -0
  213. synapse_sdk/plugins/categories/upload/actions/upload/action.py +232 -0
  214. synapse_sdk/plugins/categories/upload/actions/upload/context.py +185 -0
  215. synapse_sdk/plugins/categories/upload/actions/upload/enums.py +471 -0
  216. synapse_sdk/plugins/categories/upload/actions/upload/exceptions.py +36 -0
  217. synapse_sdk/plugins/categories/upload/actions/upload/factory.py +138 -0
  218. synapse_sdk/plugins/categories/upload/actions/upload/models.py +203 -0
  219. synapse_sdk/plugins/categories/upload/actions/upload/orchestrator.py +183 -0
  220. synapse_sdk/plugins/categories/upload/actions/upload/registry.py +113 -0
  221. synapse_sdk/plugins/categories/upload/actions/upload/run.py +179 -0
  222. synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py +1 -0
  223. synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py +107 -0
  224. synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py +62 -0
  225. synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py +63 -0
  226. synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py +84 -0
  227. synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py +82 -0
  228. synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py +235 -0
  229. synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py +203 -0
  230. synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py +97 -0
  231. synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py +71 -0
  232. synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py +1 -0
  233. synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py +82 -0
  234. synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py +1 -0
  235. synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py +39 -0
  236. synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py +29 -0
  237. synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py +1 -0
  238. synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py +258 -0
  239. synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py +281 -0
  240. synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/__init__.py +1 -0
  241. synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/excel.py +174 -0
  242. synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/none.py +16 -0
  243. synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/__init__.py +1 -0
  244. synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/sync.py +84 -0
  245. synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/__init__.py +1 -0
  246. synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py +60 -0
  247. synapse_sdk/plugins/categories/upload/actions/upload/utils.py +250 -0
  248. synapse_sdk/plugins/categories/upload/templates/README.md +470 -0
  249. synapse_sdk/plugins/categories/upload/templates/config.yaml +29 -2
  250. synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py +294 -0
  251. synapse_sdk/plugins/categories/upload/templates/plugin/upload.py +88 -30
  252. synapse_sdk/plugins/models.py +122 -16
  253. synapse_sdk/plugins/templates/plugin-config-schema.json +406 -0
  254. synapse_sdk/plugins/templates/schema.json +491 -0
  255. synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/requirements.txt +1 -1
  256. synapse_sdk/plugins/utils/__init__.py +46 -0
  257. synapse_sdk/plugins/utils/actions.py +119 -0
  258. synapse_sdk/plugins/utils/config.py +203 -0
  259. synapse_sdk/plugins/{utils.py → utils/legacy.py} +26 -46
  260. synapse_sdk/plugins/utils/ray_gcs.py +66 -0
  261. synapse_sdk/plugins/utils/registry.py +58 -0
  262. synapse_sdk/shared/__init__.py +25 -0
  263. synapse_sdk/shared/enums.py +93 -0
  264. synapse_sdk/utils/converters/__init__.py +240 -0
  265. synapse_sdk/utils/converters/coco/__init__.py +0 -0
  266. synapse_sdk/utils/converters/coco/from_dm.py +322 -0
  267. synapse_sdk/utils/converters/coco/to_dm.py +215 -0
  268. synapse_sdk/utils/converters/dm/__init__.py +56 -0
  269. synapse_sdk/utils/converters/dm/from_v1.py +627 -0
  270. synapse_sdk/utils/converters/dm/to_v1.py +367 -0
  271. synapse_sdk/utils/converters/pascal/__init__.py +0 -0
  272. synapse_sdk/utils/converters/pascal/from_dm.py +244 -0
  273. synapse_sdk/utils/converters/pascal/to_dm.py +214 -0
  274. synapse_sdk/utils/converters/yolo/__init__.py +0 -0
  275. synapse_sdk/utils/converters/yolo/from_dm.py +384 -0
  276. synapse_sdk/utils/converters/yolo/to_dm.py +267 -0
  277. synapse_sdk/utils/dataset.py +46 -0
  278. synapse_sdk/utils/encryption.py +158 -0
  279. synapse_sdk/utils/file/__init__.py +39 -0
  280. synapse_sdk/utils/file/archive.py +32 -0
  281. synapse_sdk/utils/file/checksum.py +56 -0
  282. synapse_sdk/utils/file/chunking.py +31 -0
  283. synapse_sdk/utils/file/download.py +385 -0
  284. synapse_sdk/utils/file/encoding.py +40 -0
  285. synapse_sdk/utils/file/io.py +22 -0
  286. synapse_sdk/utils/file/video/__init__.py +29 -0
  287. synapse_sdk/utils/file/video/transcode.py +307 -0
  288. synapse_sdk/utils/{file.py → file.py.backup} +84 -2
  289. synapse_sdk/utils/http.py +138 -0
  290. synapse_sdk/utils/network.py +293 -0
  291. synapse_sdk/utils/storage/__init__.py +36 -2
  292. synapse_sdk/utils/storage/providers/__init__.py +141 -0
  293. synapse_sdk/utils/storage/providers/file_system.py +134 -0
  294. synapse_sdk/utils/storage/providers/http.py +190 -0
  295. synapse_sdk/utils/storage/providers/s3.py +54 -6
  296. synapse_sdk/utils/storage/providers/sftp.py +31 -0
  297. synapse_sdk/utils/storage/registry.py +6 -0
  298. synapse_sdk-2025.11.7.dist-info/METADATA +122 -0
  299. synapse_sdk-2025.11.7.dist-info/RECORD +386 -0
  300. {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/WHEEL +1 -1
  301. synapse_sdk/clients/backend/dataset.py +0 -102
  302. synapse_sdk/plugins/categories/upload/actions/upload.py +0 -293
  303. synapse_sdk-1.0.0a35.dist-info/METADATA +0 -47
  304. synapse_sdk-1.0.0a35.dist-info/RECORD +0 -137
  305. {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/entry_points.txt +0 -0
  306. {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/licenses/LICENSE +0 -0
  307. {synapse_sdk-1.0.0a35.dist-info → synapse_sdk-2025.11.7.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,471 @@
1
+ from enum import Enum
2
+
3
+ from synapse_sdk.shared.enums import Context
4
+
5
+
6
+ class UploadStatus(str, Enum):
7
+ """Upload processing status enumeration.
8
+
9
+ Defines the possible states for upload operations, data files, and data units
10
+ throughout the upload process.
11
+
12
+ Attributes:
13
+ SUCCESS: Upload completed successfully
14
+ FAILED: Upload failed with errors
15
+ """
16
+
17
+ SUCCESS = 'success'
18
+ FAILED = 'failed'
19
+
20
+
21
+ class LogCode(str, Enum):
22
+ """Type-safe logging codes for upload operations.
23
+
24
+ Enumeration of all possible log events during upload processing. Each code
25
+ corresponds to a specific event or error state with predefined message
26
+ templates and log levels.
27
+
28
+ The codes are organized by category:
29
+ - Validation codes (VALIDATION_FAILED, STORAGE_VALIDATION_FAILED, etc.)
30
+ - File processing codes (NO_FILES_FOUND, FILES_DISCOVERED, etc.)
31
+ - Excel processing codes (EXCEL_SECURITY_VIOLATION, EXCEL_PARSING_ERROR, etc.)
32
+ - Progress tracking codes (UPLOADING_DATA_FILES, GENERATING_DATA_UNITS, etc.)
33
+
34
+ Each code maps to a configuration in LOG_MESSAGES with message template
35
+ and appropriate log level.
36
+ """
37
+
38
+ STORAGE_VALIDATION_FAILED = 'STORAGE_VALIDATION_FAILED'
39
+ COLLECTION_VALIDATION_FAILED = 'COLLECTION_VALIDATION_FAILED'
40
+ PROJECT_VALIDATION_FAILED = 'PROJECT_VALIDATION_FAILED'
41
+ VALIDATION_FAILED = 'VALIDATION_FAILED'
42
+ NO_FILES_FOUND = 'NO_FILES_FOUND'
43
+ NO_FILES_UPLOADED = 'NO_FILES_UPLOADED'
44
+ NO_DATA_UNITS_GENERATED = 'NO_DATA_UNITS_GENERATED'
45
+ NO_TYPE_DIRECTORIES = 'NO_TYPE_DIRECTORIES'
46
+ EXCEL_SECURITY_VIOLATION = 'EXCEL_SECURITY_VIOLATION'
47
+ EXCEL_PARSING_ERROR = 'EXCEL_PARSING_ERROR'
48
+ EXCEL_METADATA_LOADED = 'EXCEL_METADATA_LOADED'
49
+ UPLOADING_DATA_FILES = 'UPLOADING_DATA_FILES'
50
+ GENERATING_DATA_UNITS = 'GENERATING_DATA_UNITS'
51
+ IMPORT_COMPLETED = 'IMPORT_COMPLETED'
52
+ TYPE_DIRECTORIES_FOUND = 'TYPE_DIRECTORIES_FOUND'
53
+ TYPE_STRUCTURE_DETECTED = 'TYPE_STRUCTURE_DETECTED'
54
+ FILES_DISCOVERED = 'FILES_DISCOVERED'
55
+ NO_FILES_FOUND_WARNING = 'NO_FILES_FOUND_WARNING'
56
+ FILE_UPLOAD_FAILED = 'FILE_UPLOAD_FAILED'
57
+ DATA_UNIT_BATCH_FAILED = 'DATA_UNIT_BATCH_FAILED'
58
+ FILENAME_TOO_LONG = 'FILENAME_TOO_LONG'
59
+ MISSING_REQUIRED_FILES = 'MISSING_REQUIRED_FILES'
60
+ EXCEL_FILE_NOT_FOUND = 'EXCEL_FILE_NOT_FOUND'
61
+ EXCEL_FILE_VALIDATION_STARTED = 'EXCEL_FILE_VALIDATION_STARTED'
62
+ EXCEL_WORKBOOK_LOADED = 'EXCEL_WORKBOOK_LOADED'
63
+ FILE_ORGANIZATION_STARTED = 'FILE_ORGANIZATION_STARTED'
64
+ BATCH_PROCESSING_STARTED = 'BATCH_PROCESSING_STARTED'
65
+ EXCEL_SECURITY_VALIDATION_STARTED = 'EXCEL_SECURITY_VALIDATION_STARTED'
66
+ EXCEL_MEMORY_ESTIMATION = 'EXCEL_MEMORY_ESTIMATION'
67
+ EXCEL_FILE_NOT_FOUND_PATH = 'EXCEL_FILE_NOT_FOUND_PATH'
68
+ EXCEL_SECURITY_VALIDATION_FAILED = 'EXCEL_SECURITY_VALIDATION_FAILED'
69
+ EXCEL_PARSING_FAILED = 'EXCEL_PARSING_FAILED'
70
+ EXCEL_INVALID_FILE_FORMAT = 'EXCEL_INVALID_FILE_FORMAT'
71
+ EXCEL_FILE_TOO_LARGE = 'EXCEL_FILE_TOO_LARGE'
72
+ EXCEL_FILE_ACCESS_ERROR = 'EXCEL_FILE_ACCESS_ERROR'
73
+ EXCEL_UNEXPECTED_ERROR = 'EXCEL_UNEXPECTED_ERROR'
74
+ # Excel path resolution codes (from HEAD)
75
+ EXCEL_PATH_RESOLVED_STORAGE = 'EXCEL_PATH_RESOLVED_STORAGE'
76
+ EXCEL_PATH_RESOLUTION_FAILED = 'EXCEL_PATH_RESOLUTION_FAILED'
77
+ EXCEL_PATH_RESOLUTION_ERROR = 'EXCEL_PATH_RESOLUTION_ERROR'
78
+ # Asset path codes
79
+ ASSET_PATH_ACCESS_ERROR = 'ASSET_PATH_ACCESS_ERROR'
80
+ ASSET_PATH_NOT_FOUND = 'ASSET_PATH_NOT_FOUND'
81
+ # Step lifecycle codes
82
+ STEP_STARTING = 'STEP_STARTING'
83
+ STEP_COMPLETED = 'STEP_COMPLETED'
84
+ STEP_SKIPPED = 'STEP_SKIPPED'
85
+ STEP_ERROR = 'STEP_ERROR'
86
+ # Rollback codes
87
+ ROLLBACK_INITIALIZATION = 'ROLLBACK_INITIALIZATION'
88
+ ROLLBACK_DATA_UNIT_GENERATION = 'ROLLBACK_DATA_UNIT_GENERATION'
89
+ ROLLBACK_FILE_VALIDATION = 'ROLLBACK_FILE_VALIDATION'
90
+ ROLLBACK_FILE_UPLOADS = 'ROLLBACK_FILE_UPLOADS'
91
+ ROLLBACK_COLLECTION_ANALYSIS = 'ROLLBACK_COLLECTION_ANALYSIS'
92
+ ROLLBACK_FILE_ORGANIZATION = 'ROLLBACK_FILE_ORGANIZATION'
93
+ ROLLBACK_CLEANUP = 'ROLLBACK_CLEANUP'
94
+ # Metadata processing codes
95
+ NO_METADATA_STRATEGY = 'NO_METADATA_STRATEGY'
96
+ METADATA_FILE_ATTRIBUTE_PROCESSING = 'METADATA_FILE_ATTRIBUTE_PROCESSING'
97
+ METADATA_TEMP_FILE_CLEANUP = 'METADATA_TEMP_FILE_CLEANUP'
98
+ METADATA_TEMP_FILE_CLEANUP_FAILED = 'METADATA_TEMP_FILE_CLEANUP_FAILED'
99
+ METADATA_BASE64_DECODED = 'METADATA_BASE64_DECODED'
100
+ METADATA_BASE64_DECODE_FAILED = 'METADATA_BASE64_DECODE_FAILED'
101
+ # Multi-path mode codes
102
+ MULTI_PATH_MODE_ENABLED = 'MULTI_PATH_MODE_ENABLED'
103
+ OPTIONAL_SPEC_SKIPPED = 'OPTIONAL_SPEC_SKIPPED'
104
+ DISCOVERING_FILES_FOR_ASSET = 'DISCOVERING_FILES_FOR_ASSET'
105
+ NO_FILES_FOUND_FOR_ASSET = 'NO_FILES_FOUND_FOR_ASSET'
106
+ FILES_FOUND_FOR_ASSET = 'FILES_FOUND_FOR_ASSET'
107
+ ORGANIZING_FILES_MULTI_PATH = 'ORGANIZING_FILES_MULTI_PATH'
108
+ TYPE_DIRECTORIES_MULTI_PATH = 'TYPE_DIRECTORIES_MULTI_PATH'
109
+ DATA_UNITS_CREATED_FROM_FILES = 'DATA_UNITS_CREATED_FROM_FILES'
110
+ # Cleanup codes
111
+ CLEANUP_WARNING = 'CLEANUP_WARNING'
112
+ CLEANUP_TEMP_DIR_SUCCESS = 'CLEANUP_TEMP_DIR_SUCCESS'
113
+ CLEANUP_TEMP_DIR_FAILED = 'CLEANUP_TEMP_DIR_FAILED'
114
+ # Workflow error codes
115
+ UPLOAD_WORKFLOW_FAILED = 'UPLOAD_WORKFLOW_FAILED'
116
+ UNKNOWN_LOG_CODE = 'UNKNOWN_LOG_CODE'
117
+ # Orchestrator workflow codes
118
+ WORKFLOW_STARTING = 'WORKFLOW_STARTING'
119
+ WORKFLOW_COMPLETED = 'WORKFLOW_COMPLETED'
120
+ WORKFLOW_FAILED = 'WORKFLOW_FAILED'
121
+ STEP_FAILED = 'STEP_FAILED'
122
+ STEP_EXCEPTION = 'STEP_EXCEPTION'
123
+ STEP_TRACEBACK = 'STEP_TRACEBACK'
124
+ ROLLBACK_STARTING = 'ROLLBACK_STARTING'
125
+ ROLLBACK_COMPLETED = 'ROLLBACK_COMPLETED'
126
+ STEP_ROLLBACK = 'STEP_ROLLBACK'
127
+ ROLLBACK_ERROR = 'ROLLBACK_ERROR'
128
+ # Extension filtering codes
129
+ FILES_FILTERED_BY_EXTENSION = 'FILES_FILTERED_BY_EXTENSION'
130
+
131
+
132
+ LOG_MESSAGES = {
133
+ LogCode.STORAGE_VALIDATION_FAILED: {
134
+ 'message': 'Storage validation failed.',
135
+ 'level': Context.DANGER,
136
+ },
137
+ LogCode.COLLECTION_VALIDATION_FAILED: {
138
+ 'message': 'Collection validation failed.',
139
+ 'level': Context.DANGER,
140
+ },
141
+ LogCode.PROJECT_VALIDATION_FAILED: {
142
+ 'message': 'Project validation failed.',
143
+ 'level': Context.DANGER,
144
+ },
145
+ LogCode.VALIDATION_FAILED: {
146
+ 'message': 'Validation failed.',
147
+ 'level': Context.DANGER,
148
+ },
149
+ LogCode.NO_FILES_FOUND: {
150
+ 'message': 'Files not found on the path.',
151
+ 'level': Context.WARNING,
152
+ },
153
+ LogCode.NO_FILES_UPLOADED: {
154
+ 'message': 'No files were uploaded.',
155
+ 'level': Context.WARNING,
156
+ },
157
+ LogCode.NO_DATA_UNITS_GENERATED: {
158
+ 'message': 'No data units were generated.',
159
+ 'level': Context.WARNING,
160
+ },
161
+ LogCode.NO_TYPE_DIRECTORIES: {
162
+ 'message': 'No type-based directory structure found.',
163
+ 'level': Context.INFO,
164
+ },
165
+ LogCode.EXCEL_SECURITY_VIOLATION: {
166
+ 'message': 'Excel security validation failed: {}',
167
+ 'level': Context.DANGER,
168
+ },
169
+ LogCode.EXCEL_PARSING_ERROR: {
170
+ 'message': 'Excel parsing failed: {}',
171
+ 'level': Context.DANGER,
172
+ },
173
+ LogCode.EXCEL_METADATA_LOADED: {
174
+ 'message': 'Excel metadata loaded for {} files',
175
+ 'level': None,
176
+ },
177
+ LogCode.UPLOADING_DATA_FILES: {
178
+ 'message': 'Uploading data files...',
179
+ 'level': None,
180
+ },
181
+ LogCode.GENERATING_DATA_UNITS: {
182
+ 'message': 'Generating data units...',
183
+ 'level': None,
184
+ },
185
+ LogCode.IMPORT_COMPLETED: {
186
+ 'message': 'Import completed.',
187
+ 'level': None,
188
+ },
189
+ LogCode.TYPE_DIRECTORIES_FOUND: {
190
+ 'message': 'Found type directories: {}',
191
+ 'level': None,
192
+ },
193
+ LogCode.TYPE_STRUCTURE_DETECTED: {
194
+ 'message': 'Detected type-based directory structure',
195
+ 'level': None,
196
+ },
197
+ LogCode.FILES_DISCOVERED: {
198
+ 'message': 'Discovered {} files',
199
+ 'level': None,
200
+ },
201
+ LogCode.NO_FILES_FOUND_WARNING: {
202
+ 'message': 'No files found.',
203
+ 'level': Context.WARNING,
204
+ },
205
+ LogCode.FILE_UPLOAD_FAILED: {
206
+ 'message': 'Failed to upload file: {}',
207
+ 'level': Context.DANGER,
208
+ },
209
+ LogCode.DATA_UNIT_BATCH_FAILED: {
210
+ 'message': 'Failed to create data units batch: {}',
211
+ 'level': Context.DANGER,
212
+ },
213
+ LogCode.FILENAME_TOO_LONG: {
214
+ 'message': 'Skipping file with overly long name: {}...',
215
+ 'level': Context.WARNING,
216
+ },
217
+ LogCode.MISSING_REQUIRED_FILES: {
218
+ 'message': '{} missing required files: {}',
219
+ 'level': Context.WARNING,
220
+ },
221
+ LogCode.EXCEL_FILE_NOT_FOUND: {
222
+ 'message': 'Excel metadata file not found: {}',
223
+ 'level': Context.WARNING,
224
+ },
225
+ LogCode.EXCEL_FILE_VALIDATION_STARTED: {
226
+ 'message': 'Excel file validation started',
227
+ 'level': Context.INFO,
228
+ },
229
+ LogCode.EXCEL_WORKBOOK_LOADED: {
230
+ 'message': 'Excel workbook loaded successfully',
231
+ 'level': Context.INFO,
232
+ },
233
+ LogCode.FILE_ORGANIZATION_STARTED: {
234
+ 'message': 'File organization started',
235
+ 'level': Context.INFO,
236
+ },
237
+ LogCode.BATCH_PROCESSING_STARTED: {
238
+ 'message': 'Batch processing started: {} batches of {} items each',
239
+ 'level': Context.INFO,
240
+ },
241
+ LogCode.EXCEL_SECURITY_VALIDATION_STARTED: {
242
+ 'message': 'Excel security validation started for file size: {} bytes',
243
+ 'level': Context.INFO,
244
+ },
245
+ LogCode.EXCEL_MEMORY_ESTIMATION: {
246
+ 'message': 'Excel memory estimation: {} bytes (file) * 3 = {} bytes (estimated)',
247
+ 'level': Context.INFO,
248
+ },
249
+ LogCode.EXCEL_FILE_NOT_FOUND_PATH: {
250
+ 'message': 'Excel metadata file not found',
251
+ 'level': Context.WARNING,
252
+ },
253
+ LogCode.EXCEL_SECURITY_VALIDATION_FAILED: {
254
+ 'message': 'Excel security validation failed: {}',
255
+ 'level': Context.DANGER,
256
+ },
257
+ LogCode.EXCEL_PARSING_FAILED: {
258
+ 'message': 'Excel parsing failed: {}',
259
+ 'level': Context.DANGER,
260
+ },
261
+ LogCode.EXCEL_INVALID_FILE_FORMAT: {
262
+ 'message': 'Invalid Excel file format: {}',
263
+ 'level': Context.DANGER,
264
+ },
265
+ LogCode.EXCEL_FILE_TOO_LARGE: {
266
+ 'message': 'Excel file too large to process (memory limit exceeded)',
267
+ 'level': Context.DANGER,
268
+ },
269
+ LogCode.EXCEL_FILE_ACCESS_ERROR: {
270
+ 'message': 'File access error reading excel metadata: {}',
271
+ 'level': Context.DANGER,
272
+ },
273
+ LogCode.EXCEL_UNEXPECTED_ERROR: {
274
+ 'message': 'Unexpected error reading excel metadata: {}',
275
+ 'level': Context.DANGER,
276
+ },
277
+ # Excel path resolution messages (from HEAD)
278
+ LogCode.EXCEL_PATH_RESOLVED_STORAGE: {
279
+ 'message': 'Resolved Excel metadata path relative to storage: {}',
280
+ 'level': Context.INFO,
281
+ },
282
+ LogCode.EXCEL_PATH_RESOLUTION_FAILED: {
283
+ 'message': 'Storage path resolution failed ({}): {} - trying other strategies',
284
+ 'level': Context.INFO,
285
+ },
286
+ LogCode.EXCEL_PATH_RESOLUTION_ERROR: {
287
+ 'message': 'Unexpected error resolving storage path ({}): {} - trying other strategies',
288
+ 'level': Context.WARNING,
289
+ },
290
+ # Asset path messages
291
+ LogCode.ASSET_PATH_ACCESS_ERROR: {
292
+ 'message': 'Error accessing path for {}: {}',
293
+ 'level': Context.WARNING,
294
+ },
295
+ LogCode.ASSET_PATH_NOT_FOUND: {
296
+ 'message': 'Path does not exist for {}: {}',
297
+ 'level': Context.WARNING,
298
+ },
299
+ # Step lifecycle messages
300
+ LogCode.STEP_STARTING: {
301
+ 'message': 'Starting step: {}',
302
+ 'level': Context.INFO,
303
+ },
304
+ LogCode.STEP_COMPLETED: {
305
+ 'message': 'Completed step: {}',
306
+ 'level': Context.INFO,
307
+ },
308
+ LogCode.STEP_SKIPPED: {
309
+ 'message': 'Skipped step: {}',
310
+ 'level': Context.INFO,
311
+ },
312
+ LogCode.STEP_ERROR: {
313
+ 'message': 'Error in step {}: {}',
314
+ 'level': Context.DANGER,
315
+ },
316
+ # Rollback messages
317
+ LogCode.ROLLBACK_INITIALIZATION: {
318
+ 'message': 'Rolling back initialization step',
319
+ 'level': Context.INFO,
320
+ },
321
+ LogCode.ROLLBACK_DATA_UNIT_GENERATION: {
322
+ 'message': 'Rolled back data unit generation',
323
+ 'level': Context.INFO,
324
+ },
325
+ LogCode.ROLLBACK_FILE_VALIDATION: {
326
+ 'message': 'Rolled back file validation',
327
+ 'level': Context.INFO,
328
+ },
329
+ LogCode.ROLLBACK_FILE_UPLOADS: {
330
+ 'message': 'Rolled back file uploads',
331
+ 'level': Context.INFO,
332
+ },
333
+ LogCode.ROLLBACK_COLLECTION_ANALYSIS: {
334
+ 'message': 'Rolled back collection analysis',
335
+ 'level': Context.INFO,
336
+ },
337
+ LogCode.ROLLBACK_FILE_ORGANIZATION: {
338
+ 'message': 'Rolled back file organization',
339
+ 'level': Context.INFO,
340
+ },
341
+ LogCode.ROLLBACK_CLEANUP: {
342
+ 'message': 'Cleanup step rollback - no action needed',
343
+ 'level': Context.INFO,
344
+ },
345
+ # Metadata processing messages
346
+ LogCode.NO_METADATA_STRATEGY: {
347
+ 'message': 'No metadata strategy configured - skipping metadata processing',
348
+ 'level': Context.INFO,
349
+ },
350
+ LogCode.METADATA_FILE_ATTRIBUTE_PROCESSING: {
351
+ 'message': 'Processing metadata for file attribute: {}',
352
+ 'level': Context.INFO,
353
+ },
354
+ LogCode.METADATA_TEMP_FILE_CLEANUP: {
355
+ 'message': 'Cleaned up temporary Excel file: {}',
356
+ 'level': Context.INFO,
357
+ },
358
+ LogCode.METADATA_TEMP_FILE_CLEANUP_FAILED: {
359
+ 'message': 'Failed to clean up temporary file {}: {}',
360
+ 'level': Context.WARNING,
361
+ },
362
+ LogCode.METADATA_BASE64_DECODED: {
363
+ 'message': 'Decoded base64 Excel metadata to temporary file: {}',
364
+ 'level': Context.INFO,
365
+ },
366
+ LogCode.METADATA_BASE64_DECODE_FAILED: {
367
+ 'message': 'Failed to decode base64 Excel metadata: {}',
368
+ 'level': Context.DANGER,
369
+ },
370
+ # Multi-path mode messages
371
+ LogCode.MULTI_PATH_MODE_ENABLED: {
372
+ 'message': 'Using multi-path mode with {} asset configurations',
373
+ 'level': Context.INFO,
374
+ },
375
+ LogCode.OPTIONAL_SPEC_SKIPPED: {
376
+ 'message': 'Skipping optional spec {}: no asset path configured',
377
+ 'level': Context.INFO,
378
+ },
379
+ LogCode.DISCOVERING_FILES_FOR_ASSET: {
380
+ 'message': 'Discovering files for {} at {} (recursive={})',
381
+ 'level': Context.INFO,
382
+ },
383
+ LogCode.NO_FILES_FOUND_FOR_ASSET: {
384
+ 'message': 'No files found for {}',
385
+ 'level': Context.WARNING,
386
+ },
387
+ LogCode.FILES_FOUND_FOR_ASSET: {
388
+ 'message': 'Found {} files for {}',
389
+ 'level': Context.INFO,
390
+ },
391
+ LogCode.ORGANIZING_FILES_MULTI_PATH: {
392
+ 'message': 'Organizing {} files across {} specs',
393
+ 'level': Context.INFO,
394
+ },
395
+ LogCode.TYPE_DIRECTORIES_MULTI_PATH: {
396
+ 'message': 'Type directories: {}',
397
+ 'level': Context.INFO,
398
+ },
399
+ LogCode.DATA_UNITS_CREATED_FROM_FILES: {
400
+ 'message': 'Created {} data units from {} files',
401
+ 'level': Context.INFO,
402
+ },
403
+ # Cleanup messages
404
+ LogCode.CLEANUP_WARNING: {
405
+ 'message': 'Cleanup warning: {}',
406
+ 'level': Context.WARNING,
407
+ },
408
+ LogCode.CLEANUP_TEMP_DIR_SUCCESS: {
409
+ 'message': 'Cleaned up temporary directory: {}',
410
+ 'level': Context.INFO,
411
+ },
412
+ LogCode.CLEANUP_TEMP_DIR_FAILED: {
413
+ 'message': 'Failed to cleanup temporary directory: {}',
414
+ 'level': Context.WARNING,
415
+ },
416
+ # Workflow error messages
417
+ LogCode.UPLOAD_WORKFLOW_FAILED: {
418
+ 'message': 'Upload workflow failed: {}',
419
+ 'level': Context.DANGER,
420
+ },
421
+ LogCode.UNKNOWN_LOG_CODE: {
422
+ 'message': 'Unknown log code: {}',
423
+ 'level': Context.WARNING,
424
+ },
425
+ # Orchestrator workflow messages
426
+ LogCode.WORKFLOW_STARTING: {
427
+ 'message': 'Starting upload workflow with {} steps: {}',
428
+ 'level': Context.INFO,
429
+ },
430
+ LogCode.WORKFLOW_COMPLETED: {
431
+ 'message': 'Upload workflow completed successfully',
432
+ 'level': Context.INFO,
433
+ },
434
+ LogCode.WORKFLOW_FAILED: {
435
+ 'message': 'Upload workflow failed: {}',
436
+ 'level': Context.DANGER,
437
+ },
438
+ LogCode.STEP_FAILED: {
439
+ 'message': "Step '{}' failed: {}",
440
+ 'level': Context.DANGER,
441
+ },
442
+ LogCode.STEP_EXCEPTION: {
443
+ 'message': "Exception in step '{}': {}",
444
+ 'level': Context.DANGER,
445
+ },
446
+ LogCode.STEP_TRACEBACK: {
447
+ 'message': 'Traceback: {}',
448
+ 'level': Context.DANGER,
449
+ },
450
+ LogCode.ROLLBACK_STARTING: {
451
+ 'message': 'Starting rollback of {} executed steps',
452
+ 'level': Context.WARNING,
453
+ },
454
+ LogCode.ROLLBACK_COMPLETED: {
455
+ 'message': 'Rollback completed',
456
+ 'level': Context.INFO,
457
+ },
458
+ LogCode.STEP_ROLLBACK: {
459
+ 'message': 'Rolling back step: {}',
460
+ 'level': Context.INFO,
461
+ },
462
+ LogCode.ROLLBACK_ERROR: {
463
+ 'message': "Error rolling back step '{}': {}",
464
+ 'level': Context.WARNING,
465
+ },
466
+ # Extension filtering messages
467
+ LogCode.FILES_FILTERED_BY_EXTENSION: {
468
+ 'message': 'Filtered {} {} files with unavailable extensions: {} (allowed: {})',
469
+ 'level': Context.WARNING,
470
+ },
471
+ }
@@ -0,0 +1,36 @@
1
+ class ExcelSecurityError(Exception):
2
+ """Exception raised when Excel file security validation fails.
3
+
4
+ This exception is raised when an Excel file violates security constraints
5
+ such as file size limits, memory usage limits, or contains potentially
6
+ dangerous content.
7
+
8
+ Used during Excel metadata processing to enforce security policies
9
+ and prevent processing of files that could pose security risks.
10
+
11
+ Example:
12
+ >>> if file_size > max_size:
13
+ ... raise ExcelSecurityError(f"File size {file_size} exceeds limit {max_size}")
14
+ """
15
+
16
+ pass
17
+
18
+
19
+ class ExcelParsingError(Exception):
20
+ """Exception raised when Excel file parsing encounters errors.
21
+
22
+ This exception is raised when an Excel file cannot be parsed due to
23
+ format issues, corruption, or other parsing-related problems that
24
+ prevent successful metadata extraction.
25
+
26
+ Used during Excel metadata loading to distinguish parsing errors
27
+ from security violations or other types of errors.
28
+
29
+ Example:
30
+ >>> try:
31
+ ... workbook = load_workbook(excel_file)
32
+ ... except InvalidFileException as e:
33
+ ... raise ExcelParsingError(f"Failed to parse Excel file: {e}")
34
+ """
35
+
36
+ pass
@@ -0,0 +1,138 @@
1
+ from typing import Any, Dict
2
+
3
+ from .strategies.base import (
4
+ DataUnitStrategy,
5
+ FileDiscoveryStrategy,
6
+ MetadataStrategy,
7
+ UploadStrategy,
8
+ ValidationStrategy,
9
+ )
10
+
11
+
12
+ class StrategyFactory:
13
+ """Factory for creating strategy instances based on configuration."""
14
+
15
+ def __init__(self):
16
+ self._validation_strategies = {}
17
+ self._file_discovery_strategies = {}
18
+ self._metadata_strategies = {}
19
+ self._upload_strategies = {}
20
+ self._data_unit_strategies = {}
21
+
22
+ def register_validation_strategy(self, name: str, strategy_class: type) -> None:
23
+ """Register a validation strategy class."""
24
+ self._validation_strategies[name] = strategy_class
25
+
26
+ def register_file_discovery_strategy(self, name: str, strategy_class: type) -> None:
27
+ """Register a file discovery strategy class."""
28
+ self._file_discovery_strategies[name] = strategy_class
29
+
30
+ def register_metadata_strategy(self, name: str, strategy_class: type) -> None:
31
+ """Register a metadata strategy class."""
32
+ self._metadata_strategies[name] = strategy_class
33
+
34
+ def register_upload_strategy(self, name: str, strategy_class: type) -> None:
35
+ """Register an upload strategy class."""
36
+ self._upload_strategies[name] = strategy_class
37
+
38
+ def register_data_unit_strategy(self, name: str, strategy_class: type) -> None:
39
+ """Register a data unit strategy class."""
40
+ self._data_unit_strategies[name] = strategy_class
41
+
42
+ def create_validation_strategy(self, params: Dict[str, Any], context=None) -> ValidationStrategy:
43
+ """Create validation strategy based on parameters."""
44
+ strategy_name = params.get('validation_strategy', 'default')
45
+
46
+ if strategy_name not in self._validation_strategies:
47
+ # Import default strategy if not registered
48
+ from .strategies.validation.default import DefaultValidationStrategy
49
+
50
+ self.register_validation_strategy('default', DefaultValidationStrategy)
51
+ strategy_name = 'default'
52
+
53
+ strategy_class = self._validation_strategies[strategy_name]
54
+ return strategy_class()
55
+
56
+ def create_file_discovery_strategy(self, params: Dict[str, Any], context=None) -> FileDiscoveryStrategy:
57
+ """Create file discovery strategy based on parameters."""
58
+ is_recursive = params.get('is_recursive', True)
59
+ strategy_name = 'recursive' if is_recursive else 'flat'
60
+
61
+ if strategy_name not in self._file_discovery_strategies:
62
+ # Import default strategies if not registered
63
+ if strategy_name == 'recursive':
64
+ from .strategies.file_discovery.recursive import RecursiveFileDiscoveryStrategy
65
+
66
+ self.register_file_discovery_strategy('recursive', RecursiveFileDiscoveryStrategy)
67
+ else:
68
+ from .strategies.file_discovery.flat import FlatFileDiscoveryStrategy
69
+
70
+ self.register_file_discovery_strategy('flat', FlatFileDiscoveryStrategy)
71
+
72
+ strategy_class = self._file_discovery_strategies[strategy_name]
73
+ return strategy_class()
74
+
75
+ def create_metadata_strategy(self, params: Dict[str, Any], context=None) -> MetadataStrategy:
76
+ """Create metadata strategy based on parameters."""
77
+ # Always use Excel strategy for metadata processing
78
+ # It will handle both specified paths and default meta.xlsx/meta.xls files
79
+ strategy_name = 'excel'
80
+
81
+ if strategy_name not in self._metadata_strategies:
82
+ from .strategies.metadata.excel import ExcelMetadataStrategy
83
+
84
+ self.register_metadata_strategy('excel', ExcelMetadataStrategy)
85
+
86
+ strategy_class = self._metadata_strategies[strategy_name]
87
+ return strategy_class()
88
+
89
+ def create_upload_strategy(self, params: Dict[str, Any], context=None) -> UploadStrategy:
90
+ """Create upload strategy (always uses synchronous upload for guaranteed ordering)."""
91
+ if context is None:
92
+ raise ValueError('Upload strategies require context parameter')
93
+
94
+ # Always use sync upload to guarantee file order
95
+ # This is critical for video frame extraction and PDF page extraction
96
+ strategy_name = 'sync'
97
+
98
+ if strategy_name not in self._upload_strategies:
99
+ from .strategies.upload.sync import SyncUploadStrategy
100
+
101
+ self.register_upload_strategy('sync', SyncUploadStrategy)
102
+
103
+ strategy_class = self._upload_strategies[strategy_name]
104
+ # Upload strategies always need context for client access
105
+ return strategy_class(context)
106
+
107
+ def create_data_unit_strategy(self, params: Dict[str, Any], context=None) -> DataUnitStrategy:
108
+ """Create data unit strategy based on parameters."""
109
+ if context is None:
110
+ raise ValueError('Data unit strategies require context parameter')
111
+
112
+ batch_size = params.get('creating_data_unit_batch_size', 1)
113
+ strategy_name = 'batch' if batch_size > 1 else 'single'
114
+
115
+ if strategy_name not in self._data_unit_strategies:
116
+ # Import default strategies if not registered
117
+ if strategy_name == 'batch':
118
+ from .strategies.data_unit.batch import BatchDataUnitStrategy
119
+
120
+ self.register_data_unit_strategy('batch', BatchDataUnitStrategy)
121
+ else:
122
+ from .strategies.data_unit.single import SingleDataUnitStrategy
123
+
124
+ self.register_data_unit_strategy('single', SingleDataUnitStrategy)
125
+
126
+ strategy_class = self._data_unit_strategies[strategy_name]
127
+ # Data unit strategies always need context for client access
128
+ return strategy_class(context)
129
+
130
+ def get_available_strategies(self) -> Dict[str, list]:
131
+ """Get all available strategy types and their registered names."""
132
+ return {
133
+ 'validation': list(self._validation_strategies.keys()),
134
+ 'file_discovery': list(self._file_discovery_strategies.keys()),
135
+ 'metadata': list(self._metadata_strategies.keys()),
136
+ 'upload': list(self._upload_strategies.keys()),
137
+ 'data_unit': list(self._data_unit_strategies.keys()),
138
+ }