data-syncmaster 0.2.5__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/LICENSE.txt +1 -1
  2. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/PKG-INFO +36 -34
  3. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/README.rst +14 -13
  4. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/pyproject.toml +46 -67
  5. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/__init__.py +2 -2
  6. data_syncmaster-0.3.0/syncmaster/db/__init__.py +2 -0
  7. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/factory.py +1 -1
  8. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/__main__.py +1 -1
  9. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/env.py +14 -3
  10. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py +1 -1
  11. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py +1 -1
  12. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py +1 -1
  13. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py +1 -1
  14. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py +1 -1
  15. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py +1 -1
  16. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py +1 -1
  17. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py +1 -1
  18. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py +1 -1
  19. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py +1 -1
  20. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py +1 -1
  21. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py +1 -1
  22. data_syncmaster-0.3.0/syncmaster/db/migrations/versions/__init__.py +2 -0
  23. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/mixins/__init__.py +1 -1
  24. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/mixins/resource.py +1 -1
  25. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/mixins/timestamp.py +1 -1
  26. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/__init__.py +1 -1
  27. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/apscheduler_job.py +1 -1
  28. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/auth_data.py +1 -1
  29. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/base.py +1 -1
  30. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/connection.py +2 -1
  31. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/group.py +1 -1
  32. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/queue.py +1 -1
  33. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/run.py +1 -1
  34. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/transfer.py +1 -1
  35. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/user.py +1 -1
  36. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/__init__.py +1 -1
  37. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/base.py +1 -1
  38. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/connection.py +1 -1
  39. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/credentials_repository.py +1 -1
  40. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/group.py +1 -1
  41. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/queue.py +1 -1
  42. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/repository_with_owner.py +1 -1
  43. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/run.py +1 -1
  44. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/search.py +1 -5
  45. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/transfer.py +1 -1
  46. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/user.py +1 -1
  47. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/utils.py +1 -8
  48. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/utils.py +1 -1
  49. data_syncmaster-0.3.0/syncmaster/dto/__init__.py +2 -0
  50. data_syncmaster-0.3.0/syncmaster/dto/connections.py +231 -0
  51. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/runs.py +2 -2
  52. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/transfers.py +38 -7
  53. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/transfers_resources.py +1 -1
  54. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/transfers_strategy.py +1 -1
  55. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/__init__.py +1 -1
  56. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/base.py +1 -1
  57. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/registration.py +1 -1
  58. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/__init__.py +1 -1
  59. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/bad_request.py +1 -1
  60. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/invalid_request.py +1 -1
  61. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/not_authorized.py +1 -1
  62. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/__init__.py +1 -1
  63. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/auth.py +1 -1
  64. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/base.py +1 -1
  65. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/connection.py +1 -1
  66. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/credentials.py +1 -1
  67. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/group.py +1 -1
  68. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/queue.py +1 -1
  69. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/redirect.py +1 -1
  70. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/run.py +1 -1
  71. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/transfer.py +1 -1
  72. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/user.py +1 -1
  73. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/__init__.py +1 -1
  74. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/__main__.py +3 -3
  75. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/celery.py +1 -1
  76. data_syncmaster-0.3.0/syncmaster/scheduler/settings/__init__.py +76 -0
  77. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/transfer_fetcher.py +1 -1
  78. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/transfer_job_manager.py +3 -3
  79. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/utils.py +1 -1
  80. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/__init__.py +1 -1
  81. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/ping.py +1 -1
  82. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/__init__.py +2 -12
  83. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/auth/__init__.py +1 -7
  84. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/basic.py +23 -0
  85. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py +31 -0
  86. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py +22 -0
  87. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py +32 -0
  88. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py +31 -0
  89. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py +25 -0
  90. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py +34 -0
  91. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/mixins.py +12 -0
  92. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/s3.py +23 -0
  93. data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/samba.py +24 -0
  94. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/auth/token.py +1 -1
  95. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connection_types.py +26 -7
  96. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection.py → data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/__init__.py +39 -31
  97. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/clickhouse.py +14 -19
  98. data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/connection_base.py +46 -0
  99. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/ftp.py +15 -17
  100. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/ftps.py +12 -17
  101. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/hdfs.py +14 -15
  102. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/hive.py +14 -15
  103. data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/iceberg.py +105 -0
  104. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/mssql.py +15 -19
  105. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/mysql.py +16 -20
  106. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/oracle.py +15 -20
  107. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/postgres.py +15 -19
  108. data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/s3.py +65 -0
  109. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/samba.py +7 -18
  110. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/sftp.py +14 -16
  111. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/webdav.py +15 -18
  112. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/file_formats.py +1 -1
  113. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/groups.py +5 -10
  114. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/page.py +1 -1
  115. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/queue.py +8 -9
  116. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/__init__.py +94 -0
  117. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/db.py +62 -0
  118. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/__init__.py +55 -0
  119. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/file/base.py +9 -28
  120. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/ftp.py +17 -0
  121. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/ftps.py +17 -0
  122. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/hdfs.py +17 -0
  123. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/s3.py +17 -0
  124. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/samba.py +17 -0
  125. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/sftp.py +17 -0
  126. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/webdav.py +17 -0
  127. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/file_format.py +1 -1
  128. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/resources.py +17 -0
  129. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/run.py +10 -11
  130. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/strategy.py +23 -0
  131. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/transformations/__init__.py +25 -0
  132. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py +2 -4
  133. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py +2 -4
  134. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py +2 -4
  135. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transformation_types.py +1 -1
  136. data_syncmaster-0.3.0/syncmaster/schemas/v1/types.py +8 -0
  137. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/users.py +1 -1
  138. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/__init__.py +5 -2
  139. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/__main__.py +1 -1
  140. data_syncmaster-0.3.0/syncmaster/server/api/__init__.py +2 -0
  141. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/monitoring.py +1 -1
  142. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/router.py +1 -1
  143. data_syncmaster-0.3.0/syncmaster/server/api/v1/__init__.py +2 -0
  144. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/auth.py +2 -2
  145. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/connections.py +18 -18
  146. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/groups.py +9 -9
  147. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/queue.py +7 -7
  148. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/router.py +1 -1
  149. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/runs.py +7 -7
  150. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/transfers.py +17 -17
  151. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/users.py +5 -5
  152. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/dependencies/__init__.py +1 -1
  153. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/dependencies/get_access_token.py +1 -1
  154. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/handler.py +1 -1
  155. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/__init__.py +1 -7
  156. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/cors.py +1 -1
  157. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/monitoring/__init__.py +1 -1
  158. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/monitoring/metrics.py +1 -1
  159. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/openapi.py +1 -1
  160. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/request_id.py +1 -1
  161. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/static_files.py +1 -1
  162. data_syncmaster-0.3.0/syncmaster/server/providers/__init__.py +2 -0
  163. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/__init__.py +5 -6
  164. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/base_provider.py +1 -1
  165. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/dummy_provider.py +5 -3
  166. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/keycloak_provider.py +23 -6
  167. data_syncmaster-0.3.0/syncmaster/server/providers/auth/oauth2_gateway_provider.py +116 -0
  168. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/scripts/export_openapi_schema.py +1 -1
  169. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/scripts/manage_superusers.py +32 -8
  170. data_syncmaster-0.3.0/syncmaster/server/services/__init__.py +2 -0
  171. data_syncmaster-0.3.0/syncmaster/server/services/get_user.py +61 -0
  172. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/services/unit_of_work.py +1 -1
  173. data_syncmaster-0.3.0/syncmaster/server/settings/__init__.py +70 -0
  174. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/auth/__init__.py +8 -6
  175. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/auth/dummy.py +7 -4
  176. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/auth/jwt.py +7 -4
  177. data_syncmaster-0.3.0/syncmaster/server/settings/auth/keycloak.py +135 -0
  178. data_syncmaster-0.3.0/syncmaster/server/settings/auth/oauth2_gateway.py +35 -0
  179. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/__init__.py +15 -10
  180. data_syncmaster-0.3.0/syncmaster/server/settings/server/cors.py +63 -0
  181. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/monitoring.py +16 -5
  182. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/openapi.py +48 -22
  183. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/request_id.py +8 -4
  184. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/static_files.py +12 -6
  185. data_syncmaster-0.3.0/syncmaster/server/utils/__init__.py +2 -0
  186. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/utils/jwt.py +1 -1
  187. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/utils/slug.py +1 -1
  188. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/__init__.py +6 -3
  189. data_syncmaster-0.3.0/syncmaster/settings/base.py +37 -0
  190. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/broker.py +7 -6
  191. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/credentials.py +5 -4
  192. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/database.py +8 -5
  193. data_syncmaster-0.3.0/syncmaster/settings/logging.py +288 -0
  194. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/__init__.py +1 -1
  195. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/base.py +1 -1
  196. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/celery.py +1 -1
  197. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/controller.py +39 -4
  198. data_syncmaster-0.3.0/syncmaster/worker/handlers/__init__.py +2 -0
  199. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/base.py +1 -1
  200. data_syncmaster-0.3.0/syncmaster/worker/handlers/db/__init__.py +2 -0
  201. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/base.py +1 -1
  202. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/clickhouse.py +8 -1
  203. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/hive.py +8 -1
  204. data_syncmaster-0.3.0/syncmaster/worker/handlers/db/iceberg.py +134 -0
  205. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/mssql.py +11 -1
  206. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/mysql.py +11 -1
  207. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/oracle.py +11 -1
  208. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/postgres.py +11 -1
  209. data_syncmaster-0.3.0/syncmaster/worker/handlers/file/__init__.py +2 -0
  210. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/base.py +1 -1
  211. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/ftp.py +13 -2
  212. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/ftps.py +13 -2
  213. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/hdfs.py +13 -2
  214. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/local_df.py +1 -1
  215. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/remote_df.py +1 -1
  216. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/s3.py +9 -1
  217. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/samba.py +13 -2
  218. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/sftp.py +13 -2
  219. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/webdav.py +12 -2
  220. data_syncmaster-0.3.0/syncmaster/worker/ivy2.py +59 -0
  221. data_syncmaster-0.3.0/syncmaster/worker/settings/__init__.py +95 -0
  222. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/settings/hwm_store.py +11 -9
  223. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/spark.py +33 -49
  224. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/transfer.py +3 -3
  225. data_syncmaster-0.2.5/syncmaster/db/__init__.py +0 -2
  226. data_syncmaster-0.2.5/syncmaster/db/migrations/versions/__init__.py +0 -2
  227. data_syncmaster-0.2.5/syncmaster/dto/__init__.py +0 -2
  228. data_syncmaster-0.2.5/syncmaster/dto/connections.py +0 -144
  229. data_syncmaster-0.2.5/syncmaster/scheduler/settings/__init__.py +0 -78
  230. data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/basic.py +0 -26
  231. data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/s3.py +0 -26
  232. data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/samba.py +0 -28
  233. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/__init__.py +0 -2
  234. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection_base.py +0 -30
  235. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/s3.py +0 -73
  236. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfer_types.py +0 -6
  237. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/__init__.py +0 -248
  238. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/db.py +0 -42
  239. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/__init__.py +0 -2
  240. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftp.py +0 -27
  241. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftps.py +0 -27
  242. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -27
  243. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/s3.py +0 -27
  244. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/samba.py +0 -27
  245. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/sftp.py +0 -27
  246. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/webdav.py +0 -27
  247. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/resources.py +0 -14
  248. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/strategy.py +0 -14
  249. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/transformations/__init__.py +0 -2
  250. data_syncmaster-0.2.5/syncmaster/schemas/v1/types.py +0 -7
  251. data_syncmaster-0.2.5/syncmaster/server/api/__init__.py +0 -2
  252. data_syncmaster-0.2.5/syncmaster/server/api/v1/__init__.py +0 -2
  253. data_syncmaster-0.2.5/syncmaster/server/middlewares/session.py +0 -17
  254. data_syncmaster-0.2.5/syncmaster/server/providers/__init__.py +0 -2
  255. data_syncmaster-0.2.5/syncmaster/server/services/__init__.py +0 -2
  256. data_syncmaster-0.2.5/syncmaster/server/services/get_user.py +0 -41
  257. data_syncmaster-0.2.5/syncmaster/server/settings/__init__.py +0 -61
  258. data_syncmaster-0.2.5/syncmaster/server/settings/auth/keycloak.py +0 -22
  259. data_syncmaster-0.2.5/syncmaster/server/settings/server/cors.py +0 -70
  260. data_syncmaster-0.2.5/syncmaster/server/settings/server/session.py +0 -75
  261. data_syncmaster-0.2.5/syncmaster/server/utils/__init__.py +0 -2
  262. data_syncmaster-0.2.5/syncmaster/settings/log/__init__.py +0 -110
  263. data_syncmaster-0.2.5/syncmaster/settings/log/colored.yml +0 -57
  264. data_syncmaster-0.2.5/syncmaster/settings/log/json.yml +0 -56
  265. data_syncmaster-0.2.5/syncmaster/settings/log/plain.yml +0 -57
  266. data_syncmaster-0.2.5/syncmaster/worker/handlers/__init__.py +0 -2
  267. data_syncmaster-0.2.5/syncmaster/worker/handlers/db/__init__.py +0 -2
  268. data_syncmaster-0.2.5/syncmaster/worker/handlers/file/__init__.py +0 -2
  269. data_syncmaster-0.2.5/syncmaster/worker/settings/__init__.py +0 -79
  270. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/README +0 -0
  271. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/alembic.ini +0 -0
  272. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/script.py.mako +0 -0
  273. {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/dependencies/stub.py +0 -0
@@ -1,4 +1,4 @@
1
- Copyright 2023-2025 MTS PJSC. All rights reserved.
1
+ Copyright 2023-present MTS PJSC. All rights reserved.
2
2
 
3
3
  Apache License
4
4
  Version 2.0, January 2004
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: data-syncmaster
3
- Version: 0.2.5
3
+ Version: 0.3.0
4
4
  Summary: Syncmaster REST API + Worker
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE.txt
7
7
  Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
8
- Author: DataOps.ETL
8
+ Author: MWS Data Bridge
9
9
  Author-email: onetools@mts.ru
10
10
  Requires-Python: >=3.11,<4.0
11
11
  Classifier: Development Status :: 3 - Alpha
@@ -27,31 +27,32 @@ Classifier: Typing :: Typed
27
27
  Provides-Extra: scheduler
28
28
  Provides-Extra: server
29
29
  Provides-Extra: worker
30
- Requires-Dist: alembic (>=1.15.2,<2.0.0) ; extra == "server"
31
- Requires-Dist: apscheduler (>=3.11.0,<4.0.0) ; extra == "scheduler"
32
- Requires-Dist: asgi-correlation-id (>=4.3.4,<5.0.0) ; extra == "server" or extra == "worker"
33
- Requires-Dist: asyncpg (>=0.30.0,<0.31.0) ; extra == "server" or extra == "scheduler"
34
- Requires-Dist: celery (>=5.5.0,<6.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
35
- Requires-Dist: coloredlogs ; extra == "server" or extra == "worker" or extra == "scheduler"
36
- Requires-Dist: fastapi (>=0.115.12,<0.119.0) ; extra == "server"
37
- Requires-Dist: horizon-hwm-store (>=1.1.2,<2.0.0) ; extra == "worker"
30
+ Requires-Dist: alembic (>=1.17.1,<2.0.0) ; extra == "server"
31
+ Requires-Dist: apscheduler (>=3.11.1,<4.0.0) ; extra == "scheduler"
32
+ Requires-Dist: asgi-correlation-id (>=4.3.4,<5.0.0)
33
+ Requires-Dist: asyncpg (>=0.30,<0.32) ; extra == "server" or extra == "scheduler"
34
+ Requires-Dist: celery (>=5.5.3,<6.0.0)
35
+ Requires-Dist: coloredlogs (>=15.0.1,<16.0.0)
36
+ Requires-Dist: fastapi (>=0.124.0,<0.125.0) ; extra == "server"
37
+ Requires-Dist: horizon-hwm-store (>=1.1.4,<2.0.0) ; extra == "worker"
38
38
  Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
39
39
  Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
40
- Requires-Dist: onetl[all] (>=0.13.5,<0.15.0) ; extra == "worker"
41
- Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0) ; extra == "server" or extra == "worker"
42
- Requires-Dist: pydantic (>=2.11.7,<3.0.0)
43
- Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
40
+ Requires-Dist: onetl[all] (>=0.15.0,<0.16.0) ; extra == "worker"
41
+ Requires-Dist: psycopg2-binary (>=2.9.11,<3.0.0) ; extra == "server" or extra == "worker"
42
+ Requires-Dist: pydantic (>=2.12.4,<3.0.0)
43
+ Requires-Dist: pydantic-settings (>=2.12.0,<3.0.0)
44
+ Requires-Dist: pydantic-settings-logging (>=0.1.1,<0.2.0)
44
45
  Requires-Dist: pyjwt (>=2.10.1,<3.0.0) ; extra == "server"
45
46
  Requires-Dist: pyspark (<4.0.0) ; extra == "worker"
46
- Requires-Dist: python-json-logger ; extra == "server" or extra == "worker" or extra == "scheduler"
47
- Requires-Dist: python-keycloak (>=5.3.1,<6.0.0) ; extra == "server"
47
+ Requires-Dist: python-json-logger (>=4.0.0,<5.0.0)
48
+ Requires-Dist: python-keycloak (>=5.8.1,<6.0.0) ; extra == "server"
48
49
  Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
49
- Requires-Dist: pyyaml ; extra == "server" or extra == "worker" or extra == "scheduler"
50
- Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
51
- Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.43.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
50
+ Requires-Dist: pyyaml (>=6.0.3,<7.0.0) ; extra == "server"
51
+ Requires-Dist: sqlalchemy (>=2.0.44,<3.0.0)
52
+ Requires-Dist: sqlalchemy-utils (>=0.42.0,<0.43.0)
52
53
  Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
53
- Requires-Dist: uuid6 (>=2025.0.0,<2026.0.0) ; extra == "server" or extra == "worker"
54
- Requires-Dist: uvicorn (>=0.34,<0.38) ; extra == "server"
54
+ Requires-Dist: uuid6 (>=2025.0.1,<2026.0.0)
55
+ Requires-Dist: uvicorn (>=0.38.0,<0.39.0) ; extra == "server"
55
56
  Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
56
57
  Project-URL: Documentation, https://syncmaster.readthedocs.io
57
58
  Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
@@ -61,8 +62,11 @@ Description-Content-Type: text/x-rst
61
62
 
62
63
  .. _readme:
63
64
 
64
- Data.SyncMaster
65
- ===============
65
+ |Logo|
66
+
67
+ .. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/e3afdfe835fc6ee943f1105cfe3d624c01123bd6/docs/_static/logo_wide_red_text.svg
68
+ :alt: Data.SyncMaster logo
69
+ :target: https://github.com/MobileTeleSystems/data-syncmaster
66
70
 
67
71
  |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
68
72
  |Build Status| |Coverage| |pre-commit.ci|
@@ -87,31 +91,29 @@ Data.SyncMaster
87
91
  .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
88
92
  :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
89
93
 
90
-
91
94
  What is Data.SyncMaster?
92
95
  ------------------------
93
96
 
94
- Data.SyncMaster is as low-code ETL tool for transfering data between databases and file systems.
97
+ Data.SyncMaster is as no-code ETL tool for transfering data between databases and file systems.
95
98
  List of currently supported connections:
96
99
 
97
100
  * Apache Hive
98
101
  * Clickhouse
99
- * Postgres
100
- * Oracle
102
+ * Iceberg (REST Catalog + S3)
101
103
  * MSSQL
102
104
  * MySQL
103
- * HDFS
104
- * S3
105
+ * Oracle
106
+ * Postgres
105
107
  * FTP
106
108
  * FTPS
107
- * SFTP
109
+ * HDFS
110
+ * S3
108
111
  * Samba
112
+ * SFTP
109
113
  * WebDAV
110
114
 
111
115
  Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
112
116
 
113
- **Note**: service is under active development, and is not ready to use.
114
-
115
117
  Goals
116
118
  -----
117
119
 
@@ -122,8 +124,8 @@ Goals
122
124
  Non-goals
123
125
  ---------
124
126
 
125
- * This is not a backup system
126
- * Only batch, no streaming
127
+ * No streaming support for now, only batch operations
128
+ * This is not a change data capture (CDC) solution
127
129
 
128
130
  .. documentation
129
131
 
@@ -1,7 +1,10 @@
1
1
  .. _readme:
2
2
 
3
- Data.SyncMaster
4
- ===============
3
+ |Logo|
4
+
5
+ .. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/e3afdfe835fc6ee943f1105cfe3d624c01123bd6/docs/_static/logo_wide_red_text.svg
6
+ :alt: Data.SyncMaster logo
7
+ :target: https://github.com/MobileTeleSystems/data-syncmaster
5
8
 
6
9
  |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
7
10
  |Build Status| |Coverage| |pre-commit.ci|
@@ -26,31 +29,29 @@ Data.SyncMaster
26
29
  .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
27
30
  :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
28
31
 
29
-
30
32
  What is Data.SyncMaster?
31
33
  ------------------------
32
34
 
33
- Data.SyncMaster is as low-code ETL tool for transfering data between databases and file systems.
35
+ Data.SyncMaster is as no-code ETL tool for transfering data between databases and file systems.
34
36
  List of currently supported connections:
35
37
 
36
38
  * Apache Hive
37
39
  * Clickhouse
38
- * Postgres
39
- * Oracle
40
+ * Iceberg (REST Catalog + S3)
40
41
  * MSSQL
41
42
  * MySQL
42
- * HDFS
43
- * S3
43
+ * Oracle
44
+ * Postgres
44
45
  * FTP
45
46
  * FTPS
46
- * SFTP
47
+ * HDFS
48
+ * S3
47
49
  * Samba
50
+ * SFTP
48
51
  * WebDAV
49
52
 
50
53
  Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
51
54
 
52
- **Note**: service is under active development, and is not ready to use.
53
-
54
55
  Goals
55
56
  -----
56
57
 
@@ -61,8 +62,8 @@ Goals
61
62
  Non-goals
62
63
  ---------
63
64
 
64
- * This is not a backup system
65
- * Only batch, no streaming
65
+ * No streaming support for now, only batch operations
66
+ * This is not a change data capture (CDC) solution
66
67
 
67
68
  .. documentation
68
69
 
@@ -4,10 +4,10 @@ build-backend = "poetry.core.masonry.api"
4
4
 
5
5
  [tool.poetry]
6
6
  name = "data-syncmaster"
7
- version = "0.2.5"
7
+ version = "0.3.0"
8
8
  license = "Apache-2.0"
9
9
  description = "Syncmaster REST API + Worker"
10
- authors = ["DataOps.ETL <onetools@mts.ru>"]
10
+ authors = ["MWS Data Bridge <onetools@mts.ru>"]
11
11
  readme = "README.rst"
12
12
  classifiers = [
13
13
  "Development Status :: 3 - Alpha",
@@ -44,118 +44,95 @@ exclude = [
44
44
 
45
45
  [tool.poetry.dependencies]
46
46
  python = "^3.11"
47
- pydantic = "^2.11.7"
48
- pydantic-settings = { version = "^2.8.1", optional = true }
49
- sqlalchemy = { version = "^2.0.40", optional = true }
50
- sqlalchemy-utils = { version = ">=0.41.2,<0.43.0", optional = true }
51
- fastapi = { version = ">=0.115.12,<0.119.0", optional = true}
52
- asgi-correlation-id = {version = "^4.3.4", optional = true}
53
- uvicorn = { version = ">=0.34,<0.38", optional = true }
54
- alembic = { version = "^1.15.2", optional = true }
47
+ pydantic = "^2.12.4"
48
+ pydantic-settings = "^2.12.0"
49
+ pydantic-settings-logging = "^0.1.1"
50
+ sqlalchemy = "^2.0.44"
51
+ sqlalchemy-utils = "^0.42.0"
52
+ pyyaml = "^6.0.3"
53
+ coloredlogs = "^15.0.1"
54
+ python-json-logger = "^4.0.0"
55
+ celery = "^5.5.3"
56
+ asgi-correlation-id = "^4.3.4"
57
+ uuid6 = "^2025.0.1"
58
+ fastapi = {version = "^0.124.0", optional = true}
59
+ uvicorn = { version = "^0.38.0", optional = true }
60
+ alembic = { version = "^1.17.1", optional = true }
55
61
  pyjwt = { version = "^2.10.1", optional = true }
56
62
  jinja2 = { version = "^3.1.6", optional = true }
57
63
  python-multipart = { version = "^0.0.20", optional = true }
58
- celery = { version = "^5.5.0", optional = true }
59
- onetl = { version = ">=0.13.5,<0.15.0", extras = ["all"], optional = true }
64
+ onetl = { version = "^0.15.0", extras = ["all"], optional = true }
60
65
  pyspark = { version = "<4.0.0", optional = true }
61
- pyyaml = { version = "*", optional = true }
62
- psycopg2-binary = { version = "^2.9.10", optional = true }
63
- uuid6 = {version = "^2025.0.0", optional = true}
64
- coloredlogs = {version = "*", optional = true}
65
- python-json-logger = {version = "*", optional = true}
66
- asyncpg = { version = "^0.30.0", optional = true }
67
- apscheduler = { version = "^3.11.0", optional = true }
66
+ psycopg2-binary = { version = "^2.9.11", optional = true }
67
+ asyncpg = { version = ">=0.30,<0.32", optional = true }
68
+ apscheduler = { version = "^3.11.1", optional = true }
68
69
  starlette-exporter = { version = "^0.23.0", optional = true }
69
- python-keycloak = { version = "^5.3.1", optional = true }
70
+ python-keycloak = { version = "^5.8.1", optional = true }
70
71
  itsdangerous = { version = "^2.2.0", optional = true }
71
- horizon-hwm-store = { version = "^1.1.2", optional = true }
72
+ horizon-hwm-store = { version = "^1.1.4", optional = true }
72
73
 
73
74
  [tool.poetry.extras]
74
75
  server = [
75
- "pydantic-settings",
76
- "sqlalchemy",
77
- "sqlalchemy-utils",
78
76
  "fastapi",
79
77
  "starlette-exporter",
80
78
  "uvicorn",
81
79
  "alembic",
82
80
  "python-multipart",
83
81
  "pyjwt",
84
- "asgi-correlation-id",
85
82
  "jinja2",
86
83
  "psycopg2-binary",
87
- "uuid6",
88
- "coloredlogs",
89
- "python-json-logger",
90
84
  "asyncpg",
91
85
  "python-keycloak",
92
86
  "itsdangerous",
93
87
  "pyyaml",
94
- "celery",
95
88
  ]
96
89
 
97
90
  worker = [
98
- "pydantic-settings",
99
- "sqlalchemy",
100
- "sqlalchemy-utils",
101
- "celery",
102
91
  "onetl",
103
92
  "pyspark",
104
- "asgi-correlation-id",
105
93
  "jinja2",
106
94
  "psycopg2-binary",
107
- "uuid6",
108
- "coloredlogs",
109
- "python-json-logger",
110
- "pyyaml",
111
95
  "horizon-hwm-store",
112
96
  ]
113
97
 
114
98
  scheduler = [
115
99
  "apscheduler",
116
- "pydantic-settings",
117
- "sqlalchemy",
118
- "sqlalchemy-utils",
119
100
  "asyncpg",
120
- "pyyaml",
121
- "coloredlogs",
122
- "python-json-logger",
123
- "celery",
124
101
  ]
125
102
 
126
103
  [tool.poetry.group.test.dependencies]
127
- pytest = "^8.3.5"
104
+ pytest = "^8.4.2"
128
105
  httpx = "^0.28.1"
129
106
  pytest-asyncio = ">=0.26.0,<1.0"
130
- pytest-randomly = ">=3.15,<5.0"
131
- pytest-rerunfailures = ">=15,<17"
132
- pytest-deadfixtures = "^2.2.1"
133
- pytest-mock = "^3.14.0"
134
- pytest-lazy-fixtures = "^1.1.1"
135
- faker = "^37.4.0"
136
- coverage = "^7.9.1"
137
- gevent = ">=24.11.1,<26.0.0"
107
+ pytest-randomly = "^4.0.1"
108
+ pytest-rerunfailures = "^16.1"
109
+ pytest-deadfixtures = "^3.0.0"
110
+ pytest-mock = "^3.15.1"
111
+ pytest-lazy-fixtures = "^1.4.0"
112
+ faker = "^38.2.0"
113
+ coverage = "^7.12.0"
114
+ gevent = "^25.9.1"
138
115
  respx = "^0.22.0"
139
- dirty-equals = "^0.9.0"
116
+ dirty-equals = "^0.11.0"
140
117
 
141
118
  [tool.poetry.group.dev.dependencies]
142
- mypy = "^1.15.0"
143
- pre-commit = "^4.2.0"
144
- black = "^25.1.0"
145
- flake8 = "^7.2.0"
119
+ mypy = "^1.18.2"
120
+ pre-commit = "^4.5.0"
121
+ black = "^25.12.0"
122
+ flake8 = "^7.3.0"
146
123
  flake8-pyproject = "^1.2.3"
147
- sqlalchemy = {extras = ["mypy"], version = "^2.0.40"}
124
+ sqlalchemy = {extras = ["mypy"], version = "^2.0.44"}
148
125
  types-jwcrypto = "^1.5.0"
149
126
 
150
127
  [tool.poetry.group.docs.dependencies]
151
128
  autodoc-pydantic = "^2.2.0"
152
- numpydoc = "^1.8.0"
129
+ numpydoc = "^1.9.0"
153
130
  sphinx = "^8.2.3"
154
- furo = ">=2024.8.6,<2026.0.0"
131
+ furo = "^2025.9.25"
155
132
  sphinx-copybutton = "^0.5.2"
156
133
  sphinxcontrib-towncrier = "^0.5.0a0"
157
- towncrier = "^24.8.0"
158
- sphinx-issues = "^5.0.0"
134
+ towncrier = "^25.8.0"
135
+ sphinx-issues = "^5.0.1"
159
136
  sphinx-design = "^0.6.1"
160
137
  sphinx-favicon = "^1.0.1"
161
138
  sphinx-argparse = "^0.5.2"
@@ -285,7 +262,7 @@ max-annotation-complexity = 4
285
262
  max-returns = 5
286
263
  max-awaits = 5
287
264
  max-local-variables = 20
288
- max-name-length = 60
265
+ max-name-length = 65
289
266
  # Max of expressions in a function
290
267
  max-expressions = 15
291
268
  # Max args in a function
@@ -459,8 +436,10 @@ ignore = [
459
436
  per-file-ignores = [
460
437
  # WPS102 Found incorrect module name pattern
461
438
  # WPS432 Found magic number: 256
462
- "*migrations/*.py:WPS102,WPS432",
463
- "*db/models/*.py:WPS102,WPS432",
439
+ # WPS226 Found string literal over-use: value > 15
440
+ # WPS342 Found implicit raw string
441
+ "*migrations/*.py:WPS102,WPS432,WPS226,WPS342",
442
+ "*db/models/*.py:WPS102,WPS432,WPS342",
464
443
  "*db/mixins/*.py:WPS102,WPS432",
465
444
  # WPS432 Found magic number: 180
466
445
  "*settings/*.py:WPS432",
@@ -1,6 +1,6 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
 
4
- _raw_version = "0.2.5"
4
+ _raw_version = "0.3.0"
5
5
  # version always contain only release number like 0.0.1
6
6
  __version__ = ".".join(_raw_version.split(".")[:3]) # noqa: WPS410
@@ -0,0 +1,2 @@
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
+ # SPDX-License-Identifier: Apache-2.0
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from collections.abc import AsyncGenerator, Callable
4
4
  from typing import Any
@@ -1,6 +1,6 @@
1
1
  #!/bin/env python3
2
2
 
3
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
3
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
4
4
  # SPDX-License-Identifier: Apache-2.0
5
5
 
6
6
  from __future__ import annotations
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  import asyncio
4
4
  import os
@@ -7,23 +7,34 @@ from logging.config import fileConfig
7
7
  from alembic import context
8
8
  from alembic.script import ScriptDirectory
9
9
  from celery.backends.database.session import ResultModelBase
10
+ from pydantic import Field
10
11
  from sqlalchemy import pool
11
12
  from sqlalchemy.engine import Connection
12
13
  from sqlalchemy.ext.asyncio import async_engine_from_config
13
14
 
14
15
  from syncmaster.db.models import Base
15
- from syncmaster.server.settings import ServerAppSettings as Settings
16
+ from syncmaster.server.settings import (
17
+ DEFAULT_LOGGING_SETTINGS,
18
+ BaseSettings,
19
+ DatabaseSettings,
20
+ LoggingSettings,
21
+ )
16
22
 
17
23
  config = context.config
18
24
 
19
25
 
26
+ class MigrationAppSettings(BaseSettings):
27
+ database: DatabaseSettings = Field(default_factory=DatabaseSettings, description="Database settings")
28
+ logging: LoggingSettings = Field(default=DEFAULT_LOGGING_SETTINGS, description="Logging settings")
29
+
30
+
20
31
  if config.config_file_name is not None:
21
32
  fileConfig(config.config_file_name)
22
33
 
23
34
  if not config.get_main_option("sqlalchemy.url"):
24
35
  # read application settings only if sqlalchemy.url is not being passed via cli arguments
25
36
  # TODO: remove settings object creating during import
26
- config.set_main_option("sqlalchemy.url", Settings().database.url)
37
+ config.set_main_option("sqlalchemy.url", MigrationAppSettings().database.url)
27
38
 
28
39
  target_metadata = (
29
40
  Base.metadata,
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create user table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create group table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create queue table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create connection table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create user_group table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create auth_data table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create transfer table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create run table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create celery_taskmeta and celery_tasksetmeta tables
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Add pg_trgm extension to apply fuzzy search
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Create apscheduler table
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  """Update text search
4
4
 
@@ -0,0 +1,2 @@
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
+ # SPDX-License-Identifier: Apache-2.0
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from syncmaster.db.mixins.resource import ResourceMixin
4
4
  from syncmaster.db.mixins.timestamp import TimestampMixin
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
 
4
4
  from sqlalchemy import BigInteger, ForeignKey, String
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from datetime import datetime, timezone
4
4
  from functools import partial
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from syncmaster.db.models.apscheduler_job import APSchedulerJob
4
4
  from syncmaster.db.models.auth_data import AuthData
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from sqlalchemy import Float, Index, LargeBinary, String
4
4
  from sqlalchemy.orm import Mapped, mapped_column
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  import re
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
@@ -18,6 +18,7 @@ from syncmaster.db.models.group import Group
18
18
  class ConnectionType(StrEnum):
19
19
  POSTGRES = "postgres"
20
20
  HIVE = "hive"
21
+ ICEBERG = "iceberg"
21
22
  ORACLE = "oracle"
22
23
  CLICKHOUSE = "clickhouse"
23
24
  MSSQL = "mssql"
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
@@ -1,4 +1,4 @@
1
- # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
1
+ # SPDX-FileCopyrightText: 2023-present MTS PJSC
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4