data-syncmaster 0.2.5__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/LICENSE.txt +1 -1
  2. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/PKG-INFO +60 -59
  3. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/README.rst +15 -14
  4. data_syncmaster-0.3.1/data_syncmaster.egg-info/PKG-INFO +135 -0
  5. data_syncmaster-0.3.1/data_syncmaster.egg-info/SOURCES.txt +232 -0
  6. data_syncmaster-0.3.1/data_syncmaster.egg-info/dependency_links.txt +1 -0
  7. data_syncmaster-0.3.1/data_syncmaster.egg-info/requires.txt +36 -0
  8. data_syncmaster-0.3.1/data_syncmaster.egg-info/top_level.txt +1 -0
  9. data_syncmaster-0.3.1/pyproject.toml +326 -0
  10. data_syncmaster-0.3.1/setup.cfg +4 -0
  11. data_syncmaster-0.3.1/syncmaster/VERSION +1 -0
  12. data_syncmaster-0.3.1/syncmaster/__init__.py +11 -0
  13. data_syncmaster-0.3.1/syncmaster/db/__init__.py +2 -0
  14. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/factory.py +1 -1
  15. data_syncmaster-0.3.1/syncmaster/db/migrations/__init__.py +2 -0
  16. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/__main__.py +1 -1
  17. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/env.py +19 -10
  18. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py +1 -1
  19. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py +1 -1
  20. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py +1 -1
  21. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py +1 -1
  22. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py +1 -1
  23. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py +1 -1
  24. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py +1 -1
  25. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py +1 -1
  26. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py +1 -1
  27. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py +2 -1
  28. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py +8 -2
  29. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py +35 -8
  30. data_syncmaster-0.3.1/syncmaster/db/migrations/versions/__init__.py +2 -0
  31. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/mixins/__init__.py +1 -1
  32. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/mixins/resource.py +1 -1
  33. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/mixins/timestamp.py +3 -3
  34. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/__init__.py +8 -8
  35. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/apscheduler_job.py +1 -1
  36. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/auth_data.py +3 -1
  37. data_syncmaster-0.3.1/syncmaster/db/models/base.py +23 -0
  38. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/connection.py +11 -11
  39. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/group.py +6 -7
  40. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/queue.py +5 -9
  41. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/run.py +4 -2
  42. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/transfer.py +15 -18
  43. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/user.py +3 -1
  44. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/__init__.py +1 -1
  45. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/base.py +12 -10
  46. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/connection.py +5 -7
  47. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/credentials_repository.py +3 -2
  48. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/group.py +5 -5
  49. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/queue.py +8 -22
  50. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/repository_with_owner.py +4 -5
  51. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/run.py +9 -8
  52. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/search.py +2 -6
  53. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/transfer.py +11 -12
  54. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/user.py +4 -4
  55. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/utils.py +1 -8
  56. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/utils.py +2 -2
  57. data_syncmaster-0.3.1/syncmaster/dto/__init__.py +2 -0
  58. data_syncmaster-0.3.1/syncmaster/dto/connections.py +231 -0
  59. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/runs.py +2 -2
  60. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/transfers.py +49 -21
  61. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/transfers_resources.py +1 -1
  62. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/transfers_strategy.py +3 -2
  63. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/__init__.py +1 -1
  64. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/base.py +1 -1
  65. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/registration.py +9 -5
  66. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/__init__.py +2 -2
  67. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/bad_request.py +1 -1
  68. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/invalid_request.py +1 -1
  69. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/not_authorized.py +1 -1
  70. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/__init__.py +1 -1
  71. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/auth.py +1 -1
  72. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/base.py +1 -1
  73. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/connection.py +1 -1
  74. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/credentials.py +1 -1
  75. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/group.py +1 -1
  76. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/queue.py +1 -1
  77. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/redirect.py +2 -2
  78. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/run.py +1 -1
  79. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/transfer.py +2 -4
  80. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/user.py +1 -1
  81. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/__init__.py +3 -4
  82. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/__main__.py +10 -7
  83. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/celery.py +1 -1
  84. data_syncmaster-0.3.1/syncmaster/scheduler/settings/__init__.py +76 -0
  85. data_syncmaster-0.3.1/syncmaster/scheduler/transfer_fetcher.py +34 -0
  86. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/transfer_job_manager.py +14 -7
  87. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/utils.py +1 -1
  88. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/__init__.py +1 -1
  89. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/ping.py +1 -1
  90. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/__init__.py +27 -37
  91. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/auth/__init__.py +7 -13
  92. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/basic.py +23 -0
  93. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py +31 -0
  94. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py +22 -0
  95. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py +32 -0
  96. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py +31 -0
  97. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py +25 -0
  98. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py +34 -0
  99. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/mixins.py +11 -0
  100. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/s3.py +23 -0
  101. data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/samba.py +24 -0
  102. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/auth/token.py +1 -1
  103. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connection_types.py +26 -7
  104. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection.py → data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/__init__.py +39 -31
  105. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/clickhouse.py +14 -19
  106. data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/connection_base.py +46 -0
  107. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/ftp.py +15 -17
  108. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/ftps.py +12 -17
  109. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/hdfs.py +14 -15
  110. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/hive.py +14 -15
  111. data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/iceberg.py +107 -0
  112. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/mssql.py +15 -19
  113. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/mysql.py +16 -20
  114. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/oracle.py +18 -21
  115. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/postgres.py +15 -19
  116. data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/s3.py +65 -0
  117. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/samba.py +7 -18
  118. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/sftp.py +14 -16
  119. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/webdav.py +15 -18
  120. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/file_formats.py +1 -1
  121. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/groups.py +9 -14
  122. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/page.py +1 -1
  123. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/queue.py +10 -11
  124. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/__init__.py +102 -0
  125. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/db.py +87 -0
  126. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/__init__.py +55 -0
  127. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/file/base.py +33 -38
  128. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/ftp.py +17 -0
  129. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/ftps.py +17 -0
  130. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/hdfs.py +17 -0
  131. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/s3.py +17 -0
  132. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/samba.py +17 -0
  133. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/sftp.py +17 -0
  134. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/webdav.py +17 -0
  135. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/file_format.py +1 -1
  136. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/resources.py +17 -0
  137. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/run.py +10 -11
  138. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/strategy.py +23 -0
  139. data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/transformations/__init__.py +25 -0
  140. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py +2 -4
  141. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py +2 -4
  142. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py +8 -6
  143. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transformation_types.py +1 -1
  144. data_syncmaster-0.3.1/syncmaster/schemas/v1/types.py +8 -0
  145. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/users.py +1 -1
  146. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/__init__.py +13 -7
  147. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/__main__.py +1 -1
  148. data_syncmaster-0.3.1/syncmaster/server/api/__init__.py +2 -0
  149. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/monitoring.py +1 -1
  150. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/router.py +1 -1
  151. data_syncmaster-0.3.1/syncmaster/server/api/v1/__init__.py +2 -0
  152. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/auth.py +3 -3
  153. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/connections.py +43 -37
  154. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/groups.py +35 -33
  155. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/queue.py +29 -28
  156. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/router.py +1 -1
  157. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/runs.py +21 -20
  158. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/transfers.py +47 -41
  159. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/users.py +11 -14
  160. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/dependencies/__init__.py +2 -2
  161. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/dependencies/get_access_token.py +1 -1
  162. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/dependencies/stub.py +1 -3
  163. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/handler.py +8 -4
  164. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/__init__.py +1 -7
  165. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/cors.py +1 -1
  166. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/monitoring/__init__.py +1 -1
  167. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/monitoring/metrics.py +1 -1
  168. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/openapi.py +10 -7
  169. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/request_id.py +1 -1
  170. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/static_files.py +1 -1
  171. data_syncmaster-0.3.1/syncmaster/server/providers/__init__.py +2 -0
  172. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/__init__.py +5 -6
  173. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/base_provider.py +2 -2
  174. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/dummy_provider.py +16 -11
  175. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/keycloak_provider.py +31 -12
  176. data_syncmaster-0.3.1/syncmaster/server/providers/auth/oauth2_gateway_provider.py +123 -0
  177. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/scripts/export_openapi_schema.py +5 -3
  178. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/scripts/manage_superusers.py +46 -20
  179. data_syncmaster-0.3.1/syncmaster/server/services/__init__.py +2 -0
  180. data_syncmaster-0.3.1/syncmaster/server/services/get_user.py +67 -0
  181. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/services/unit_of_work.py +7 -3
  182. data_syncmaster-0.3.1/syncmaster/server/settings/__init__.py +70 -0
  183. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/auth/__init__.py +8 -6
  184. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/auth/dummy.py +7 -4
  185. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/auth/jwt.py +7 -4
  186. data_syncmaster-0.3.1/syncmaster/server/settings/auth/keycloak.py +135 -0
  187. data_syncmaster-0.3.1/syncmaster/server/settings/auth/oauth2_gateway.py +35 -0
  188. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/__init__.py +15 -10
  189. data_syncmaster-0.3.1/syncmaster/server/settings/server/cors.py +63 -0
  190. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/monitoring.py +16 -5
  191. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/openapi.py +48 -22
  192. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/request_id.py +8 -4
  193. data_syncmaster-0.3.1/syncmaster/server/settings/server/static_files.py +42 -0
  194. data_syncmaster-0.3.1/syncmaster/server/utils/__init__.py +2 -0
  195. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/utils/jwt.py +7 -5
  196. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/utils/slug.py +1 -1
  197. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/settings/__init__.py +6 -3
  198. data_syncmaster-0.3.1/syncmaster/settings/base.py +37 -0
  199. data_syncmaster-0.3.1/syncmaster/settings/broker.py +30 -0
  200. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/settings/credentials.py +5 -4
  201. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/settings/database.py +8 -5
  202. data_syncmaster-0.3.1/syncmaster/settings/logging.py +289 -0
  203. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/__init__.py +3 -4
  204. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/base.py +1 -2
  205. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/celery.py +1 -1
  206. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/controller.py +54 -19
  207. data_syncmaster-0.3.1/syncmaster/worker/handlers/__init__.py +2 -0
  208. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/base.py +9 -9
  209. data_syncmaster-0.3.1/syncmaster/worker/handlers/db/__init__.py +2 -0
  210. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/base.py +21 -19
  211. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/clickhouse.py +20 -12
  212. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/hive.py +18 -10
  213. data_syncmaster-0.3.1/syncmaster/worker/handlers/db/iceberg.py +135 -0
  214. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/mssql.py +25 -11
  215. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/mysql.py +21 -10
  216. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/oracle.py +21 -10
  217. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/postgres.py +21 -10
  218. data_syncmaster-0.3.1/syncmaster/worker/handlers/file/__init__.py +2 -0
  219. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/base.py +25 -21
  220. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/ftp.py +16 -4
  221. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/ftps.py +16 -4
  222. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/hdfs.py +16 -4
  223. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/local_df.py +22 -23
  224. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/remote_df.py +5 -6
  225. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/s3.py +14 -5
  226. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/samba.py +16 -4
  227. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/sftp.py +16 -4
  228. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/webdav.py +14 -3
  229. data_syncmaster-0.3.1/syncmaster/worker/ivy2.py +68 -0
  230. data_syncmaster-0.3.1/syncmaster/worker/settings/__init__.py +95 -0
  231. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/settings/hwm_store.py +11 -9
  232. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/spark.py +38 -53
  233. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/transfer.py +10 -10
  234. data_syncmaster-0.2.5/pyproject.toml +0 -546
  235. data_syncmaster-0.2.5/syncmaster/__init__.py +0 -6
  236. data_syncmaster-0.2.5/syncmaster/db/__init__.py +0 -2
  237. data_syncmaster-0.2.5/syncmaster/db/migrations/README +0 -1
  238. data_syncmaster-0.2.5/syncmaster/db/migrations/script.py.mako +0 -24
  239. data_syncmaster-0.2.5/syncmaster/db/migrations/versions/__init__.py +0 -2
  240. data_syncmaster-0.2.5/syncmaster/db/models/base.py +0 -28
  241. data_syncmaster-0.2.5/syncmaster/dto/__init__.py +0 -2
  242. data_syncmaster-0.2.5/syncmaster/dto/connections.py +0 -144
  243. data_syncmaster-0.2.5/syncmaster/scheduler/settings/__init__.py +0 -78
  244. data_syncmaster-0.2.5/syncmaster/scheduler/transfer_fetcher.py +0 -24
  245. data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/basic.py +0 -26
  246. data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/s3.py +0 -26
  247. data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/samba.py +0 -28
  248. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/__init__.py +0 -2
  249. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection_base.py +0 -30
  250. data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/s3.py +0 -73
  251. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfer_types.py +0 -6
  252. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/__init__.py +0 -248
  253. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/db.py +0 -42
  254. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/__init__.py +0 -2
  255. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftp.py +0 -27
  256. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftps.py +0 -27
  257. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -27
  258. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/s3.py +0 -27
  259. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/samba.py +0 -27
  260. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/sftp.py +0 -27
  261. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/webdav.py +0 -27
  262. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/resources.py +0 -14
  263. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/strategy.py +0 -14
  264. data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/transformations/__init__.py +0 -2
  265. data_syncmaster-0.2.5/syncmaster/schemas/v1/types.py +0 -7
  266. data_syncmaster-0.2.5/syncmaster/server/api/__init__.py +0 -2
  267. data_syncmaster-0.2.5/syncmaster/server/api/v1/__init__.py +0 -2
  268. data_syncmaster-0.2.5/syncmaster/server/middlewares/session.py +0 -17
  269. data_syncmaster-0.2.5/syncmaster/server/providers/__init__.py +0 -2
  270. data_syncmaster-0.2.5/syncmaster/server/services/__init__.py +0 -2
  271. data_syncmaster-0.2.5/syncmaster/server/services/get_user.py +0 -41
  272. data_syncmaster-0.2.5/syncmaster/server/settings/__init__.py +0 -61
  273. data_syncmaster-0.2.5/syncmaster/server/settings/auth/keycloak.py +0 -22
  274. data_syncmaster-0.2.5/syncmaster/server/settings/server/cors.py +0 -70
  275. data_syncmaster-0.2.5/syncmaster/server/settings/server/session.py +0 -75
  276. data_syncmaster-0.2.5/syncmaster/server/settings/server/static_files.py +0 -34
  277. data_syncmaster-0.2.5/syncmaster/server/utils/__init__.py +0 -2
  278. data_syncmaster-0.2.5/syncmaster/settings/broker.py +0 -29
  279. data_syncmaster-0.2.5/syncmaster/settings/log/__init__.py +0 -110
  280. data_syncmaster-0.2.5/syncmaster/settings/log/colored.yml +0 -57
  281. data_syncmaster-0.2.5/syncmaster/settings/log/json.yml +0 -56
  282. data_syncmaster-0.2.5/syncmaster/settings/log/plain.yml +0 -57
  283. data_syncmaster-0.2.5/syncmaster/worker/handlers/__init__.py +0 -2
  284. data_syncmaster-0.2.5/syncmaster/worker/handlers/db/__init__.py +0 -2
  285. data_syncmaster-0.2.5/syncmaster/worker/handlers/file/__init__.py +0 -2
  286. data_syncmaster-0.2.5/syncmaster/worker/settings/__init__.py +0 -79
  287. {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/alembic.ini +0 -0
@@ -1,4 +1,4 @@
1
- Copyright 2023-2025 MTS PJSC. All rights reserved.
1
+ Copyright 2023-present MTS PJSC. All rights reserved.
2
2
 
3
3
  Apache License
4
4
  Version 2.0, January 2004
@@ -1,68 +1,72 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: data-syncmaster
3
- Version: 0.2.5
4
- Summary: Syncmaster REST API + Worker
5
- License: Apache-2.0
6
- License-File: LICENSE.txt
3
+ Version: 0.3.1
4
+ Summary: Syncmaster REST API + scheduler + Worker
5
+ Author-email: MWS Data Bridge <onetools@mts.ru>
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
8
+ Project-URL: Documentation, https://syncmaster.readthedocs.io/
9
+ Project-URL: Source, https://github.com/MobileTeleSystems/syncmaster
10
+ Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
11
+ Project-URL: Tracker, https://github.com/MobileTeleSystems/syncmaster/issues
7
12
  Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
8
- Author: DataOps.ETL
9
- Author-email: onetools@mts.ru
10
- Requires-Python: >=3.11,<4.0
11
13
  Classifier: Development Status :: 3 - Alpha
12
- Classifier: Framework :: FastAPI
13
14
  Classifier: Framework :: Pydantic
14
15
  Classifier: Framework :: Pydantic :: 2
16
+ Classifier: Framework :: FastAPI
15
17
  Classifier: Intended Audience :: Developers
16
- Classifier: License :: OSI Approved :: Apache Software License
17
18
  Classifier: Operating System :: OS Independent
18
- Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3 :: Only
19
20
  Classifier: Programming Language :: Python :: 3.11
20
21
  Classifier: Programming Language :: Python :: 3.12
21
22
  Classifier: Programming Language :: Python :: 3.13
22
- Classifier: Programming Language :: Python :: 3.14
23
- Classifier: Programming Language :: Python :: 3 :: Only
24
23
  Classifier: Topic :: Software Development :: Libraries
25
24
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
26
25
  Classifier: Typing :: Typed
27
- Provides-Extra: scheduler
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/x-rst
28
+ License-File: LICENSE.txt
29
+ Requires-Dist: pydantic~=2.12.5
30
+ Requires-Dist: pydantic-settings~=2.12.0
31
+ Requires-Dist: pydantic-settings-logging~=0.1.1
32
+ Requires-Dist: sqlalchemy~=2.0.44
33
+ Requires-Dist: sqlalchemy-utils~=0.42.0
34
+ Requires-Dist: pyyaml~=6.0.3
35
+ Requires-Dist: coloredlogs~=15.0.1
36
+ Requires-Dist: python-json-logger~=4.0.0
37
+ Requires-Dist: asgi-correlation-id~=4.3.4
38
+ Requires-Dist: uuid6~=2025.0.1
39
+ Requires-Dist: celery~=5.6.0
40
+ Requires-Dist: psycopg2-binary~=2.9.11
28
41
  Provides-Extra: server
42
+ Requires-Dist: fastapi~=0.128.0; extra == "server"
43
+ Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
44
+ Requires-Dist: python-multipart~=0.0.21; extra == "server"
45
+ Requires-Dist: uvicorn~=0.40.0; extra == "server"
46
+ Requires-Dist: alembic~=1.18.1; extra == "server"
47
+ Requires-Dist: pyjwt~=2.10.1; extra == "server"
48
+ Requires-Dist: asyncpg~=0.31; extra == "server"
49
+ Requires-Dist: python-keycloak~=7.0.1; extra == "server"
50
+ Requires-Dist: itsdangerous~=2.2.0; extra == "server"
51
+ Provides-Extra: scheduler
52
+ Requires-Dist: asyncpg~=0.31; extra == "scheduler"
53
+ Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
29
54
  Provides-Extra: worker
30
- Requires-Dist: alembic (>=1.15.2,<2.0.0) ; extra == "server"
31
- Requires-Dist: apscheduler (>=3.11.0,<4.0.0) ; extra == "scheduler"
32
- Requires-Dist: asgi-correlation-id (>=4.3.4,<5.0.0) ; extra == "server" or extra == "worker"
33
- Requires-Dist: asyncpg (>=0.30.0,<0.31.0) ; extra == "server" or extra == "scheduler"
34
- Requires-Dist: celery (>=5.5.0,<6.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
35
- Requires-Dist: coloredlogs ; extra == "server" or extra == "worker" or extra == "scheduler"
36
- Requires-Dist: fastapi (>=0.115.12,<0.119.0) ; extra == "server"
37
- Requires-Dist: horizon-hwm-store (>=1.1.2,<2.0.0) ; extra == "worker"
38
- Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
39
- Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
40
- Requires-Dist: onetl[all] (>=0.13.5,<0.15.0) ; extra == "worker"
41
- Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0) ; extra == "server" or extra == "worker"
42
- Requires-Dist: pydantic (>=2.11.7,<3.0.0)
43
- Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
44
- Requires-Dist: pyjwt (>=2.10.1,<3.0.0) ; extra == "server"
45
- Requires-Dist: pyspark (<4.0.0) ; extra == "worker"
46
- Requires-Dist: python-json-logger ; extra == "server" or extra == "worker" or extra == "scheduler"
47
- Requires-Dist: python-keycloak (>=5.3.1,<6.0.0) ; extra == "server"
48
- Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
49
- Requires-Dist: pyyaml ; extra == "server" or extra == "worker" or extra == "scheduler"
50
- Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
51
- Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.43.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
52
- Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
53
- Requires-Dist: uuid6 (>=2025.0.0,<2026.0.0) ; extra == "server" or extra == "worker"
54
- Requires-Dist: uvicorn (>=0.34,<0.38) ; extra == "server"
55
- Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
56
- Project-URL: Documentation, https://syncmaster.readthedocs.io
57
- Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
58
- Project-URL: Source, https://github.com/MobileTeleSystems/syncmaster
59
- Project-URL: Tracker, https://github.com/MobileTeleSystems/syncmaster/issues
60
- Description-Content-Type: text/x-rst
55
+ Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
56
+ Requires-Dist: pyspark<4.0.0; extra == "worker"
57
+ Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
58
+ Requires-Dist: jinja2~=3.1.6; extra == "worker"
59
+ Provides-Extra: kerberos
60
+ Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
61
+ Dynamic: license-file
61
62
 
62
63
  .. _readme:
63
64
 
64
- Data.SyncMaster
65
- ===============
65
+ |Logo|
66
+
67
+ .. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/d14bf1b1272f15d42f441d6e33264bd2f95216c9/docs/_static/logo_wide.svg
68
+ :alt: Data.SyncMaster logo
69
+ :target: https://github.com/MobileTeleSystems/data-syncmaster
66
70
 
67
71
  |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
68
72
  |Build Status| |Coverage| |pre-commit.ci|
@@ -87,43 +91,41 @@ Data.SyncMaster
87
91
  .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
88
92
  :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
89
93
 
90
-
91
94
  What is Data.SyncMaster?
92
95
  ------------------------
93
96
 
94
- Data.SyncMaster is as low-code ETL tool for transfering data between databases and file systems.
97
+ Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
95
98
  List of currently supported connections:
96
99
 
97
100
  * Apache Hive
98
101
  * Clickhouse
99
- * Postgres
100
- * Oracle
102
+ * Iceberg (REST Catalog + S3)
101
103
  * MSSQL
102
104
  * MySQL
103
- * HDFS
104
- * S3
105
+ * Oracle
106
+ * Postgres
105
107
  * FTP
106
108
  * FTPS
107
- * SFTP
109
+ * HDFS
110
+ * S3
108
111
  * Samba
112
+ * SFTP
109
113
  * WebDAV
110
114
 
111
115
  Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
112
116
 
113
- **Note**: service is under active development, and is not ready to use.
114
-
115
117
  Goals
116
118
  -----
117
119
 
118
- * Make transfering data between databases and file systems as simple as possible
120
+ * Make transferring data between databases and file systems as simple as possible
119
121
  * Provide a lot of builtin connectors to transfer data in heterogeneous environment
120
122
  * RBAC and multitenancy support
121
123
 
122
124
  Non-goals
123
125
  ---------
124
126
 
125
- * This is not a backup system
126
- * Only batch, no streaming
127
+ * No streaming support for now, only batch operations
128
+ * This is not a change data capture (CDC) solution
127
129
 
128
130
  .. documentation
129
131
 
@@ -131,4 +133,3 @@ Documentation
131
133
  -------------
132
134
 
133
135
  See https://syncmaster.readthedocs.io
134
-
@@ -1,7 +1,10 @@
1
1
  .. _readme:
2
2
 
3
- Data.SyncMaster
4
- ===============
3
+ |Logo|
4
+
5
+ .. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/d14bf1b1272f15d42f441d6e33264bd2f95216c9/docs/_static/logo_wide.svg
6
+ :alt: Data.SyncMaster logo
7
+ :target: https://github.com/MobileTeleSystems/data-syncmaster
5
8
 
6
9
  |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
7
10
  |Build Status| |Coverage| |pre-commit.ci|
@@ -26,43 +29,41 @@ Data.SyncMaster
26
29
  .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
27
30
  :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
28
31
 
29
-
30
32
  What is Data.SyncMaster?
31
33
  ------------------------
32
34
 
33
- Data.SyncMaster is as low-code ETL tool for transfering data between databases and file systems.
35
+ Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
34
36
  List of currently supported connections:
35
37
 
36
38
  * Apache Hive
37
39
  * Clickhouse
38
- * Postgres
39
- * Oracle
40
+ * Iceberg (REST Catalog + S3)
40
41
  * MSSQL
41
42
  * MySQL
42
- * HDFS
43
- * S3
43
+ * Oracle
44
+ * Postgres
44
45
  * FTP
45
46
  * FTPS
46
- * SFTP
47
+ * HDFS
48
+ * S3
47
49
  * Samba
50
+ * SFTP
48
51
  * WebDAV
49
52
 
50
53
  Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
51
54
 
52
- **Note**: service is under active development, and is not ready to use.
53
-
54
55
  Goals
55
56
  -----
56
57
 
57
- * Make transfering data between databases and file systems as simple as possible
58
+ * Make transferring data between databases and file systems as simple as possible
58
59
  * Provide a lot of builtin connectors to transfer data in heterogeneous environment
59
60
  * RBAC and multitenancy support
60
61
 
61
62
  Non-goals
62
63
  ---------
63
64
 
64
- * This is not a backup system
65
- * Only batch, no streaming
65
+ * No streaming support for now, only batch operations
66
+ * This is not a change data capture (CDC) solution
66
67
 
67
68
  .. documentation
68
69
 
@@ -0,0 +1,135 @@
1
+ Metadata-Version: 2.4
2
+ Name: data-syncmaster
3
+ Version: 0.3.1
4
+ Summary: Syncmaster REST API + scheduler + Worker
5
+ Author-email: MWS Data Bridge <onetools@mts.ru>
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
8
+ Project-URL: Documentation, https://syncmaster.readthedocs.io/
9
+ Project-URL: Source, https://github.com/MobileTeleSystems/syncmaster
10
+ Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
11
+ Project-URL: Tracker, https://github.com/MobileTeleSystems/syncmaster/issues
12
+ Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Framework :: Pydantic
15
+ Classifier: Framework :: Pydantic :: 2
16
+ Classifier: Framework :: FastAPI
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Software Development :: Libraries
24
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/x-rst
28
+ License-File: LICENSE.txt
29
+ Requires-Dist: pydantic~=2.12.5
30
+ Requires-Dist: pydantic-settings~=2.12.0
31
+ Requires-Dist: pydantic-settings-logging~=0.1.1
32
+ Requires-Dist: sqlalchemy~=2.0.44
33
+ Requires-Dist: sqlalchemy-utils~=0.42.0
34
+ Requires-Dist: pyyaml~=6.0.3
35
+ Requires-Dist: coloredlogs~=15.0.1
36
+ Requires-Dist: python-json-logger~=4.0.0
37
+ Requires-Dist: asgi-correlation-id~=4.3.4
38
+ Requires-Dist: uuid6~=2025.0.1
39
+ Requires-Dist: celery~=5.6.0
40
+ Requires-Dist: psycopg2-binary~=2.9.11
41
+ Provides-Extra: server
42
+ Requires-Dist: fastapi~=0.128.0; extra == "server"
43
+ Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
44
+ Requires-Dist: python-multipart~=0.0.21; extra == "server"
45
+ Requires-Dist: uvicorn~=0.40.0; extra == "server"
46
+ Requires-Dist: alembic~=1.18.1; extra == "server"
47
+ Requires-Dist: pyjwt~=2.10.1; extra == "server"
48
+ Requires-Dist: asyncpg~=0.31; extra == "server"
49
+ Requires-Dist: python-keycloak~=7.0.1; extra == "server"
50
+ Requires-Dist: itsdangerous~=2.2.0; extra == "server"
51
+ Provides-Extra: scheduler
52
+ Requires-Dist: asyncpg~=0.31; extra == "scheduler"
53
+ Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
54
+ Provides-Extra: worker
55
+ Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
56
+ Requires-Dist: pyspark<4.0.0; extra == "worker"
57
+ Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
58
+ Requires-Dist: jinja2~=3.1.6; extra == "worker"
59
+ Provides-Extra: kerberos
60
+ Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
61
+ Dynamic: license-file
62
+
63
+ .. _readme:
64
+
65
+ |Logo|
66
+
67
+ .. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/d14bf1b1272f15d42f441d6e33264bd2f95216c9/docs/_static/logo_wide.svg
68
+ :alt: Data.SyncMaster logo
69
+ :target: https://github.com/MobileTeleSystems/data-syncmaster
70
+
71
+ |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
72
+ |Build Status| |Coverage| |pre-commit.ci|
73
+
74
+ .. |Repo Status| image:: https://www.repostatus.org/badges/latest/wip.svg
75
+ :target: https://www.repostatus.org/#wip
76
+ .. |Docker image| image:: https://img.shields.io/docker/v/mtsrus/syncmaster-server?sort=semver&label=docker
77
+ :target: https://hub.docker.com/r/mtsrus/syncmaster-server
78
+ .. |PyPI| image:: https://img.shields.io/pypi/v/data-syncmaster
79
+ :target: https://pypi.org/project/data-syncmaster/
80
+ .. |PyPI License| image:: https://img.shields.io/pypi/l/data-syncmaster.svg
81
+ :target: https://github.com/MobileTeleSystems/syncmaster/blob/develop/LICENSE.txt
82
+ .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/data-syncmaster.svg
83
+ :target: https://badge.fury.io/py/data-syncmaster
84
+ .. |Documentation| image:: https://readthedocs.org/projects/syncmaster/badge/?version=stable
85
+ :target: https://syncmaster.readthedocs.io
86
+ .. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Run%20All%20Tests/badge.svg
87
+ :target: https://github.com/MobileTeleSystems/syncmaster/actions
88
+ .. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
89
+ MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
90
+ :target: https://github.com/MobileTeleSystems/syncmaster/actions
91
+ .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
92
+ :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
93
+
94
+ What is Data.SyncMaster?
95
+ ------------------------
96
+
97
+ Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
98
+ List of currently supported connections:
99
+
100
+ * Apache Hive
101
+ * Clickhouse
102
+ * Iceberg (REST Catalog + S3)
103
+ * MSSQL
104
+ * MySQL
105
+ * Oracle
106
+ * Postgres
107
+ * FTP
108
+ * FTPS
109
+ * HDFS
110
+ * S3
111
+ * Samba
112
+ * SFTP
113
+ * WebDAV
114
+
115
+ Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
116
+
117
+ Goals
118
+ -----
119
+
120
+ * Make transferring data between databases and file systems as simple as possible
121
+ * Provide a lot of builtin connectors to transfer data in heterogeneous environment
122
+ * RBAC and multitenancy support
123
+
124
+ Non-goals
125
+ ---------
126
+
127
+ * No streaming support for now, only batch operations
128
+ * This is not a change data capture (CDC) solution
129
+
130
+ .. documentation
131
+
132
+ Documentation
133
+ -------------
134
+
135
+ See https://syncmaster.readthedocs.io
@@ -0,0 +1,232 @@
1
+ LICENSE.txt
2
+ README.rst
3
+ pyproject.toml
4
+ data_syncmaster.egg-info/PKG-INFO
5
+ data_syncmaster.egg-info/SOURCES.txt
6
+ data_syncmaster.egg-info/dependency_links.txt
7
+ data_syncmaster.egg-info/requires.txt
8
+ data_syncmaster.egg-info/top_level.txt
9
+ syncmaster/VERSION
10
+ syncmaster/__init__.py
11
+ syncmaster/db/__init__.py
12
+ syncmaster/db/factory.py
13
+ syncmaster/db/utils.py
14
+ syncmaster/db/migrations/__init__.py
15
+ syncmaster/db/migrations/__main__.py
16
+ syncmaster/db/migrations/alembic.ini
17
+ syncmaster/db/migrations/env.py
18
+ syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py
19
+ syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py
20
+ syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py
21
+ syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py
22
+ syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py
23
+ syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py
24
+ syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py
25
+ syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py
26
+ syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py
27
+ syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py
28
+ syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py
29
+ syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py
30
+ syncmaster/db/migrations/versions/__init__.py
31
+ syncmaster/db/mixins/__init__.py
32
+ syncmaster/db/mixins/resource.py
33
+ syncmaster/db/mixins/timestamp.py
34
+ syncmaster/db/models/__init__.py
35
+ syncmaster/db/models/apscheduler_job.py
36
+ syncmaster/db/models/auth_data.py
37
+ syncmaster/db/models/base.py
38
+ syncmaster/db/models/connection.py
39
+ syncmaster/db/models/group.py
40
+ syncmaster/db/models/queue.py
41
+ syncmaster/db/models/run.py
42
+ syncmaster/db/models/transfer.py
43
+ syncmaster/db/models/user.py
44
+ syncmaster/db/repositories/__init__.py
45
+ syncmaster/db/repositories/base.py
46
+ syncmaster/db/repositories/connection.py
47
+ syncmaster/db/repositories/credentials_repository.py
48
+ syncmaster/db/repositories/group.py
49
+ syncmaster/db/repositories/queue.py
50
+ syncmaster/db/repositories/repository_with_owner.py
51
+ syncmaster/db/repositories/run.py
52
+ syncmaster/db/repositories/search.py
53
+ syncmaster/db/repositories/transfer.py
54
+ syncmaster/db/repositories/user.py
55
+ syncmaster/db/repositories/utils.py
56
+ syncmaster/dto/__init__.py
57
+ syncmaster/dto/connections.py
58
+ syncmaster/dto/runs.py
59
+ syncmaster/dto/transfers.py
60
+ syncmaster/dto/transfers_resources.py
61
+ syncmaster/dto/transfers_strategy.py
62
+ syncmaster/errors/__init__.py
63
+ syncmaster/errors/base.py
64
+ syncmaster/errors/registration.py
65
+ syncmaster/errors/schemas/__init__.py
66
+ syncmaster/errors/schemas/bad_request.py
67
+ syncmaster/errors/schemas/invalid_request.py
68
+ syncmaster/errors/schemas/not_authorized.py
69
+ syncmaster/exceptions/__init__.py
70
+ syncmaster/exceptions/auth.py
71
+ syncmaster/exceptions/base.py
72
+ syncmaster/exceptions/connection.py
73
+ syncmaster/exceptions/credentials.py
74
+ syncmaster/exceptions/group.py
75
+ syncmaster/exceptions/queue.py
76
+ syncmaster/exceptions/redirect.py
77
+ syncmaster/exceptions/run.py
78
+ syncmaster/exceptions/transfer.py
79
+ syncmaster/exceptions/user.py
80
+ syncmaster/scheduler/__init__.py
81
+ syncmaster/scheduler/__main__.py
82
+ syncmaster/scheduler/celery.py
83
+ syncmaster/scheduler/transfer_fetcher.py
84
+ syncmaster/scheduler/transfer_job_manager.py
85
+ syncmaster/scheduler/utils.py
86
+ syncmaster/scheduler/settings/__init__.py
87
+ syncmaster/schemas/__init__.py
88
+ syncmaster/schemas/ping.py
89
+ syncmaster/schemas/v1/__init__.py
90
+ syncmaster/schemas/v1/connection_types.py
91
+ syncmaster/schemas/v1/file_formats.py
92
+ syncmaster/schemas/v1/groups.py
93
+ syncmaster/schemas/v1/page.py
94
+ syncmaster/schemas/v1/queue.py
95
+ syncmaster/schemas/v1/transformation_types.py
96
+ syncmaster/schemas/v1/types.py
97
+ syncmaster/schemas/v1/users.py
98
+ syncmaster/schemas/v1/auth/__init__.py
99
+ syncmaster/schemas/v1/auth/basic.py
100
+ syncmaster/schemas/v1/auth/mixins.py
101
+ syncmaster/schemas/v1/auth/s3.py
102
+ syncmaster/schemas/v1/auth/samba.py
103
+ syncmaster/schemas/v1/auth/token.py
104
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py
105
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py
106
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py
107
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py
108
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py
109
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py
110
+ syncmaster/schemas/v1/connections/__init__.py
111
+ syncmaster/schemas/v1/connections/clickhouse.py
112
+ syncmaster/schemas/v1/connections/connection_base.py
113
+ syncmaster/schemas/v1/connections/ftp.py
114
+ syncmaster/schemas/v1/connections/ftps.py
115
+ syncmaster/schemas/v1/connections/hdfs.py
116
+ syncmaster/schemas/v1/connections/hive.py
117
+ syncmaster/schemas/v1/connections/iceberg.py
118
+ syncmaster/schemas/v1/connections/mssql.py
119
+ syncmaster/schemas/v1/connections/mysql.py
120
+ syncmaster/schemas/v1/connections/oracle.py
121
+ syncmaster/schemas/v1/connections/postgres.py
122
+ syncmaster/schemas/v1/connections/s3.py
123
+ syncmaster/schemas/v1/connections/samba.py
124
+ syncmaster/schemas/v1/connections/sftp.py
125
+ syncmaster/schemas/v1/connections/webdav.py
126
+ syncmaster/schemas/v1/transfers/__init__.py
127
+ syncmaster/schemas/v1/transfers/db.py
128
+ syncmaster/schemas/v1/transfers/file_format.py
129
+ syncmaster/schemas/v1/transfers/resources.py
130
+ syncmaster/schemas/v1/transfers/run.py
131
+ syncmaster/schemas/v1/transfers/strategy.py
132
+ syncmaster/schemas/v1/transfers/file/__init__.py
133
+ syncmaster/schemas/v1/transfers/file/base.py
134
+ syncmaster/schemas/v1/transfers/file/ftp.py
135
+ syncmaster/schemas/v1/transfers/file/ftps.py
136
+ syncmaster/schemas/v1/transfers/file/hdfs.py
137
+ syncmaster/schemas/v1/transfers/file/s3.py
138
+ syncmaster/schemas/v1/transfers/file/samba.py
139
+ syncmaster/schemas/v1/transfers/file/sftp.py
140
+ syncmaster/schemas/v1/transfers/file/webdav.py
141
+ syncmaster/schemas/v1/transfers/transformations/__init__.py
142
+ syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py
143
+ syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py
144
+ syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py
145
+ syncmaster/server/__init__.py
146
+ syncmaster/server/__main__.py
147
+ syncmaster/server/handler.py
148
+ syncmaster/server/api/__init__.py
149
+ syncmaster/server/api/monitoring.py
150
+ syncmaster/server/api/router.py
151
+ syncmaster/server/api/v1/__init__.py
152
+ syncmaster/server/api/v1/auth.py
153
+ syncmaster/server/api/v1/connections.py
154
+ syncmaster/server/api/v1/groups.py
155
+ syncmaster/server/api/v1/queue.py
156
+ syncmaster/server/api/v1/router.py
157
+ syncmaster/server/api/v1/runs.py
158
+ syncmaster/server/api/v1/transfers.py
159
+ syncmaster/server/api/v1/users.py
160
+ syncmaster/server/dependencies/__init__.py
161
+ syncmaster/server/dependencies/get_access_token.py
162
+ syncmaster/server/dependencies/stub.py
163
+ syncmaster/server/middlewares/__init__.py
164
+ syncmaster/server/middlewares/cors.py
165
+ syncmaster/server/middlewares/openapi.py
166
+ syncmaster/server/middlewares/request_id.py
167
+ syncmaster/server/middlewares/static_files.py
168
+ syncmaster/server/middlewares/monitoring/__init__.py
169
+ syncmaster/server/middlewares/monitoring/metrics.py
170
+ syncmaster/server/providers/__init__.py
171
+ syncmaster/server/providers/auth/__init__.py
172
+ syncmaster/server/providers/auth/base_provider.py
173
+ syncmaster/server/providers/auth/dummy_provider.py
174
+ syncmaster/server/providers/auth/keycloak_provider.py
175
+ syncmaster/server/providers/auth/oauth2_gateway_provider.py
176
+ syncmaster/server/scripts/export_openapi_schema.py
177
+ syncmaster/server/scripts/manage_superusers.py
178
+ syncmaster/server/services/__init__.py
179
+ syncmaster/server/services/get_user.py
180
+ syncmaster/server/services/unit_of_work.py
181
+ syncmaster/server/settings/__init__.py
182
+ syncmaster/server/settings/auth/__init__.py
183
+ syncmaster/server/settings/auth/dummy.py
184
+ syncmaster/server/settings/auth/jwt.py
185
+ syncmaster/server/settings/auth/keycloak.py
186
+ syncmaster/server/settings/auth/oauth2_gateway.py
187
+ syncmaster/server/settings/server/__init__.py
188
+ syncmaster/server/settings/server/cors.py
189
+ syncmaster/server/settings/server/monitoring.py
190
+ syncmaster/server/settings/server/openapi.py
191
+ syncmaster/server/settings/server/request_id.py
192
+ syncmaster/server/settings/server/static_files.py
193
+ syncmaster/server/utils/__init__.py
194
+ syncmaster/server/utils/jwt.py
195
+ syncmaster/server/utils/slug.py
196
+ syncmaster/settings/__init__.py
197
+ syncmaster/settings/base.py
198
+ syncmaster/settings/broker.py
199
+ syncmaster/settings/credentials.py
200
+ syncmaster/settings/database.py
201
+ syncmaster/settings/logging.py
202
+ syncmaster/worker/__init__.py
203
+ syncmaster/worker/base.py
204
+ syncmaster/worker/celery.py
205
+ syncmaster/worker/controller.py
206
+ syncmaster/worker/ivy2.py
207
+ syncmaster/worker/spark.py
208
+ syncmaster/worker/transfer.py
209
+ syncmaster/worker/handlers/__init__.py
210
+ syncmaster/worker/handlers/base.py
211
+ syncmaster/worker/handlers/db/__init__.py
212
+ syncmaster/worker/handlers/db/base.py
213
+ syncmaster/worker/handlers/db/clickhouse.py
214
+ syncmaster/worker/handlers/db/hive.py
215
+ syncmaster/worker/handlers/db/iceberg.py
216
+ syncmaster/worker/handlers/db/mssql.py
217
+ syncmaster/worker/handlers/db/mysql.py
218
+ syncmaster/worker/handlers/db/oracle.py
219
+ syncmaster/worker/handlers/db/postgres.py
220
+ syncmaster/worker/handlers/file/__init__.py
221
+ syncmaster/worker/handlers/file/base.py
222
+ syncmaster/worker/handlers/file/ftp.py
223
+ syncmaster/worker/handlers/file/ftps.py
224
+ syncmaster/worker/handlers/file/hdfs.py
225
+ syncmaster/worker/handlers/file/local_df.py
226
+ syncmaster/worker/handlers/file/remote_df.py
227
+ syncmaster/worker/handlers/file/s3.py
228
+ syncmaster/worker/handlers/file/samba.py
229
+ syncmaster/worker/handlers/file/sftp.py
230
+ syncmaster/worker/handlers/file/webdav.py
231
+ syncmaster/worker/settings/__init__.py
232
+ syncmaster/worker/settings/hwm_store.py
@@ -0,0 +1,36 @@
1
+ pydantic~=2.12.5
2
+ pydantic-settings~=2.12.0
3
+ pydantic-settings-logging~=0.1.1
4
+ sqlalchemy~=2.0.44
5
+ sqlalchemy-utils~=0.42.0
6
+ pyyaml~=6.0.3
7
+ coloredlogs~=15.0.1
8
+ python-json-logger~=4.0.0
9
+ asgi-correlation-id~=4.3.4
10
+ uuid6~=2025.0.1
11
+ celery~=5.6.0
12
+ psycopg2-binary~=2.9.11
13
+
14
+ [kerberos]
15
+ onetl[kerberos]~=0.15.0
16
+
17
+ [scheduler]
18
+ asyncpg~=0.31
19
+ apscheduler~=3.11.1
20
+
21
+ [server]
22
+ fastapi~=0.128.0
23
+ starlette-exporter~=0.23.0
24
+ python-multipart~=0.0.21
25
+ uvicorn~=0.40.0
26
+ alembic~=1.18.1
27
+ pyjwt~=2.10.1
28
+ asyncpg~=0.31
29
+ python-keycloak~=7.0.1
30
+ itsdangerous~=2.2.0
31
+
32
+ [worker]
33
+ onetl[files]~=0.15.0
34
+ pyspark<4.0.0
35
+ horizon-hwm-store~=1.1.4
36
+ jinja2~=3.1.6