data-syncmaster 0.3.0__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. data_syncmaster-0.3.2/PKG-INFO +135 -0
  2. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/README.rst +10 -10
  3. data_syncmaster-0.3.2/data_syncmaster.egg-info/PKG-INFO +135 -0
  4. data_syncmaster-0.3.2/data_syncmaster.egg-info/SOURCES.txt +233 -0
  5. data_syncmaster-0.3.2/data_syncmaster.egg-info/dependency_links.txt +1 -0
  6. data_syncmaster-0.3.2/data_syncmaster.egg-info/requires.txt +36 -0
  7. data_syncmaster-0.3.2/data_syncmaster.egg-info/top_level.txt +1 -0
  8. data_syncmaster-0.3.2/pyproject.toml +327 -0
  9. data_syncmaster-0.3.2/setup.cfg +4 -0
  10. data_syncmaster-0.3.2/syncmaster/VERSION +1 -0
  11. data_syncmaster-0.3.2/syncmaster/__init__.py +11 -0
  12. data_syncmaster-0.3.2/syncmaster/db/migrations/__init__.py +2 -0
  13. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/env.py +6 -8
  14. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py +2 -4
  15. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py +1 -0
  16. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py +7 -1
  17. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py +33 -7
  18. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/mixins/timestamp.py +2 -2
  19. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/auth_data.py +2 -0
  20. data_syncmaster-0.3.2/syncmaster/db/models/base.py +23 -0
  21. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/connection.py +9 -10
  22. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/group.py +5 -6
  23. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/queue.py +4 -8
  24. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/run.py +3 -1
  25. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/transfer.py +14 -17
  26. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/user.py +2 -0
  27. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/base.py +11 -9
  28. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/connection.py +4 -6
  29. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/credentials_repository.py +2 -1
  30. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/group.py +4 -4
  31. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/queue.py +7 -21
  32. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/repository_with_owner.py +3 -4
  33. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/run.py +8 -7
  34. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/search.py +1 -1
  35. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/transfer.py +10 -11
  36. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/user.py +3 -3
  37. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/utils.py +1 -1
  38. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/dto/transfers.py +14 -17
  39. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/dto/transfers_strategy.py +2 -1
  40. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/registration.py +8 -4
  41. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/redirect.py +1 -1
  42. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/transfer.py +1 -3
  43. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/scheduler/__init__.py +2 -3
  44. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/scheduler/__main__.py +7 -4
  45. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/scheduler/settings/__init__.py +4 -4
  46. data_syncmaster-0.3.2/syncmaster/scheduler/transfer_fetcher.py +34 -0
  47. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/scheduler/transfer_job_manager.py +11 -4
  48. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/basic.py +3 -3
  49. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py +3 -3
  50. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py +6 -3
  51. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py +7 -4
  52. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py +7 -4
  53. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/mixins.py +0 -1
  54. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/s3.py +3 -3
  55. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/samba.py +3 -3
  56. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/clickhouse.py +1 -1
  57. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/ftp.py +1 -1
  58. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/ftps.py +1 -1
  59. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/iceberg.py +7 -5
  60. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/mssql.py +1 -1
  61. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/mysql.py +1 -1
  62. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/oracle.py +4 -2
  63. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/postgres.py +1 -1
  64. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/s3.py +2 -2
  65. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/samba.py +1 -1
  66. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/sftp.py +1 -1
  67. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/webdav.py +1 -1
  68. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/groups.py +4 -4
  69. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/queue.py +2 -2
  70. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/__init__.py +15 -7
  71. data_syncmaster-0.3.2/syncmaster/schemas/v1/transfers/db.py +95 -0
  72. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/base.py +26 -12
  73. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file_format.py +12 -12
  74. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/resources.py +6 -6
  75. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/transformations/__init__.py +4 -1
  76. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py +6 -6
  77. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py +24 -24
  78. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py +15 -11
  79. data_syncmaster-0.3.2/syncmaster/schemas/v1/transfers/transformations/sql.py +28 -0
  80. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transformation_types.py +1 -0
  81. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/types.py +1 -1
  82. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/__init__.py +8 -5
  83. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/auth.py +1 -1
  84. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/connections.py +31 -25
  85. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/groups.py +34 -32
  86. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/queue.py +27 -26
  87. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/runs.py +18 -17
  88. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/transfers.py +36 -30
  89. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/users.py +8 -11
  90. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/dependencies/stub.py +1 -3
  91. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/handler.py +7 -3
  92. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/openapi.py +9 -6
  93. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/providers/auth/base_provider.py +1 -1
  94. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/providers/auth/dummy_provider.py +15 -12
  95. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/providers/auth/keycloak_provider.py +13 -11
  96. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/providers/auth/oauth2_gateway_provider.py +18 -11
  97. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/scripts/export_openapi_schema.py +4 -2
  98. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/scripts/manage_superusers.py +14 -12
  99. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/services/get_user.py +12 -6
  100. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/services/unit_of_work.py +6 -2
  101. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/server/static_files.py +4 -2
  102. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/utils/jwt.py +6 -4
  103. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/settings/broker.py +1 -1
  104. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/settings/logging.py +6 -5
  105. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/__init__.py +2 -3
  106. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/base.py +0 -1
  107. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/controller.py +20 -20
  108. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/base.py +8 -8
  109. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/base.py +39 -19
  110. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/clickhouse.py +12 -11
  111. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/hive.py +10 -9
  112. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/iceberg.py +10 -9
  113. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/mssql.py +14 -10
  114. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/mysql.py +10 -9
  115. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/oracle.py +10 -9
  116. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/postgres.py +10 -9
  117. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/base.py +30 -20
  118. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/ftp.py +3 -2
  119. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/ftps.py +3 -2
  120. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/hdfs.py +3 -2
  121. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/local_df.py +31 -22
  122. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/remote_df.py +14 -5
  123. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/s3.py +15 -4
  124. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/samba.py +3 -2
  125. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/sftp.py +3 -2
  126. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/webdav.py +2 -1
  127. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/ivy2.py +12 -3
  128. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/settings/__init__.py +4 -4
  129. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/spark.py +6 -5
  130. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/transfer.py +7 -7
  131. data_syncmaster-0.3.0/PKG-INFO +0 -136
  132. data_syncmaster-0.3.0/pyproject.toml +0 -525
  133. data_syncmaster-0.3.0/syncmaster/__init__.py +0 -6
  134. data_syncmaster-0.3.0/syncmaster/db/migrations/README +0 -1
  135. data_syncmaster-0.3.0/syncmaster/db/migrations/script.py.mako +0 -24
  136. data_syncmaster-0.3.0/syncmaster/db/models/base.py +0 -28
  137. data_syncmaster-0.3.0/syncmaster/scheduler/transfer_fetcher.py +0 -24
  138. data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/db.py +0 -62
  139. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/LICENSE.txt +0 -0
  140. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/__init__.py +0 -0
  141. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/factory.py +0 -0
  142. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/__main__.py +0 -0
  143. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/alembic.ini +0 -0
  144. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py +0 -0
  145. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py +0 -0
  146. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py +0 -0
  147. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py +0 -0
  148. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py +0 -0
  149. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py +0 -0
  150. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py +0 -0
  151. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py +0 -0
  152. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/migrations/versions/__init__.py +0 -0
  153. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/mixins/__init__.py +0 -0
  154. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/mixins/resource.py +0 -0
  155. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/__init__.py +7 -7
  156. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/models/apscheduler_job.py +0 -0
  157. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/__init__.py +0 -0
  158. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/db/repositories/utils.py +0 -0
  159. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/dto/__init__.py +0 -0
  160. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/dto/connections.py +0 -0
  161. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/dto/runs.py +0 -0
  162. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/dto/transfers_resources.py +0 -0
  163. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/__init__.py +0 -0
  164. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/base.py +0 -0
  165. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/schemas/__init__.py +1 -1
  166. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/schemas/bad_request.py +0 -0
  167. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/schemas/invalid_request.py +0 -0
  168. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/errors/schemas/not_authorized.py +0 -0
  169. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/__init__.py +0 -0
  170. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/auth.py +0 -0
  171. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/base.py +0 -0
  172. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/connection.py +0 -0
  173. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/credentials.py +0 -0
  174. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/group.py +0 -0
  175. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/queue.py +0 -0
  176. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/run.py +0 -0
  177. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/exceptions/user.py +0 -0
  178. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/scheduler/celery.py +0 -0
  179. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/scheduler/utils.py +0 -0
  180. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/__init__.py +0 -0
  181. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/ping.py +0 -0
  182. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/__init__.py +25 -25
  183. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/__init__.py +6 -6
  184. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py +0 -0
  185. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py +0 -0
  186. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/auth/token.py +0 -0
  187. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connection_types.py +0 -0
  188. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/__init__.py +0 -0
  189. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/connection_base.py +0 -0
  190. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/hdfs.py +0 -0
  191. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/connections/hive.py +0 -0
  192. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/file_formats.py +0 -0
  193. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/page.py +0 -0
  194. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/__init__.py +0 -0
  195. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/ftp.py +0 -0
  196. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/ftps.py +0 -0
  197. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -0
  198. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/s3.py +0 -0
  199. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/samba.py +0 -0
  200. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/sftp.py +0 -0
  201. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/file/webdav.py +0 -0
  202. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/run.py +0 -0
  203. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/transfers/strategy.py +1 -1
  204. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/schemas/v1/users.py +0 -0
  205. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/__main__.py +0 -0
  206. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/__init__.py +0 -0
  207. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/monitoring.py +0 -0
  208. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/router.py +0 -0
  209. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/__init__.py +0 -0
  210. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/api/v1/router.py +0 -0
  211. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/dependencies/__init__.py +1 -1
  212. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/dependencies/get_access_token.py +0 -0
  213. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/__init__.py +0 -0
  214. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/cors.py +0 -0
  215. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/monitoring/__init__.py +0 -0
  216. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/monitoring/metrics.py +0 -0
  217. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/request_id.py +0 -0
  218. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/middlewares/static_files.py +0 -0
  219. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/providers/__init__.py +0 -0
  220. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/providers/auth/__init__.py +0 -0
  221. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/services/__init__.py +0 -0
  222. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/__init__.py +0 -0
  223. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/auth/__init__.py +0 -0
  224. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/auth/dummy.py +0 -0
  225. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/auth/jwt.py +0 -0
  226. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/auth/keycloak.py +0 -0
  227. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/auth/oauth2_gateway.py +0 -0
  228. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/server/__init__.py +0 -0
  229. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/server/cors.py +0 -0
  230. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/server/monitoring.py +0 -0
  231. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/server/openapi.py +0 -0
  232. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/settings/server/request_id.py +0 -0
  233. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/utils/__init__.py +0 -0
  234. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/server/utils/slug.py +0 -0
  235. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/settings/__init__.py +1 -1
  236. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/settings/base.py +0 -0
  237. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/settings/credentials.py +0 -0
  238. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/settings/database.py +0 -0
  239. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/celery.py +0 -0
  240. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/__init__.py +0 -0
  241. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/db/__init__.py +0 -0
  242. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/handlers/file/__init__.py +0 -0
  243. {data_syncmaster-0.3.0 → data_syncmaster-0.3.2}/syncmaster/worker/settings/hwm_store.py +0 -0
@@ -0,0 +1,135 @@
1
+ Metadata-Version: 2.4
2
+ Name: data-syncmaster
3
+ Version: 0.3.2
4
+ Summary: Syncmaster REST API + scheduler + Worker
5
+ Author-email: MWS Data Bridge <onetools@mts.ru>
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/MTSWebServices/syncmaster
8
+ Project-URL: Documentation, https://syncmaster.readthedocs.io/
9
+ Project-URL: Source, https://github.com/MTSWebServices/syncmaster
10
+ Project-URL: CI/CD, https://github.com/MTSWebServices/syncmaster/actions
11
+ Project-URL: Tracker, https://github.com/MTSWebServices/syncmaster/issues
12
+ Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Framework :: Pydantic
15
+ Classifier: Framework :: Pydantic :: 2
16
+ Classifier: Framework :: FastAPI
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Software Development :: Libraries
24
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/x-rst
28
+ License-File: LICENSE.txt
29
+ Requires-Dist: pydantic~=2.12.5
30
+ Requires-Dist: pydantic-settings<2.14,>=2.12
31
+ Requires-Dist: pydantic-settings-logging~=0.1.1
32
+ Requires-Dist: sqlalchemy~=2.0.44
33
+ Requires-Dist: sqlalchemy-utils~=0.42.0
34
+ Requires-Dist: pyyaml~=6.0.3
35
+ Requires-Dist: coloredlogs~=15.0.1
36
+ Requires-Dist: python-json-logger~=4.0.0
37
+ Requires-Dist: asgi-correlation-id~=4.3.4
38
+ Requires-Dist: uuid6~=2025.0.1
39
+ Requires-Dist: celery~=5.6.0
40
+ Requires-Dist: psycopg2-binary~=2.9.11
41
+ Provides-Extra: server
42
+ Requires-Dist: fastapi~=0.129; extra == "server"
43
+ Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
44
+ Requires-Dist: python-multipart~=0.0.21; extra == "server"
45
+ Requires-Dist: uvicorn~=0.41.0; extra == "server"
46
+ Requires-Dist: alembic~=1.18.1; extra == "server"
47
+ Requires-Dist: pyjwt~=2.11.0; extra == "server"
48
+ Requires-Dist: asyncpg~=0.31; extra == "server"
49
+ Requires-Dist: python-keycloak<7.2.0,>=7.0.1; extra == "server"
50
+ Requires-Dist: itsdangerous~=2.2.0; extra == "server"
51
+ Provides-Extra: scheduler
52
+ Requires-Dist: asyncpg~=0.31; extra == "scheduler"
53
+ Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
54
+ Provides-Extra: worker
55
+ Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
56
+ Requires-Dist: pyspark<4.0.0; extra == "worker"
57
+ Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
58
+ Requires-Dist: jinja2~=3.1.6; extra == "worker"
59
+ Provides-Extra: kerberos
60
+ Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
61
+ Dynamic: license-file
62
+
63
+ .. _readme:
64
+
65
+ |Logo|
66
+
67
+ .. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/b6622b1b139137070f2287e288a4105c1110c1e4/docs/_static/logo_wide.svg
68
+ :alt: Data.SyncMaster logo
69
+ :target: https://github.com/MTSWebServices/data-syncmaster
70
+
71
+ |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
72
+ |Build Status| |Coverage| |pre-commit.ci|
73
+
74
+ .. |Repo Status| image:: https://www.repostatus.org/badges/latest/wip.svg
75
+ :target: https://www.repostatus.org/#wip
76
+ .. |Docker image| image:: https://img.shields.io/docker/v/mtsrus/syncmaster-server?sort=semver&label=docker
77
+ :target: https://hub.docker.com/r/mtsrus/syncmaster-server
78
+ .. |PyPI| image:: https://img.shields.io/pypi/v/data-syncmaster
79
+ :target: https://pypi.org/project/data-syncmaster/
80
+ .. |PyPI License| image:: https://img.shields.io/pypi/l/data-syncmaster.svg
81
+ :target: https://github.com/MTSWebServices/syncmaster/blob/develop/LICENSE.txt
82
+ .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/data-syncmaster.svg
83
+ :target: https://badge.fury.io/py/data-syncmaster
84
+ .. |Documentation| image:: https://readthedocs.org/projects/syncmaster/badge/?version=stable
85
+ :target: https://syncmaster.readthedocs.io
86
+ .. |Build Status| image:: https://github.com/MTSWebServices/syncmaster/workflows/Run%20All%20Tests/badge.svg
87
+ :target: https://github.com/MTSWebServices/syncmaster/actions
88
+ .. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
89
+ MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
90
+ :target: https://github.com/MTSWebServices/syncmaster/actions
91
+ .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MTSWebServices/syncmaster/develop.svg
92
+ :target: https://results.pre-commit.ci/latest/github/MTSWebServices/syncmaster/develop
93
+
94
+ What is Data.SyncMaster?
95
+ ------------------------
96
+
97
+ Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
98
+ List of currently supported connections:
99
+
100
+ * Apache Hive
101
+ * Clickhouse
102
+ * Iceberg (REST Catalog + S3)
103
+ * MSSQL
104
+ * MySQL
105
+ * Oracle
106
+ * Postgres
107
+ * FTP
108
+ * FTPS
109
+ * HDFS
110
+ * S3
111
+ * Samba
112
+ * SFTP
113
+ * WebDAV
114
+
115
+ Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
116
+
117
+ Goals
118
+ -----
119
+
120
+ * Make transferring data between databases and file systems as simple as possible
121
+ * Provide a lot of builtin connectors to transfer data in heterogeneous environment
122
+ * RBAC and multitenancy support
123
+
124
+ Non-goals
125
+ ---------
126
+
127
+ * No streaming support for now, only batch operations
128
+ * This is not a change data capture (CDC) solution
129
+
130
+ .. documentation
131
+
132
+ Documentation
133
+ -------------
134
+
135
+ See https://syncmaster.readthedocs.io
@@ -2,9 +2,9 @@
2
2
 
3
3
  |Logo|
4
4
 
5
- .. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/e3afdfe835fc6ee943f1105cfe3d624c01123bd6/docs/_static/logo_wide_red_text.svg
5
+ .. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/b6622b1b139137070f2287e288a4105c1110c1e4/docs/_static/logo_wide.svg
6
6
  :alt: Data.SyncMaster logo
7
- :target: https://github.com/MobileTeleSystems/data-syncmaster
7
+ :target: https://github.com/MTSWebServices/data-syncmaster
8
8
 
9
9
  |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
10
10
  |Build Status| |Coverage| |pre-commit.ci|
@@ -16,23 +16,23 @@
16
16
  .. |PyPI| image:: https://img.shields.io/pypi/v/data-syncmaster
17
17
  :target: https://pypi.org/project/data-syncmaster/
18
18
  .. |PyPI License| image:: https://img.shields.io/pypi/l/data-syncmaster.svg
19
- :target: https://github.com/MobileTeleSystems/syncmaster/blob/develop/LICENSE.txt
19
+ :target: https://github.com/MTSWebServices/syncmaster/blob/develop/LICENSE.txt
20
20
  .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/data-syncmaster.svg
21
21
  :target: https://badge.fury.io/py/data-syncmaster
22
22
  .. |Documentation| image:: https://readthedocs.org/projects/syncmaster/badge/?version=stable
23
23
  :target: https://syncmaster.readthedocs.io
24
- .. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Run%20All%20Tests/badge.svg
25
- :target: https://github.com/MobileTeleSystems/syncmaster/actions
24
+ .. |Build Status| image:: https://github.com/MTSWebServices/syncmaster/workflows/Run%20All%20Tests/badge.svg
25
+ :target: https://github.com/MTSWebServices/syncmaster/actions
26
26
  .. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
27
27
  MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
28
- :target: https://github.com/MobileTeleSystems/syncmaster/actions
29
- .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
30
- :target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
28
+ :target: https://github.com/MTSWebServices/syncmaster/actions
29
+ .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MTSWebServices/syncmaster/develop.svg
30
+ :target: https://results.pre-commit.ci/latest/github/MTSWebServices/syncmaster/develop
31
31
 
32
32
  What is Data.SyncMaster?
33
33
  ------------------------
34
34
 
35
- Data.SyncMaster is as no-code ETL tool for transfering data between databases and file systems.
35
+ Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
36
36
  List of currently supported connections:
37
37
 
38
38
  * Apache Hive
@@ -55,7 +55,7 @@ Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spa
55
55
  Goals
56
56
  -----
57
57
 
58
- * Make transfering data between databases and file systems as simple as possible
58
+ * Make transferring data between databases and file systems as simple as possible
59
59
  * Provide a lot of builtin connectors to transfer data in heterogeneous environment
60
60
  * RBAC and multitenancy support
61
61
 
@@ -0,0 +1,135 @@
1
+ Metadata-Version: 2.4
2
+ Name: data-syncmaster
3
+ Version: 0.3.2
4
+ Summary: Syncmaster REST API + scheduler + Worker
5
+ Author-email: MWS Data Bridge <onetools@mts.ru>
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/MTSWebServices/syncmaster
8
+ Project-URL: Documentation, https://syncmaster.readthedocs.io/
9
+ Project-URL: Source, https://github.com/MTSWebServices/syncmaster
10
+ Project-URL: CI/CD, https://github.com/MTSWebServices/syncmaster/actions
11
+ Project-URL: Tracker, https://github.com/MTSWebServices/syncmaster/issues
12
+ Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Framework :: Pydantic
15
+ Classifier: Framework :: Pydantic :: 2
16
+ Classifier: Framework :: FastAPI
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Software Development :: Libraries
24
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/x-rst
28
+ License-File: LICENSE.txt
29
+ Requires-Dist: pydantic~=2.12.5
30
+ Requires-Dist: pydantic-settings<2.14,>=2.12
31
+ Requires-Dist: pydantic-settings-logging~=0.1.1
32
+ Requires-Dist: sqlalchemy~=2.0.44
33
+ Requires-Dist: sqlalchemy-utils~=0.42.0
34
+ Requires-Dist: pyyaml~=6.0.3
35
+ Requires-Dist: coloredlogs~=15.0.1
36
+ Requires-Dist: python-json-logger~=4.0.0
37
+ Requires-Dist: asgi-correlation-id~=4.3.4
38
+ Requires-Dist: uuid6~=2025.0.1
39
+ Requires-Dist: celery~=5.6.0
40
+ Requires-Dist: psycopg2-binary~=2.9.11
41
+ Provides-Extra: server
42
+ Requires-Dist: fastapi~=0.129; extra == "server"
43
+ Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
44
+ Requires-Dist: python-multipart~=0.0.21; extra == "server"
45
+ Requires-Dist: uvicorn~=0.41.0; extra == "server"
46
+ Requires-Dist: alembic~=1.18.1; extra == "server"
47
+ Requires-Dist: pyjwt~=2.11.0; extra == "server"
48
+ Requires-Dist: asyncpg~=0.31; extra == "server"
49
+ Requires-Dist: python-keycloak<7.2.0,>=7.0.1; extra == "server"
50
+ Requires-Dist: itsdangerous~=2.2.0; extra == "server"
51
+ Provides-Extra: scheduler
52
+ Requires-Dist: asyncpg~=0.31; extra == "scheduler"
53
+ Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
54
+ Provides-Extra: worker
55
+ Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
56
+ Requires-Dist: pyspark<4.0.0; extra == "worker"
57
+ Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
58
+ Requires-Dist: jinja2~=3.1.6; extra == "worker"
59
+ Provides-Extra: kerberos
60
+ Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
61
+ Dynamic: license-file
62
+
63
+ .. _readme:
64
+
65
+ |Logo|
66
+
67
+ .. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/b6622b1b139137070f2287e288a4105c1110c1e4/docs/_static/logo_wide.svg
68
+ :alt: Data.SyncMaster logo
69
+ :target: https://github.com/MTSWebServices/data-syncmaster
70
+
71
+ |Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
72
+ |Build Status| |Coverage| |pre-commit.ci|
73
+
74
+ .. |Repo Status| image:: https://www.repostatus.org/badges/latest/wip.svg
75
+ :target: https://www.repostatus.org/#wip
76
+ .. |Docker image| image:: https://img.shields.io/docker/v/mtsrus/syncmaster-server?sort=semver&label=docker
77
+ :target: https://hub.docker.com/r/mtsrus/syncmaster-server
78
+ .. |PyPI| image:: https://img.shields.io/pypi/v/data-syncmaster
79
+ :target: https://pypi.org/project/data-syncmaster/
80
+ .. |PyPI License| image:: https://img.shields.io/pypi/l/data-syncmaster.svg
81
+ :target: https://github.com/MTSWebServices/syncmaster/blob/develop/LICENSE.txt
82
+ .. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/data-syncmaster.svg
83
+ :target: https://badge.fury.io/py/data-syncmaster
84
+ .. |Documentation| image:: https://readthedocs.org/projects/syncmaster/badge/?version=stable
85
+ :target: https://syncmaster.readthedocs.io
86
+ .. |Build Status| image:: https://github.com/MTSWebServices/syncmaster/workflows/Run%20All%20Tests/badge.svg
87
+ :target: https://github.com/MTSWebServices/syncmaster/actions
88
+ .. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
89
+ MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
90
+ :target: https://github.com/MTSWebServices/syncmaster/actions
91
+ .. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MTSWebServices/syncmaster/develop.svg
92
+ :target: https://results.pre-commit.ci/latest/github/MTSWebServices/syncmaster/develop
93
+
94
+ What is Data.SyncMaster?
95
+ ------------------------
96
+
97
+ Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
98
+ List of currently supported connections:
99
+
100
+ * Apache Hive
101
+ * Clickhouse
102
+ * Iceberg (REST Catalog + S3)
103
+ * MSSQL
104
+ * MySQL
105
+ * Oracle
106
+ * Postgres
107
+ * FTP
108
+ * FTPS
109
+ * HDFS
110
+ * S3
111
+ * Samba
112
+ * SFTP
113
+ * WebDAV
114
+
115
+ Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
116
+
117
+ Goals
118
+ -----
119
+
120
+ * Make transferring data between databases and file systems as simple as possible
121
+ * Provide a lot of builtin connectors to transfer data in heterogeneous environment
122
+ * RBAC and multitenancy support
123
+
124
+ Non-goals
125
+ ---------
126
+
127
+ * No streaming support for now, only batch operations
128
+ * This is not a change data capture (CDC) solution
129
+
130
+ .. documentation
131
+
132
+ Documentation
133
+ -------------
134
+
135
+ See https://syncmaster.readthedocs.io
@@ -0,0 +1,233 @@
1
+ LICENSE.txt
2
+ README.rst
3
+ pyproject.toml
4
+ data_syncmaster.egg-info/PKG-INFO
5
+ data_syncmaster.egg-info/SOURCES.txt
6
+ data_syncmaster.egg-info/dependency_links.txt
7
+ data_syncmaster.egg-info/requires.txt
8
+ data_syncmaster.egg-info/top_level.txt
9
+ syncmaster/VERSION
10
+ syncmaster/__init__.py
11
+ syncmaster/db/__init__.py
12
+ syncmaster/db/factory.py
13
+ syncmaster/db/utils.py
14
+ syncmaster/db/migrations/__init__.py
15
+ syncmaster/db/migrations/__main__.py
16
+ syncmaster/db/migrations/alembic.ini
17
+ syncmaster/db/migrations/env.py
18
+ syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py
19
+ syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py
20
+ syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py
21
+ syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py
22
+ syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py
23
+ syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py
24
+ syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py
25
+ syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py
26
+ syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py
27
+ syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py
28
+ syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py
29
+ syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py
30
+ syncmaster/db/migrations/versions/__init__.py
31
+ syncmaster/db/mixins/__init__.py
32
+ syncmaster/db/mixins/resource.py
33
+ syncmaster/db/mixins/timestamp.py
34
+ syncmaster/db/models/__init__.py
35
+ syncmaster/db/models/apscheduler_job.py
36
+ syncmaster/db/models/auth_data.py
37
+ syncmaster/db/models/base.py
38
+ syncmaster/db/models/connection.py
39
+ syncmaster/db/models/group.py
40
+ syncmaster/db/models/queue.py
41
+ syncmaster/db/models/run.py
42
+ syncmaster/db/models/transfer.py
43
+ syncmaster/db/models/user.py
44
+ syncmaster/db/repositories/__init__.py
45
+ syncmaster/db/repositories/base.py
46
+ syncmaster/db/repositories/connection.py
47
+ syncmaster/db/repositories/credentials_repository.py
48
+ syncmaster/db/repositories/group.py
49
+ syncmaster/db/repositories/queue.py
50
+ syncmaster/db/repositories/repository_with_owner.py
51
+ syncmaster/db/repositories/run.py
52
+ syncmaster/db/repositories/search.py
53
+ syncmaster/db/repositories/transfer.py
54
+ syncmaster/db/repositories/user.py
55
+ syncmaster/db/repositories/utils.py
56
+ syncmaster/dto/__init__.py
57
+ syncmaster/dto/connections.py
58
+ syncmaster/dto/runs.py
59
+ syncmaster/dto/transfers.py
60
+ syncmaster/dto/transfers_resources.py
61
+ syncmaster/dto/transfers_strategy.py
62
+ syncmaster/errors/__init__.py
63
+ syncmaster/errors/base.py
64
+ syncmaster/errors/registration.py
65
+ syncmaster/errors/schemas/__init__.py
66
+ syncmaster/errors/schemas/bad_request.py
67
+ syncmaster/errors/schemas/invalid_request.py
68
+ syncmaster/errors/schemas/not_authorized.py
69
+ syncmaster/exceptions/__init__.py
70
+ syncmaster/exceptions/auth.py
71
+ syncmaster/exceptions/base.py
72
+ syncmaster/exceptions/connection.py
73
+ syncmaster/exceptions/credentials.py
74
+ syncmaster/exceptions/group.py
75
+ syncmaster/exceptions/queue.py
76
+ syncmaster/exceptions/redirect.py
77
+ syncmaster/exceptions/run.py
78
+ syncmaster/exceptions/transfer.py
79
+ syncmaster/exceptions/user.py
80
+ syncmaster/scheduler/__init__.py
81
+ syncmaster/scheduler/__main__.py
82
+ syncmaster/scheduler/celery.py
83
+ syncmaster/scheduler/transfer_fetcher.py
84
+ syncmaster/scheduler/transfer_job_manager.py
85
+ syncmaster/scheduler/utils.py
86
+ syncmaster/scheduler/settings/__init__.py
87
+ syncmaster/schemas/__init__.py
88
+ syncmaster/schemas/ping.py
89
+ syncmaster/schemas/v1/__init__.py
90
+ syncmaster/schemas/v1/connection_types.py
91
+ syncmaster/schemas/v1/file_formats.py
92
+ syncmaster/schemas/v1/groups.py
93
+ syncmaster/schemas/v1/page.py
94
+ syncmaster/schemas/v1/queue.py
95
+ syncmaster/schemas/v1/transformation_types.py
96
+ syncmaster/schemas/v1/types.py
97
+ syncmaster/schemas/v1/users.py
98
+ syncmaster/schemas/v1/auth/__init__.py
99
+ syncmaster/schemas/v1/auth/basic.py
100
+ syncmaster/schemas/v1/auth/mixins.py
101
+ syncmaster/schemas/v1/auth/s3.py
102
+ syncmaster/schemas/v1/auth/samba.py
103
+ syncmaster/schemas/v1/auth/token.py
104
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py
105
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py
106
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py
107
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py
108
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py
109
+ syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py
110
+ syncmaster/schemas/v1/connections/__init__.py
111
+ syncmaster/schemas/v1/connections/clickhouse.py
112
+ syncmaster/schemas/v1/connections/connection_base.py
113
+ syncmaster/schemas/v1/connections/ftp.py
114
+ syncmaster/schemas/v1/connections/ftps.py
115
+ syncmaster/schemas/v1/connections/hdfs.py
116
+ syncmaster/schemas/v1/connections/hive.py
117
+ syncmaster/schemas/v1/connections/iceberg.py
118
+ syncmaster/schemas/v1/connections/mssql.py
119
+ syncmaster/schemas/v1/connections/mysql.py
120
+ syncmaster/schemas/v1/connections/oracle.py
121
+ syncmaster/schemas/v1/connections/postgres.py
122
+ syncmaster/schemas/v1/connections/s3.py
123
+ syncmaster/schemas/v1/connections/samba.py
124
+ syncmaster/schemas/v1/connections/sftp.py
125
+ syncmaster/schemas/v1/connections/webdav.py
126
+ syncmaster/schemas/v1/transfers/__init__.py
127
+ syncmaster/schemas/v1/transfers/db.py
128
+ syncmaster/schemas/v1/transfers/file_format.py
129
+ syncmaster/schemas/v1/transfers/resources.py
130
+ syncmaster/schemas/v1/transfers/run.py
131
+ syncmaster/schemas/v1/transfers/strategy.py
132
+ syncmaster/schemas/v1/transfers/file/__init__.py
133
+ syncmaster/schemas/v1/transfers/file/base.py
134
+ syncmaster/schemas/v1/transfers/file/ftp.py
135
+ syncmaster/schemas/v1/transfers/file/ftps.py
136
+ syncmaster/schemas/v1/transfers/file/hdfs.py
137
+ syncmaster/schemas/v1/transfers/file/s3.py
138
+ syncmaster/schemas/v1/transfers/file/samba.py
139
+ syncmaster/schemas/v1/transfers/file/sftp.py
140
+ syncmaster/schemas/v1/transfers/file/webdav.py
141
+ syncmaster/schemas/v1/transfers/transformations/__init__.py
142
+ syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py
143
+ syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py
144
+ syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py
145
+ syncmaster/schemas/v1/transfers/transformations/sql.py
146
+ syncmaster/server/__init__.py
147
+ syncmaster/server/__main__.py
148
+ syncmaster/server/handler.py
149
+ syncmaster/server/api/__init__.py
150
+ syncmaster/server/api/monitoring.py
151
+ syncmaster/server/api/router.py
152
+ syncmaster/server/api/v1/__init__.py
153
+ syncmaster/server/api/v1/auth.py
154
+ syncmaster/server/api/v1/connections.py
155
+ syncmaster/server/api/v1/groups.py
156
+ syncmaster/server/api/v1/queue.py
157
+ syncmaster/server/api/v1/router.py
158
+ syncmaster/server/api/v1/runs.py
159
+ syncmaster/server/api/v1/transfers.py
160
+ syncmaster/server/api/v1/users.py
161
+ syncmaster/server/dependencies/__init__.py
162
+ syncmaster/server/dependencies/get_access_token.py
163
+ syncmaster/server/dependencies/stub.py
164
+ syncmaster/server/middlewares/__init__.py
165
+ syncmaster/server/middlewares/cors.py
166
+ syncmaster/server/middlewares/openapi.py
167
+ syncmaster/server/middlewares/request_id.py
168
+ syncmaster/server/middlewares/static_files.py
169
+ syncmaster/server/middlewares/monitoring/__init__.py
170
+ syncmaster/server/middlewares/monitoring/metrics.py
171
+ syncmaster/server/providers/__init__.py
172
+ syncmaster/server/providers/auth/__init__.py
173
+ syncmaster/server/providers/auth/base_provider.py
174
+ syncmaster/server/providers/auth/dummy_provider.py
175
+ syncmaster/server/providers/auth/keycloak_provider.py
176
+ syncmaster/server/providers/auth/oauth2_gateway_provider.py
177
+ syncmaster/server/scripts/export_openapi_schema.py
178
+ syncmaster/server/scripts/manage_superusers.py
179
+ syncmaster/server/services/__init__.py
180
+ syncmaster/server/services/get_user.py
181
+ syncmaster/server/services/unit_of_work.py
182
+ syncmaster/server/settings/__init__.py
183
+ syncmaster/server/settings/auth/__init__.py
184
+ syncmaster/server/settings/auth/dummy.py
185
+ syncmaster/server/settings/auth/jwt.py
186
+ syncmaster/server/settings/auth/keycloak.py
187
+ syncmaster/server/settings/auth/oauth2_gateway.py
188
+ syncmaster/server/settings/server/__init__.py
189
+ syncmaster/server/settings/server/cors.py
190
+ syncmaster/server/settings/server/monitoring.py
191
+ syncmaster/server/settings/server/openapi.py
192
+ syncmaster/server/settings/server/request_id.py
193
+ syncmaster/server/settings/server/static_files.py
194
+ syncmaster/server/utils/__init__.py
195
+ syncmaster/server/utils/jwt.py
196
+ syncmaster/server/utils/slug.py
197
+ syncmaster/settings/__init__.py
198
+ syncmaster/settings/base.py
199
+ syncmaster/settings/broker.py
200
+ syncmaster/settings/credentials.py
201
+ syncmaster/settings/database.py
202
+ syncmaster/settings/logging.py
203
+ syncmaster/worker/__init__.py
204
+ syncmaster/worker/base.py
205
+ syncmaster/worker/celery.py
206
+ syncmaster/worker/controller.py
207
+ syncmaster/worker/ivy2.py
208
+ syncmaster/worker/spark.py
209
+ syncmaster/worker/transfer.py
210
+ syncmaster/worker/handlers/__init__.py
211
+ syncmaster/worker/handlers/base.py
212
+ syncmaster/worker/handlers/db/__init__.py
213
+ syncmaster/worker/handlers/db/base.py
214
+ syncmaster/worker/handlers/db/clickhouse.py
215
+ syncmaster/worker/handlers/db/hive.py
216
+ syncmaster/worker/handlers/db/iceberg.py
217
+ syncmaster/worker/handlers/db/mssql.py
218
+ syncmaster/worker/handlers/db/mysql.py
219
+ syncmaster/worker/handlers/db/oracle.py
220
+ syncmaster/worker/handlers/db/postgres.py
221
+ syncmaster/worker/handlers/file/__init__.py
222
+ syncmaster/worker/handlers/file/base.py
223
+ syncmaster/worker/handlers/file/ftp.py
224
+ syncmaster/worker/handlers/file/ftps.py
225
+ syncmaster/worker/handlers/file/hdfs.py
226
+ syncmaster/worker/handlers/file/local_df.py
227
+ syncmaster/worker/handlers/file/remote_df.py
228
+ syncmaster/worker/handlers/file/s3.py
229
+ syncmaster/worker/handlers/file/samba.py
230
+ syncmaster/worker/handlers/file/sftp.py
231
+ syncmaster/worker/handlers/file/webdav.py
232
+ syncmaster/worker/settings/__init__.py
233
+ syncmaster/worker/settings/hwm_store.py
@@ -0,0 +1,36 @@
1
+ pydantic~=2.12.5
2
+ pydantic-settings<2.14,>=2.12
3
+ pydantic-settings-logging~=0.1.1
4
+ sqlalchemy~=2.0.44
5
+ sqlalchemy-utils~=0.42.0
6
+ pyyaml~=6.0.3
7
+ coloredlogs~=15.0.1
8
+ python-json-logger~=4.0.0
9
+ asgi-correlation-id~=4.3.4
10
+ uuid6~=2025.0.1
11
+ celery~=5.6.0
12
+ psycopg2-binary~=2.9.11
13
+
14
+ [kerberos]
15
+ onetl[kerberos]~=0.15.0
16
+
17
+ [scheduler]
18
+ asyncpg~=0.31
19
+ apscheduler~=3.11.1
20
+
21
+ [server]
22
+ fastapi~=0.129
23
+ starlette-exporter~=0.23.0
24
+ python-multipart~=0.0.21
25
+ uvicorn~=0.41.0
26
+ alembic~=1.18.1
27
+ pyjwt~=2.11.0
28
+ asyncpg~=0.31
29
+ python-keycloak<7.2.0,>=7.0.1
30
+ itsdangerous~=2.2.0
31
+
32
+ [worker]
33
+ onetl[files]~=0.15.0
34
+ pyspark<4.0.0
35
+ horizon-hwm-store~=1.1.4
36
+ jinja2~=3.1.6