data-syncmaster 0.2.5__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/LICENSE.txt +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/PKG-INFO +60 -59
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/README.rst +15 -14
- data_syncmaster-0.3.1/data_syncmaster.egg-info/PKG-INFO +135 -0
- data_syncmaster-0.3.1/data_syncmaster.egg-info/SOURCES.txt +232 -0
- data_syncmaster-0.3.1/data_syncmaster.egg-info/dependency_links.txt +1 -0
- data_syncmaster-0.3.1/data_syncmaster.egg-info/requires.txt +36 -0
- data_syncmaster-0.3.1/data_syncmaster.egg-info/top_level.txt +1 -0
- data_syncmaster-0.3.1/pyproject.toml +326 -0
- data_syncmaster-0.3.1/setup.cfg +4 -0
- data_syncmaster-0.3.1/syncmaster/VERSION +1 -0
- data_syncmaster-0.3.1/syncmaster/__init__.py +11 -0
- data_syncmaster-0.3.1/syncmaster/db/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/factory.py +1 -1
- data_syncmaster-0.3.1/syncmaster/db/migrations/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/__main__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/env.py +19 -10
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py +2 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py +8 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py +35 -8
- data_syncmaster-0.3.1/syncmaster/db/migrations/versions/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/mixins/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/mixins/resource.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/mixins/timestamp.py +3 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/__init__.py +8 -8
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/apscheduler_job.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/auth_data.py +3 -1
- data_syncmaster-0.3.1/syncmaster/db/models/base.py +23 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/connection.py +11 -11
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/group.py +6 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/queue.py +5 -9
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/run.py +4 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/transfer.py +15 -18
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/models/user.py +3 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/base.py +12 -10
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/connection.py +5 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/credentials_repository.py +3 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/group.py +5 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/queue.py +8 -22
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/repository_with_owner.py +4 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/run.py +9 -8
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/search.py +2 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/transfer.py +11 -12
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/user.py +4 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/repositories/utils.py +1 -8
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/utils.py +2 -2
- data_syncmaster-0.3.1/syncmaster/dto/__init__.py +2 -0
- data_syncmaster-0.3.1/syncmaster/dto/connections.py +231 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/runs.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/transfers.py +49 -21
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/transfers_resources.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/dto/transfers_strategy.py +3 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/registration.py +9 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/__init__.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/bad_request.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/invalid_request.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/errors/schemas/not_authorized.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/auth.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/connection.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/credentials.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/group.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/queue.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/redirect.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/run.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/transfer.py +2 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/exceptions/user.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/__init__.py +3 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/__main__.py +10 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/celery.py +1 -1
- data_syncmaster-0.3.1/syncmaster/scheduler/settings/__init__.py +76 -0
- data_syncmaster-0.3.1/syncmaster/scheduler/transfer_fetcher.py +34 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/transfer_job_manager.py +14 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/scheduler/utils.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/ping.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/__init__.py +27 -37
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/auth/__init__.py +7 -13
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/basic.py +23 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py +31 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py +22 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py +32 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py +31 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py +25 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py +34 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/mixins.py +11 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/s3.py +23 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/auth/samba.py +24 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/auth/token.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connection_types.py +26 -7
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection.py → data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/__init__.py +39 -31
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/clickhouse.py +14 -19
- data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/connection_base.py +46 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/ftp.py +15 -17
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/ftps.py +12 -17
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/hdfs.py +14 -15
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/hive.py +14 -15
- data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/iceberg.py +107 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/mssql.py +15 -19
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/mysql.py +16 -20
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/oracle.py +18 -21
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/postgres.py +15 -19
- data_syncmaster-0.3.1/syncmaster/schemas/v1/connections/s3.py +65 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/samba.py +7 -18
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/sftp.py +14 -16
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/connections/webdav.py +15 -18
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/file_formats.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/groups.py +9 -14
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/page.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/queue.py +10 -11
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/__init__.py +102 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/db.py +87 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/__init__.py +55 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/file/base.py +33 -38
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/ftp.py +17 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/ftps.py +17 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/hdfs.py +17 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/s3.py +17 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/samba.py +17 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/sftp.py +17 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/file/webdav.py +17 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/file_format.py +1 -1
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/resources.py +17 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/run.py +10 -11
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/strategy.py +23 -0
- data_syncmaster-0.3.1/syncmaster/schemas/v1/transfers/transformations/__init__.py +25 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py +2 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py +2 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py +8 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/transformation_types.py +1 -1
- data_syncmaster-0.3.1/syncmaster/schemas/v1/types.py +8 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/schemas/v1/users.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/__init__.py +13 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/__main__.py +1 -1
- data_syncmaster-0.3.1/syncmaster/server/api/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/monitoring.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/router.py +1 -1
- data_syncmaster-0.3.1/syncmaster/server/api/v1/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/auth.py +3 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/connections.py +43 -37
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/groups.py +35 -33
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/queue.py +29 -28
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/router.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/runs.py +21 -20
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/transfers.py +47 -41
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/api/v1/users.py +11 -14
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/dependencies/__init__.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/dependencies/get_access_token.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/dependencies/stub.py +1 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/handler.py +8 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/__init__.py +1 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/cors.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/monitoring/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/monitoring/metrics.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/openapi.py +10 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/request_id.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/middlewares/static_files.py +1 -1
- data_syncmaster-0.3.1/syncmaster/server/providers/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/__init__.py +5 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/base_provider.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/dummy_provider.py +16 -11
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/providers/auth/keycloak_provider.py +31 -12
- data_syncmaster-0.3.1/syncmaster/server/providers/auth/oauth2_gateway_provider.py +123 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/scripts/export_openapi_schema.py +5 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/scripts/manage_superusers.py +46 -20
- data_syncmaster-0.3.1/syncmaster/server/services/__init__.py +2 -0
- data_syncmaster-0.3.1/syncmaster/server/services/get_user.py +67 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/services/unit_of_work.py +7 -3
- data_syncmaster-0.3.1/syncmaster/server/settings/__init__.py +70 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/auth/__init__.py +8 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/auth/dummy.py +7 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/auth/jwt.py +7 -4
- data_syncmaster-0.3.1/syncmaster/server/settings/auth/keycloak.py +135 -0
- data_syncmaster-0.3.1/syncmaster/server/settings/auth/oauth2_gateway.py +35 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/__init__.py +15 -10
- data_syncmaster-0.3.1/syncmaster/server/settings/server/cors.py +63 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/monitoring.py +16 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/openapi.py +48 -22
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/settings/server/request_id.py +8 -4
- data_syncmaster-0.3.1/syncmaster/server/settings/server/static_files.py +42 -0
- data_syncmaster-0.3.1/syncmaster/server/utils/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/utils/jwt.py +7 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/server/utils/slug.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/settings/__init__.py +6 -3
- data_syncmaster-0.3.1/syncmaster/settings/base.py +37 -0
- data_syncmaster-0.3.1/syncmaster/settings/broker.py +30 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/settings/credentials.py +5 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/settings/database.py +8 -5
- data_syncmaster-0.3.1/syncmaster/settings/logging.py +289 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/__init__.py +3 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/base.py +1 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/celery.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/controller.py +54 -19
- data_syncmaster-0.3.1/syncmaster/worker/handlers/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/base.py +9 -9
- data_syncmaster-0.3.1/syncmaster/worker/handlers/db/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/base.py +21 -19
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/clickhouse.py +20 -12
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/hive.py +18 -10
- data_syncmaster-0.3.1/syncmaster/worker/handlers/db/iceberg.py +135 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/mssql.py +25 -11
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/mysql.py +21 -10
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/oracle.py +21 -10
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/db/postgres.py +21 -10
- data_syncmaster-0.3.1/syncmaster/worker/handlers/file/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/base.py +25 -21
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/ftp.py +16 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/ftps.py +16 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/hdfs.py +16 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/local_df.py +22 -23
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/remote_df.py +5 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/s3.py +14 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/samba.py +16 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/sftp.py +16 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/handlers/file/webdav.py +14 -3
- data_syncmaster-0.3.1/syncmaster/worker/ivy2.py +68 -0
- data_syncmaster-0.3.1/syncmaster/worker/settings/__init__.py +95 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/settings/hwm_store.py +11 -9
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/spark.py +38 -53
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/worker/transfer.py +10 -10
- data_syncmaster-0.2.5/pyproject.toml +0 -546
- data_syncmaster-0.2.5/syncmaster/__init__.py +0 -6
- data_syncmaster-0.2.5/syncmaster/db/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/db/migrations/README +0 -1
- data_syncmaster-0.2.5/syncmaster/db/migrations/script.py.mako +0 -24
- data_syncmaster-0.2.5/syncmaster/db/migrations/versions/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/db/models/base.py +0 -28
- data_syncmaster-0.2.5/syncmaster/dto/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/dto/connections.py +0 -144
- data_syncmaster-0.2.5/syncmaster/scheduler/settings/__init__.py +0 -78
- data_syncmaster-0.2.5/syncmaster/scheduler/transfer_fetcher.py +0 -24
- data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/basic.py +0 -26
- data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/s3.py +0 -26
- data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/samba.py +0 -28
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection_base.py +0 -30
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/s3.py +0 -73
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfer_types.py +0 -6
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/__init__.py +0 -248
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/db.py +0 -42
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftp.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftps.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/s3.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/samba.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/sftp.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/webdav.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/resources.py +0 -14
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/strategy.py +0 -14
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/transformations/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/schemas/v1/types.py +0 -7
- data_syncmaster-0.2.5/syncmaster/server/api/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/api/v1/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/middlewares/session.py +0 -17
- data_syncmaster-0.2.5/syncmaster/server/providers/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/services/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/services/get_user.py +0 -41
- data_syncmaster-0.2.5/syncmaster/server/settings/__init__.py +0 -61
- data_syncmaster-0.2.5/syncmaster/server/settings/auth/keycloak.py +0 -22
- data_syncmaster-0.2.5/syncmaster/server/settings/server/cors.py +0 -70
- data_syncmaster-0.2.5/syncmaster/server/settings/server/session.py +0 -75
- data_syncmaster-0.2.5/syncmaster/server/settings/server/static_files.py +0 -34
- data_syncmaster-0.2.5/syncmaster/server/utils/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/settings/broker.py +0 -29
- data_syncmaster-0.2.5/syncmaster/settings/log/__init__.py +0 -110
- data_syncmaster-0.2.5/syncmaster/settings/log/colored.yml +0 -57
- data_syncmaster-0.2.5/syncmaster/settings/log/json.yml +0 -56
- data_syncmaster-0.2.5/syncmaster/settings/log/plain.yml +0 -57
- data_syncmaster-0.2.5/syncmaster/worker/handlers/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/worker/handlers/db/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/worker/handlers/file/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/worker/settings/__init__.py +0 -79
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.1}/syncmaster/db/migrations/alembic.ini +0 -0
|
@@ -1,68 +1,72 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: data-syncmaster
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Syncmaster REST API + Worker
|
|
5
|
-
|
|
6
|
-
License-
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Syncmaster REST API + scheduler + Worker
|
|
5
|
+
Author-email: MWS Data Bridge <onetools@mts.ru>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
|
|
8
|
+
Project-URL: Documentation, https://syncmaster.readthedocs.io/
|
|
9
|
+
Project-URL: Source, https://github.com/MobileTeleSystems/syncmaster
|
|
10
|
+
Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
|
|
11
|
+
Project-URL: Tracker, https://github.com/MobileTeleSystems/syncmaster/issues
|
|
7
12
|
Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
|
|
8
|
-
Author: DataOps.ETL
|
|
9
|
-
Author-email: onetools@mts.ru
|
|
10
|
-
Requires-Python: >=3.11,<4.0
|
|
11
13
|
Classifier: Development Status :: 3 - Alpha
|
|
12
|
-
Classifier: Framework :: FastAPI
|
|
13
14
|
Classifier: Framework :: Pydantic
|
|
14
15
|
Classifier: Framework :: Pydantic :: 2
|
|
16
|
+
Classifier: Framework :: FastAPI
|
|
15
17
|
Classifier: Intended Audience :: Developers
|
|
16
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
17
18
|
Classifier: Operating System :: OS Independent
|
|
18
|
-
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
22
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
-
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
24
23
|
Classifier: Topic :: Software Development :: Libraries
|
|
25
24
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
25
|
Classifier: Typing :: Typed
|
|
27
|
-
|
|
26
|
+
Requires-Python: >=3.11
|
|
27
|
+
Description-Content-Type: text/x-rst
|
|
28
|
+
License-File: LICENSE.txt
|
|
29
|
+
Requires-Dist: pydantic~=2.12.5
|
|
30
|
+
Requires-Dist: pydantic-settings~=2.12.0
|
|
31
|
+
Requires-Dist: pydantic-settings-logging~=0.1.1
|
|
32
|
+
Requires-Dist: sqlalchemy~=2.0.44
|
|
33
|
+
Requires-Dist: sqlalchemy-utils~=0.42.0
|
|
34
|
+
Requires-Dist: pyyaml~=6.0.3
|
|
35
|
+
Requires-Dist: coloredlogs~=15.0.1
|
|
36
|
+
Requires-Dist: python-json-logger~=4.0.0
|
|
37
|
+
Requires-Dist: asgi-correlation-id~=4.3.4
|
|
38
|
+
Requires-Dist: uuid6~=2025.0.1
|
|
39
|
+
Requires-Dist: celery~=5.6.0
|
|
40
|
+
Requires-Dist: psycopg2-binary~=2.9.11
|
|
28
41
|
Provides-Extra: server
|
|
42
|
+
Requires-Dist: fastapi~=0.128.0; extra == "server"
|
|
43
|
+
Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
|
|
44
|
+
Requires-Dist: python-multipart~=0.0.21; extra == "server"
|
|
45
|
+
Requires-Dist: uvicorn~=0.40.0; extra == "server"
|
|
46
|
+
Requires-Dist: alembic~=1.18.1; extra == "server"
|
|
47
|
+
Requires-Dist: pyjwt~=2.10.1; extra == "server"
|
|
48
|
+
Requires-Dist: asyncpg~=0.31; extra == "server"
|
|
49
|
+
Requires-Dist: python-keycloak~=7.0.1; extra == "server"
|
|
50
|
+
Requires-Dist: itsdangerous~=2.2.0; extra == "server"
|
|
51
|
+
Provides-Extra: scheduler
|
|
52
|
+
Requires-Dist: asyncpg~=0.31; extra == "scheduler"
|
|
53
|
+
Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
|
|
29
54
|
Provides-Extra: worker
|
|
30
|
-
Requires-Dist:
|
|
31
|
-
Requires-Dist:
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
|
|
35
|
-
Requires-Dist:
|
|
36
|
-
|
|
37
|
-
Requires-Dist: horizon-hwm-store (>=1.1.2,<2.0.0) ; extra == "worker"
|
|
38
|
-
Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
|
|
39
|
-
Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
|
|
40
|
-
Requires-Dist: onetl[all] (>=0.13.5,<0.15.0) ; extra == "worker"
|
|
41
|
-
Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0) ; extra == "server" or extra == "worker"
|
|
42
|
-
Requires-Dist: pydantic (>=2.11.7,<3.0.0)
|
|
43
|
-
Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
44
|
-
Requires-Dist: pyjwt (>=2.10.1,<3.0.0) ; extra == "server"
|
|
45
|
-
Requires-Dist: pyspark (<4.0.0) ; extra == "worker"
|
|
46
|
-
Requires-Dist: python-json-logger ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
47
|
-
Requires-Dist: python-keycloak (>=5.3.1,<6.0.0) ; extra == "server"
|
|
48
|
-
Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
|
|
49
|
-
Requires-Dist: pyyaml ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
50
|
-
Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
51
|
-
Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.43.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
52
|
-
Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
|
|
53
|
-
Requires-Dist: uuid6 (>=2025.0.0,<2026.0.0) ; extra == "server" or extra == "worker"
|
|
54
|
-
Requires-Dist: uvicorn (>=0.34,<0.38) ; extra == "server"
|
|
55
|
-
Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
|
|
56
|
-
Project-URL: Documentation, https://syncmaster.readthedocs.io
|
|
57
|
-
Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
|
|
58
|
-
Project-URL: Source, https://github.com/MobileTeleSystems/syncmaster
|
|
59
|
-
Project-URL: Tracker, https://github.com/MobileTeleSystems/syncmaster/issues
|
|
60
|
-
Description-Content-Type: text/x-rst
|
|
55
|
+
Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
|
|
56
|
+
Requires-Dist: pyspark<4.0.0; extra == "worker"
|
|
57
|
+
Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
|
|
58
|
+
Requires-Dist: jinja2~=3.1.6; extra == "worker"
|
|
59
|
+
Provides-Extra: kerberos
|
|
60
|
+
Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
|
|
61
|
+
Dynamic: license-file
|
|
61
62
|
|
|
62
63
|
.. _readme:
|
|
63
64
|
|
|
64
|
-
|
|
65
|
-
|
|
65
|
+
|Logo|
|
|
66
|
+
|
|
67
|
+
.. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/d14bf1b1272f15d42f441d6e33264bd2f95216c9/docs/_static/logo_wide.svg
|
|
68
|
+
:alt: Data.SyncMaster logo
|
|
69
|
+
:target: https://github.com/MobileTeleSystems/data-syncmaster
|
|
66
70
|
|
|
67
71
|
|Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
|
|
68
72
|
|Build Status| |Coverage| |pre-commit.ci|
|
|
@@ -87,43 +91,41 @@ Data.SyncMaster
|
|
|
87
91
|
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
88
92
|
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
89
93
|
|
|
90
|
-
|
|
91
94
|
What is Data.SyncMaster?
|
|
92
95
|
------------------------
|
|
93
96
|
|
|
94
|
-
Data.SyncMaster is as
|
|
97
|
+
Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
|
|
95
98
|
List of currently supported connections:
|
|
96
99
|
|
|
97
100
|
* Apache Hive
|
|
98
101
|
* Clickhouse
|
|
99
|
-
*
|
|
100
|
-
* Oracle
|
|
102
|
+
* Iceberg (REST Catalog + S3)
|
|
101
103
|
* MSSQL
|
|
102
104
|
* MySQL
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
+
* Oracle
|
|
106
|
+
* Postgres
|
|
105
107
|
* FTP
|
|
106
108
|
* FTPS
|
|
107
|
-
*
|
|
109
|
+
* HDFS
|
|
110
|
+
* S3
|
|
108
111
|
* Samba
|
|
112
|
+
* SFTP
|
|
109
113
|
* WebDAV
|
|
110
114
|
|
|
111
115
|
Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
|
|
112
116
|
|
|
113
|
-
**Note**: service is under active development, and is not ready to use.
|
|
114
|
-
|
|
115
117
|
Goals
|
|
116
118
|
-----
|
|
117
119
|
|
|
118
|
-
* Make
|
|
120
|
+
* Make transferring data between databases and file systems as simple as possible
|
|
119
121
|
* Provide a lot of builtin connectors to transfer data in heterogeneous environment
|
|
120
122
|
* RBAC and multitenancy support
|
|
121
123
|
|
|
122
124
|
Non-goals
|
|
123
125
|
---------
|
|
124
126
|
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
+
* No streaming support for now, only batch operations
|
|
128
|
+
* This is not a change data capture (CDC) solution
|
|
127
129
|
|
|
128
130
|
.. documentation
|
|
129
131
|
|
|
@@ -131,4 +133,3 @@ Documentation
|
|
|
131
133
|
-------------
|
|
132
134
|
|
|
133
135
|
See https://syncmaster.readthedocs.io
|
|
134
|
-
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
.. _readme:
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
|Logo|
|
|
4
|
+
|
|
5
|
+
.. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/d14bf1b1272f15d42f441d6e33264bd2f95216c9/docs/_static/logo_wide.svg
|
|
6
|
+
:alt: Data.SyncMaster logo
|
|
7
|
+
:target: https://github.com/MobileTeleSystems/data-syncmaster
|
|
5
8
|
|
|
6
9
|
|Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
|
|
7
10
|
|Build Status| |Coverage| |pre-commit.ci|
|
|
@@ -26,43 +29,41 @@ Data.SyncMaster
|
|
|
26
29
|
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
27
30
|
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
28
31
|
|
|
29
|
-
|
|
30
32
|
What is Data.SyncMaster?
|
|
31
33
|
------------------------
|
|
32
34
|
|
|
33
|
-
Data.SyncMaster is as
|
|
35
|
+
Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
|
|
34
36
|
List of currently supported connections:
|
|
35
37
|
|
|
36
38
|
* Apache Hive
|
|
37
39
|
* Clickhouse
|
|
38
|
-
*
|
|
39
|
-
* Oracle
|
|
40
|
+
* Iceberg (REST Catalog + S3)
|
|
40
41
|
* MSSQL
|
|
41
42
|
* MySQL
|
|
42
|
-
*
|
|
43
|
-
*
|
|
43
|
+
* Oracle
|
|
44
|
+
* Postgres
|
|
44
45
|
* FTP
|
|
45
46
|
* FTPS
|
|
46
|
-
*
|
|
47
|
+
* HDFS
|
|
48
|
+
* S3
|
|
47
49
|
* Samba
|
|
50
|
+
* SFTP
|
|
48
51
|
* WebDAV
|
|
49
52
|
|
|
50
53
|
Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
|
|
51
54
|
|
|
52
|
-
**Note**: service is under active development, and is not ready to use.
|
|
53
|
-
|
|
54
55
|
Goals
|
|
55
56
|
-----
|
|
56
57
|
|
|
57
|
-
* Make
|
|
58
|
+
* Make transferring data between databases and file systems as simple as possible
|
|
58
59
|
* Provide a lot of builtin connectors to transfer data in heterogeneous environment
|
|
59
60
|
* RBAC and multitenancy support
|
|
60
61
|
|
|
61
62
|
Non-goals
|
|
62
63
|
---------
|
|
63
64
|
|
|
64
|
-
*
|
|
65
|
-
*
|
|
65
|
+
* No streaming support for now, only batch operations
|
|
66
|
+
* This is not a change data capture (CDC) solution
|
|
66
67
|
|
|
67
68
|
.. documentation
|
|
68
69
|
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: data-syncmaster
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Syncmaster REST API + scheduler + Worker
|
|
5
|
+
Author-email: MWS Data Bridge <onetools@mts.ru>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
|
|
8
|
+
Project-URL: Documentation, https://syncmaster.readthedocs.io/
|
|
9
|
+
Project-URL: Source, https://github.com/MobileTeleSystems/syncmaster
|
|
10
|
+
Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
|
|
11
|
+
Project-URL: Tracker, https://github.com/MobileTeleSystems/syncmaster/issues
|
|
12
|
+
Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Framework :: Pydantic
|
|
15
|
+
Classifier: Framework :: Pydantic :: 2
|
|
16
|
+
Classifier: Framework :: FastAPI
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.11
|
|
27
|
+
Description-Content-Type: text/x-rst
|
|
28
|
+
License-File: LICENSE.txt
|
|
29
|
+
Requires-Dist: pydantic~=2.12.5
|
|
30
|
+
Requires-Dist: pydantic-settings~=2.12.0
|
|
31
|
+
Requires-Dist: pydantic-settings-logging~=0.1.1
|
|
32
|
+
Requires-Dist: sqlalchemy~=2.0.44
|
|
33
|
+
Requires-Dist: sqlalchemy-utils~=0.42.0
|
|
34
|
+
Requires-Dist: pyyaml~=6.0.3
|
|
35
|
+
Requires-Dist: coloredlogs~=15.0.1
|
|
36
|
+
Requires-Dist: python-json-logger~=4.0.0
|
|
37
|
+
Requires-Dist: asgi-correlation-id~=4.3.4
|
|
38
|
+
Requires-Dist: uuid6~=2025.0.1
|
|
39
|
+
Requires-Dist: celery~=5.6.0
|
|
40
|
+
Requires-Dist: psycopg2-binary~=2.9.11
|
|
41
|
+
Provides-Extra: server
|
|
42
|
+
Requires-Dist: fastapi~=0.128.0; extra == "server"
|
|
43
|
+
Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
|
|
44
|
+
Requires-Dist: python-multipart~=0.0.21; extra == "server"
|
|
45
|
+
Requires-Dist: uvicorn~=0.40.0; extra == "server"
|
|
46
|
+
Requires-Dist: alembic~=1.18.1; extra == "server"
|
|
47
|
+
Requires-Dist: pyjwt~=2.10.1; extra == "server"
|
|
48
|
+
Requires-Dist: asyncpg~=0.31; extra == "server"
|
|
49
|
+
Requires-Dist: python-keycloak~=7.0.1; extra == "server"
|
|
50
|
+
Requires-Dist: itsdangerous~=2.2.0; extra == "server"
|
|
51
|
+
Provides-Extra: scheduler
|
|
52
|
+
Requires-Dist: asyncpg~=0.31; extra == "scheduler"
|
|
53
|
+
Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
|
|
54
|
+
Provides-Extra: worker
|
|
55
|
+
Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
|
|
56
|
+
Requires-Dist: pyspark<4.0.0; extra == "worker"
|
|
57
|
+
Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
|
|
58
|
+
Requires-Dist: jinja2~=3.1.6; extra == "worker"
|
|
59
|
+
Provides-Extra: kerberos
|
|
60
|
+
Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
|
|
61
|
+
Dynamic: license-file
|
|
62
|
+
|
|
63
|
+
.. _readme:
|
|
64
|
+
|
|
65
|
+
|Logo|
|
|
66
|
+
|
|
67
|
+
.. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/d14bf1b1272f15d42f441d6e33264bd2f95216c9/docs/_static/logo_wide.svg
|
|
68
|
+
:alt: Data.SyncMaster logo
|
|
69
|
+
:target: https://github.com/MobileTeleSystems/data-syncmaster
|
|
70
|
+
|
|
71
|
+
|Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
|
|
72
|
+
|Build Status| |Coverage| |pre-commit.ci|
|
|
73
|
+
|
|
74
|
+
.. |Repo Status| image:: https://www.repostatus.org/badges/latest/wip.svg
|
|
75
|
+
:target: https://www.repostatus.org/#wip
|
|
76
|
+
.. |Docker image| image:: https://img.shields.io/docker/v/mtsrus/syncmaster-server?sort=semver&label=docker
|
|
77
|
+
:target: https://hub.docker.com/r/mtsrus/syncmaster-server
|
|
78
|
+
.. |PyPI| image:: https://img.shields.io/pypi/v/data-syncmaster
|
|
79
|
+
:target: https://pypi.org/project/data-syncmaster/
|
|
80
|
+
.. |PyPI License| image:: https://img.shields.io/pypi/l/data-syncmaster.svg
|
|
81
|
+
:target: https://github.com/MobileTeleSystems/syncmaster/blob/develop/LICENSE.txt
|
|
82
|
+
.. |PyPI Python Version| image:: https://img.shields.io/pypi/pyversions/data-syncmaster.svg
|
|
83
|
+
:target: https://badge.fury.io/py/data-syncmaster
|
|
84
|
+
.. |Documentation| image:: https://readthedocs.org/projects/syncmaster/badge/?version=stable
|
|
85
|
+
:target: https://syncmaster.readthedocs.io
|
|
86
|
+
.. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Run%20All%20Tests/badge.svg
|
|
87
|
+
:target: https://github.com/MobileTeleSystems/syncmaster/actions
|
|
88
|
+
.. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
|
|
89
|
+
MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
|
|
90
|
+
:target: https://github.com/MobileTeleSystems/syncmaster/actions
|
|
91
|
+
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
92
|
+
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
93
|
+
|
|
94
|
+
What is Data.SyncMaster?
|
|
95
|
+
------------------------
|
|
96
|
+
|
|
97
|
+
Data.SyncMaster is as no-code ETL tool for transferring data between databases and file systems.
|
|
98
|
+
List of currently supported connections:
|
|
99
|
+
|
|
100
|
+
* Apache Hive
|
|
101
|
+
* Clickhouse
|
|
102
|
+
* Iceberg (REST Catalog + S3)
|
|
103
|
+
* MSSQL
|
|
104
|
+
* MySQL
|
|
105
|
+
* Oracle
|
|
106
|
+
* Postgres
|
|
107
|
+
* FTP
|
|
108
|
+
* FTPS
|
|
109
|
+
* HDFS
|
|
110
|
+
* S3
|
|
111
|
+
* Samba
|
|
112
|
+
* SFTP
|
|
113
|
+
* WebDAV
|
|
114
|
+
|
|
115
|
+
Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
|
|
116
|
+
|
|
117
|
+
Goals
|
|
118
|
+
-----
|
|
119
|
+
|
|
120
|
+
* Make transferring data between databases and file systems as simple as possible
|
|
121
|
+
* Provide a lot of builtin connectors to transfer data in heterogeneous environment
|
|
122
|
+
* RBAC and multitenancy support
|
|
123
|
+
|
|
124
|
+
Non-goals
|
|
125
|
+
---------
|
|
126
|
+
|
|
127
|
+
* No streaming support for now, only batch operations
|
|
128
|
+
* This is not a change data capture (CDC) solution
|
|
129
|
+
|
|
130
|
+
.. documentation
|
|
131
|
+
|
|
132
|
+
Documentation
|
|
133
|
+
-------------
|
|
134
|
+
|
|
135
|
+
See https://syncmaster.readthedocs.io
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
LICENSE.txt
|
|
2
|
+
README.rst
|
|
3
|
+
pyproject.toml
|
|
4
|
+
data_syncmaster.egg-info/PKG-INFO
|
|
5
|
+
data_syncmaster.egg-info/SOURCES.txt
|
|
6
|
+
data_syncmaster.egg-info/dependency_links.txt
|
|
7
|
+
data_syncmaster.egg-info/requires.txt
|
|
8
|
+
data_syncmaster.egg-info/top_level.txt
|
|
9
|
+
syncmaster/VERSION
|
|
10
|
+
syncmaster/__init__.py
|
|
11
|
+
syncmaster/db/__init__.py
|
|
12
|
+
syncmaster/db/factory.py
|
|
13
|
+
syncmaster/db/utils.py
|
|
14
|
+
syncmaster/db/migrations/__init__.py
|
|
15
|
+
syncmaster/db/migrations/__main__.py
|
|
16
|
+
syncmaster/db/migrations/alembic.ini
|
|
17
|
+
syncmaster/db/migrations/env.py
|
|
18
|
+
syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py
|
|
19
|
+
syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py
|
|
20
|
+
syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py
|
|
21
|
+
syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py
|
|
22
|
+
syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py
|
|
23
|
+
syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py
|
|
24
|
+
syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py
|
|
25
|
+
syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py
|
|
26
|
+
syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py
|
|
27
|
+
syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py
|
|
28
|
+
syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py
|
|
29
|
+
syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py
|
|
30
|
+
syncmaster/db/migrations/versions/__init__.py
|
|
31
|
+
syncmaster/db/mixins/__init__.py
|
|
32
|
+
syncmaster/db/mixins/resource.py
|
|
33
|
+
syncmaster/db/mixins/timestamp.py
|
|
34
|
+
syncmaster/db/models/__init__.py
|
|
35
|
+
syncmaster/db/models/apscheduler_job.py
|
|
36
|
+
syncmaster/db/models/auth_data.py
|
|
37
|
+
syncmaster/db/models/base.py
|
|
38
|
+
syncmaster/db/models/connection.py
|
|
39
|
+
syncmaster/db/models/group.py
|
|
40
|
+
syncmaster/db/models/queue.py
|
|
41
|
+
syncmaster/db/models/run.py
|
|
42
|
+
syncmaster/db/models/transfer.py
|
|
43
|
+
syncmaster/db/models/user.py
|
|
44
|
+
syncmaster/db/repositories/__init__.py
|
|
45
|
+
syncmaster/db/repositories/base.py
|
|
46
|
+
syncmaster/db/repositories/connection.py
|
|
47
|
+
syncmaster/db/repositories/credentials_repository.py
|
|
48
|
+
syncmaster/db/repositories/group.py
|
|
49
|
+
syncmaster/db/repositories/queue.py
|
|
50
|
+
syncmaster/db/repositories/repository_with_owner.py
|
|
51
|
+
syncmaster/db/repositories/run.py
|
|
52
|
+
syncmaster/db/repositories/search.py
|
|
53
|
+
syncmaster/db/repositories/transfer.py
|
|
54
|
+
syncmaster/db/repositories/user.py
|
|
55
|
+
syncmaster/db/repositories/utils.py
|
|
56
|
+
syncmaster/dto/__init__.py
|
|
57
|
+
syncmaster/dto/connections.py
|
|
58
|
+
syncmaster/dto/runs.py
|
|
59
|
+
syncmaster/dto/transfers.py
|
|
60
|
+
syncmaster/dto/transfers_resources.py
|
|
61
|
+
syncmaster/dto/transfers_strategy.py
|
|
62
|
+
syncmaster/errors/__init__.py
|
|
63
|
+
syncmaster/errors/base.py
|
|
64
|
+
syncmaster/errors/registration.py
|
|
65
|
+
syncmaster/errors/schemas/__init__.py
|
|
66
|
+
syncmaster/errors/schemas/bad_request.py
|
|
67
|
+
syncmaster/errors/schemas/invalid_request.py
|
|
68
|
+
syncmaster/errors/schemas/not_authorized.py
|
|
69
|
+
syncmaster/exceptions/__init__.py
|
|
70
|
+
syncmaster/exceptions/auth.py
|
|
71
|
+
syncmaster/exceptions/base.py
|
|
72
|
+
syncmaster/exceptions/connection.py
|
|
73
|
+
syncmaster/exceptions/credentials.py
|
|
74
|
+
syncmaster/exceptions/group.py
|
|
75
|
+
syncmaster/exceptions/queue.py
|
|
76
|
+
syncmaster/exceptions/redirect.py
|
|
77
|
+
syncmaster/exceptions/run.py
|
|
78
|
+
syncmaster/exceptions/transfer.py
|
|
79
|
+
syncmaster/exceptions/user.py
|
|
80
|
+
syncmaster/scheduler/__init__.py
|
|
81
|
+
syncmaster/scheduler/__main__.py
|
|
82
|
+
syncmaster/scheduler/celery.py
|
|
83
|
+
syncmaster/scheduler/transfer_fetcher.py
|
|
84
|
+
syncmaster/scheduler/transfer_job_manager.py
|
|
85
|
+
syncmaster/scheduler/utils.py
|
|
86
|
+
syncmaster/scheduler/settings/__init__.py
|
|
87
|
+
syncmaster/schemas/__init__.py
|
|
88
|
+
syncmaster/schemas/ping.py
|
|
89
|
+
syncmaster/schemas/v1/__init__.py
|
|
90
|
+
syncmaster/schemas/v1/connection_types.py
|
|
91
|
+
syncmaster/schemas/v1/file_formats.py
|
|
92
|
+
syncmaster/schemas/v1/groups.py
|
|
93
|
+
syncmaster/schemas/v1/page.py
|
|
94
|
+
syncmaster/schemas/v1/queue.py
|
|
95
|
+
syncmaster/schemas/v1/transformation_types.py
|
|
96
|
+
syncmaster/schemas/v1/types.py
|
|
97
|
+
syncmaster/schemas/v1/users.py
|
|
98
|
+
syncmaster/schemas/v1/auth/__init__.py
|
|
99
|
+
syncmaster/schemas/v1/auth/basic.py
|
|
100
|
+
syncmaster/schemas/v1/auth/mixins.py
|
|
101
|
+
syncmaster/schemas/v1/auth/s3.py
|
|
102
|
+
syncmaster/schemas/v1/auth/samba.py
|
|
103
|
+
syncmaster/schemas/v1/auth/token.py
|
|
104
|
+
syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py
|
|
105
|
+
syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py
|
|
106
|
+
syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py
|
|
107
|
+
syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py
|
|
108
|
+
syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py
|
|
109
|
+
syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py
|
|
110
|
+
syncmaster/schemas/v1/connections/__init__.py
|
|
111
|
+
syncmaster/schemas/v1/connections/clickhouse.py
|
|
112
|
+
syncmaster/schemas/v1/connections/connection_base.py
|
|
113
|
+
syncmaster/schemas/v1/connections/ftp.py
|
|
114
|
+
syncmaster/schemas/v1/connections/ftps.py
|
|
115
|
+
syncmaster/schemas/v1/connections/hdfs.py
|
|
116
|
+
syncmaster/schemas/v1/connections/hive.py
|
|
117
|
+
syncmaster/schemas/v1/connections/iceberg.py
|
|
118
|
+
syncmaster/schemas/v1/connections/mssql.py
|
|
119
|
+
syncmaster/schemas/v1/connections/mysql.py
|
|
120
|
+
syncmaster/schemas/v1/connections/oracle.py
|
|
121
|
+
syncmaster/schemas/v1/connections/postgres.py
|
|
122
|
+
syncmaster/schemas/v1/connections/s3.py
|
|
123
|
+
syncmaster/schemas/v1/connections/samba.py
|
|
124
|
+
syncmaster/schemas/v1/connections/sftp.py
|
|
125
|
+
syncmaster/schemas/v1/connections/webdav.py
|
|
126
|
+
syncmaster/schemas/v1/transfers/__init__.py
|
|
127
|
+
syncmaster/schemas/v1/transfers/db.py
|
|
128
|
+
syncmaster/schemas/v1/transfers/file_format.py
|
|
129
|
+
syncmaster/schemas/v1/transfers/resources.py
|
|
130
|
+
syncmaster/schemas/v1/transfers/run.py
|
|
131
|
+
syncmaster/schemas/v1/transfers/strategy.py
|
|
132
|
+
syncmaster/schemas/v1/transfers/file/__init__.py
|
|
133
|
+
syncmaster/schemas/v1/transfers/file/base.py
|
|
134
|
+
syncmaster/schemas/v1/transfers/file/ftp.py
|
|
135
|
+
syncmaster/schemas/v1/transfers/file/ftps.py
|
|
136
|
+
syncmaster/schemas/v1/transfers/file/hdfs.py
|
|
137
|
+
syncmaster/schemas/v1/transfers/file/s3.py
|
|
138
|
+
syncmaster/schemas/v1/transfers/file/samba.py
|
|
139
|
+
syncmaster/schemas/v1/transfers/file/sftp.py
|
|
140
|
+
syncmaster/schemas/v1/transfers/file/webdav.py
|
|
141
|
+
syncmaster/schemas/v1/transfers/transformations/__init__.py
|
|
142
|
+
syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py
|
|
143
|
+
syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py
|
|
144
|
+
syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py
|
|
145
|
+
syncmaster/server/__init__.py
|
|
146
|
+
syncmaster/server/__main__.py
|
|
147
|
+
syncmaster/server/handler.py
|
|
148
|
+
syncmaster/server/api/__init__.py
|
|
149
|
+
syncmaster/server/api/monitoring.py
|
|
150
|
+
syncmaster/server/api/router.py
|
|
151
|
+
syncmaster/server/api/v1/__init__.py
|
|
152
|
+
syncmaster/server/api/v1/auth.py
|
|
153
|
+
syncmaster/server/api/v1/connections.py
|
|
154
|
+
syncmaster/server/api/v1/groups.py
|
|
155
|
+
syncmaster/server/api/v1/queue.py
|
|
156
|
+
syncmaster/server/api/v1/router.py
|
|
157
|
+
syncmaster/server/api/v1/runs.py
|
|
158
|
+
syncmaster/server/api/v1/transfers.py
|
|
159
|
+
syncmaster/server/api/v1/users.py
|
|
160
|
+
syncmaster/server/dependencies/__init__.py
|
|
161
|
+
syncmaster/server/dependencies/get_access_token.py
|
|
162
|
+
syncmaster/server/dependencies/stub.py
|
|
163
|
+
syncmaster/server/middlewares/__init__.py
|
|
164
|
+
syncmaster/server/middlewares/cors.py
|
|
165
|
+
syncmaster/server/middlewares/openapi.py
|
|
166
|
+
syncmaster/server/middlewares/request_id.py
|
|
167
|
+
syncmaster/server/middlewares/static_files.py
|
|
168
|
+
syncmaster/server/middlewares/monitoring/__init__.py
|
|
169
|
+
syncmaster/server/middlewares/monitoring/metrics.py
|
|
170
|
+
syncmaster/server/providers/__init__.py
|
|
171
|
+
syncmaster/server/providers/auth/__init__.py
|
|
172
|
+
syncmaster/server/providers/auth/base_provider.py
|
|
173
|
+
syncmaster/server/providers/auth/dummy_provider.py
|
|
174
|
+
syncmaster/server/providers/auth/keycloak_provider.py
|
|
175
|
+
syncmaster/server/providers/auth/oauth2_gateway_provider.py
|
|
176
|
+
syncmaster/server/scripts/export_openapi_schema.py
|
|
177
|
+
syncmaster/server/scripts/manage_superusers.py
|
|
178
|
+
syncmaster/server/services/__init__.py
|
|
179
|
+
syncmaster/server/services/get_user.py
|
|
180
|
+
syncmaster/server/services/unit_of_work.py
|
|
181
|
+
syncmaster/server/settings/__init__.py
|
|
182
|
+
syncmaster/server/settings/auth/__init__.py
|
|
183
|
+
syncmaster/server/settings/auth/dummy.py
|
|
184
|
+
syncmaster/server/settings/auth/jwt.py
|
|
185
|
+
syncmaster/server/settings/auth/keycloak.py
|
|
186
|
+
syncmaster/server/settings/auth/oauth2_gateway.py
|
|
187
|
+
syncmaster/server/settings/server/__init__.py
|
|
188
|
+
syncmaster/server/settings/server/cors.py
|
|
189
|
+
syncmaster/server/settings/server/monitoring.py
|
|
190
|
+
syncmaster/server/settings/server/openapi.py
|
|
191
|
+
syncmaster/server/settings/server/request_id.py
|
|
192
|
+
syncmaster/server/settings/server/static_files.py
|
|
193
|
+
syncmaster/server/utils/__init__.py
|
|
194
|
+
syncmaster/server/utils/jwt.py
|
|
195
|
+
syncmaster/server/utils/slug.py
|
|
196
|
+
syncmaster/settings/__init__.py
|
|
197
|
+
syncmaster/settings/base.py
|
|
198
|
+
syncmaster/settings/broker.py
|
|
199
|
+
syncmaster/settings/credentials.py
|
|
200
|
+
syncmaster/settings/database.py
|
|
201
|
+
syncmaster/settings/logging.py
|
|
202
|
+
syncmaster/worker/__init__.py
|
|
203
|
+
syncmaster/worker/base.py
|
|
204
|
+
syncmaster/worker/celery.py
|
|
205
|
+
syncmaster/worker/controller.py
|
|
206
|
+
syncmaster/worker/ivy2.py
|
|
207
|
+
syncmaster/worker/spark.py
|
|
208
|
+
syncmaster/worker/transfer.py
|
|
209
|
+
syncmaster/worker/handlers/__init__.py
|
|
210
|
+
syncmaster/worker/handlers/base.py
|
|
211
|
+
syncmaster/worker/handlers/db/__init__.py
|
|
212
|
+
syncmaster/worker/handlers/db/base.py
|
|
213
|
+
syncmaster/worker/handlers/db/clickhouse.py
|
|
214
|
+
syncmaster/worker/handlers/db/hive.py
|
|
215
|
+
syncmaster/worker/handlers/db/iceberg.py
|
|
216
|
+
syncmaster/worker/handlers/db/mssql.py
|
|
217
|
+
syncmaster/worker/handlers/db/mysql.py
|
|
218
|
+
syncmaster/worker/handlers/db/oracle.py
|
|
219
|
+
syncmaster/worker/handlers/db/postgres.py
|
|
220
|
+
syncmaster/worker/handlers/file/__init__.py
|
|
221
|
+
syncmaster/worker/handlers/file/base.py
|
|
222
|
+
syncmaster/worker/handlers/file/ftp.py
|
|
223
|
+
syncmaster/worker/handlers/file/ftps.py
|
|
224
|
+
syncmaster/worker/handlers/file/hdfs.py
|
|
225
|
+
syncmaster/worker/handlers/file/local_df.py
|
|
226
|
+
syncmaster/worker/handlers/file/remote_df.py
|
|
227
|
+
syncmaster/worker/handlers/file/s3.py
|
|
228
|
+
syncmaster/worker/handlers/file/samba.py
|
|
229
|
+
syncmaster/worker/handlers/file/sftp.py
|
|
230
|
+
syncmaster/worker/handlers/file/webdav.py
|
|
231
|
+
syncmaster/worker/settings/__init__.py
|
|
232
|
+
syncmaster/worker/settings/hwm_store.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
pydantic~=2.12.5
|
|
2
|
+
pydantic-settings~=2.12.0
|
|
3
|
+
pydantic-settings-logging~=0.1.1
|
|
4
|
+
sqlalchemy~=2.0.44
|
|
5
|
+
sqlalchemy-utils~=0.42.0
|
|
6
|
+
pyyaml~=6.0.3
|
|
7
|
+
coloredlogs~=15.0.1
|
|
8
|
+
python-json-logger~=4.0.0
|
|
9
|
+
asgi-correlation-id~=4.3.4
|
|
10
|
+
uuid6~=2025.0.1
|
|
11
|
+
celery~=5.6.0
|
|
12
|
+
psycopg2-binary~=2.9.11
|
|
13
|
+
|
|
14
|
+
[kerberos]
|
|
15
|
+
onetl[kerberos]~=0.15.0
|
|
16
|
+
|
|
17
|
+
[scheduler]
|
|
18
|
+
asyncpg~=0.31
|
|
19
|
+
apscheduler~=3.11.1
|
|
20
|
+
|
|
21
|
+
[server]
|
|
22
|
+
fastapi~=0.128.0
|
|
23
|
+
starlette-exporter~=0.23.0
|
|
24
|
+
python-multipart~=0.0.21
|
|
25
|
+
uvicorn~=0.40.0
|
|
26
|
+
alembic~=1.18.1
|
|
27
|
+
pyjwt~=2.10.1
|
|
28
|
+
asyncpg~=0.31
|
|
29
|
+
python-keycloak~=7.0.1
|
|
30
|
+
itsdangerous~=2.2.0
|
|
31
|
+
|
|
32
|
+
[worker]
|
|
33
|
+
onetl[files]~=0.15.0
|
|
34
|
+
pyspark<4.0.0
|
|
35
|
+
horizon-hwm-store~=1.1.4
|
|
36
|
+
jinja2~=3.1.6
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
syncmaster
|