data-syncmaster 0.2.5__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/LICENSE.txt +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/PKG-INFO +36 -34
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/README.rst +14 -13
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/pyproject.toml +46 -67
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/__init__.py +2 -2
- data_syncmaster-0.3.0/syncmaster/db/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/factory.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/__main__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/env.py +14 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py +1 -1
- data_syncmaster-0.3.0/syncmaster/db/migrations/versions/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/mixins/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/mixins/resource.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/mixins/timestamp.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/apscheduler_job.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/auth_data.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/connection.py +2 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/group.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/queue.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/run.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/transfer.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/models/user.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/connection.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/credentials_repository.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/group.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/queue.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/repository_with_owner.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/run.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/search.py +1 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/transfer.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/user.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/repositories/utils.py +1 -8
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/utils.py +1 -1
- data_syncmaster-0.3.0/syncmaster/dto/__init__.py +2 -0
- data_syncmaster-0.3.0/syncmaster/dto/connections.py +231 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/runs.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/transfers.py +38 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/transfers_resources.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/dto/transfers_strategy.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/registration.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/bad_request.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/invalid_request.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/errors/schemas/not_authorized.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/auth.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/connection.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/credentials.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/group.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/queue.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/redirect.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/run.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/transfer.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/exceptions/user.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/__main__.py +3 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/celery.py +1 -1
- data_syncmaster-0.3.0/syncmaster/scheduler/settings/__init__.py +76 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/transfer_fetcher.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/transfer_job_manager.py +3 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/scheduler/utils.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/ping.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/__init__.py +2 -12
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/auth/__init__.py +1 -7
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/basic.py +23 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py +31 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py +22 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py +32 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py +31 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py +25 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py +34 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/mixins.py +12 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/s3.py +23 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/auth/samba.py +24 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/auth/token.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connection_types.py +26 -7
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection.py → data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/__init__.py +39 -31
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/clickhouse.py +14 -19
- data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/connection_base.py +46 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/ftp.py +15 -17
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/ftps.py +12 -17
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/hdfs.py +14 -15
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/hive.py +14 -15
- data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/iceberg.py +105 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/mssql.py +15 -19
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/mysql.py +16 -20
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/oracle.py +15 -20
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/postgres.py +15 -19
- data_syncmaster-0.3.0/syncmaster/schemas/v1/connections/s3.py +65 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/samba.py +7 -18
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/sftp.py +14 -16
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/connections/webdav.py +15 -18
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/file_formats.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/groups.py +5 -10
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/page.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/queue.py +8 -9
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/__init__.py +94 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/db.py +62 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/__init__.py +55 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/file/base.py +9 -28
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/ftp.py +17 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/ftps.py +17 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/hdfs.py +17 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/s3.py +17 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/samba.py +17 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/sftp.py +17 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/file/webdav.py +17 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/file_format.py +1 -1
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/resources.py +17 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/run.py +10 -11
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/strategy.py +23 -0
- data_syncmaster-0.3.0/syncmaster/schemas/v1/transfers/transformations/__init__.py +25 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py +2 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py +2 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py +2 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/transformation_types.py +1 -1
- data_syncmaster-0.3.0/syncmaster/schemas/v1/types.py +8 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/schemas/v1/users.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/__init__.py +5 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/__main__.py +1 -1
- data_syncmaster-0.3.0/syncmaster/server/api/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/monitoring.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/router.py +1 -1
- data_syncmaster-0.3.0/syncmaster/server/api/v1/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/auth.py +2 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/connections.py +18 -18
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/groups.py +9 -9
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/queue.py +7 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/router.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/runs.py +7 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/transfers.py +17 -17
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/api/v1/users.py +5 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/dependencies/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/dependencies/get_access_token.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/handler.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/__init__.py +1 -7
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/cors.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/monitoring/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/monitoring/metrics.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/openapi.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/request_id.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/middlewares/static_files.py +1 -1
- data_syncmaster-0.3.0/syncmaster/server/providers/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/__init__.py +5 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/base_provider.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/dummy_provider.py +5 -3
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/providers/auth/keycloak_provider.py +23 -6
- data_syncmaster-0.3.0/syncmaster/server/providers/auth/oauth2_gateway_provider.py +116 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/scripts/export_openapi_schema.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/scripts/manage_superusers.py +32 -8
- data_syncmaster-0.3.0/syncmaster/server/services/__init__.py +2 -0
- data_syncmaster-0.3.0/syncmaster/server/services/get_user.py +61 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/services/unit_of_work.py +1 -1
- data_syncmaster-0.3.0/syncmaster/server/settings/__init__.py +70 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/auth/__init__.py +8 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/auth/dummy.py +7 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/auth/jwt.py +7 -4
- data_syncmaster-0.3.0/syncmaster/server/settings/auth/keycloak.py +135 -0
- data_syncmaster-0.3.0/syncmaster/server/settings/auth/oauth2_gateway.py +35 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/__init__.py +15 -10
- data_syncmaster-0.3.0/syncmaster/server/settings/server/cors.py +63 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/monitoring.py +16 -5
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/openapi.py +48 -22
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/request_id.py +8 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/settings/server/static_files.py +12 -6
- data_syncmaster-0.3.0/syncmaster/server/utils/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/utils/jwt.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/utils/slug.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/__init__.py +6 -3
- data_syncmaster-0.3.0/syncmaster/settings/base.py +37 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/broker.py +7 -6
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/credentials.py +5 -4
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/settings/database.py +8 -5
- data_syncmaster-0.3.0/syncmaster/settings/logging.py +288 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/__init__.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/celery.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/controller.py +39 -4
- data_syncmaster-0.3.0/syncmaster/worker/handlers/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/base.py +1 -1
- data_syncmaster-0.3.0/syncmaster/worker/handlers/db/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/clickhouse.py +8 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/hive.py +8 -1
- data_syncmaster-0.3.0/syncmaster/worker/handlers/db/iceberg.py +134 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/mssql.py +11 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/mysql.py +11 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/oracle.py +11 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/db/postgres.py +11 -1
- data_syncmaster-0.3.0/syncmaster/worker/handlers/file/__init__.py +2 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/base.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/ftp.py +13 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/ftps.py +13 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/hdfs.py +13 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/local_df.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/remote_df.py +1 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/s3.py +9 -1
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/samba.py +13 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/sftp.py +13 -2
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/handlers/file/webdav.py +12 -2
- data_syncmaster-0.3.0/syncmaster/worker/ivy2.py +59 -0
- data_syncmaster-0.3.0/syncmaster/worker/settings/__init__.py +95 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/settings/hwm_store.py +11 -9
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/spark.py +33 -49
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/worker/transfer.py +3 -3
- data_syncmaster-0.2.5/syncmaster/db/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/db/migrations/versions/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/dto/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/dto/connections.py +0 -144
- data_syncmaster-0.2.5/syncmaster/scheduler/settings/__init__.py +0 -78
- data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/basic.py +0 -26
- data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/s3.py +0 -26
- data_syncmaster-0.2.5/syncmaster/schemas/v1/auth/samba.py +0 -28
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/connection_base.py +0 -30
- data_syncmaster-0.2.5/syncmaster/schemas/v1/connections/s3.py +0 -73
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfer_types.py +0 -6
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/__init__.py +0 -248
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/db.py +0 -42
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftp.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/ftps.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/s3.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/samba.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/sftp.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/file/webdav.py +0 -27
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/resources.py +0 -14
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/strategy.py +0 -14
- data_syncmaster-0.2.5/syncmaster/schemas/v1/transfers/transformations/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/schemas/v1/types.py +0 -7
- data_syncmaster-0.2.5/syncmaster/server/api/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/api/v1/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/middlewares/session.py +0 -17
- data_syncmaster-0.2.5/syncmaster/server/providers/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/services/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/server/services/get_user.py +0 -41
- data_syncmaster-0.2.5/syncmaster/server/settings/__init__.py +0 -61
- data_syncmaster-0.2.5/syncmaster/server/settings/auth/keycloak.py +0 -22
- data_syncmaster-0.2.5/syncmaster/server/settings/server/cors.py +0 -70
- data_syncmaster-0.2.5/syncmaster/server/settings/server/session.py +0 -75
- data_syncmaster-0.2.5/syncmaster/server/utils/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/settings/log/__init__.py +0 -110
- data_syncmaster-0.2.5/syncmaster/settings/log/colored.yml +0 -57
- data_syncmaster-0.2.5/syncmaster/settings/log/json.yml +0 -56
- data_syncmaster-0.2.5/syncmaster/settings/log/plain.yml +0 -57
- data_syncmaster-0.2.5/syncmaster/worker/handlers/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/worker/handlers/db/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/worker/handlers/file/__init__.py +0 -2
- data_syncmaster-0.2.5/syncmaster/worker/settings/__init__.py +0 -79
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/README +0 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/alembic.ini +0 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/db/migrations/script.py.mako +0 -0
- {data_syncmaster-0.2.5 → data_syncmaster-0.3.0}/syncmaster/server/dependencies/stub.py +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: data-syncmaster
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Syncmaster REST API + Worker
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE.txt
|
|
7
7
|
Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
|
|
8
|
-
Author:
|
|
8
|
+
Author: MWS Data Bridge
|
|
9
9
|
Author-email: onetools@mts.ru
|
|
10
10
|
Requires-Python: >=3.11,<4.0
|
|
11
11
|
Classifier: Development Status :: 3 - Alpha
|
|
@@ -27,31 +27,32 @@ Classifier: Typing :: Typed
|
|
|
27
27
|
Provides-Extra: scheduler
|
|
28
28
|
Provides-Extra: server
|
|
29
29
|
Provides-Extra: worker
|
|
30
|
-
Requires-Dist: alembic (>=1.
|
|
31
|
-
Requires-Dist: apscheduler (>=3.11.
|
|
32
|
-
Requires-Dist: asgi-correlation-id (>=4.3.4,<5.0.0)
|
|
33
|
-
Requires-Dist: asyncpg (>=0.30
|
|
34
|
-
Requires-Dist: celery (>=5.5.
|
|
35
|
-
Requires-Dist: coloredlogs
|
|
36
|
-
Requires-Dist: fastapi (>=0.
|
|
37
|
-
Requires-Dist: horizon-hwm-store (>=1.1.
|
|
30
|
+
Requires-Dist: alembic (>=1.17.1,<2.0.0) ; extra == "server"
|
|
31
|
+
Requires-Dist: apscheduler (>=3.11.1,<4.0.0) ; extra == "scheduler"
|
|
32
|
+
Requires-Dist: asgi-correlation-id (>=4.3.4,<5.0.0)
|
|
33
|
+
Requires-Dist: asyncpg (>=0.30,<0.32) ; extra == "server" or extra == "scheduler"
|
|
34
|
+
Requires-Dist: celery (>=5.5.3,<6.0.0)
|
|
35
|
+
Requires-Dist: coloredlogs (>=15.0.1,<16.0.0)
|
|
36
|
+
Requires-Dist: fastapi (>=0.124.0,<0.125.0) ; extra == "server"
|
|
37
|
+
Requires-Dist: horizon-hwm-store (>=1.1.4,<2.0.0) ; extra == "worker"
|
|
38
38
|
Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
|
|
39
39
|
Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
|
|
40
|
-
Requires-Dist: onetl[all] (>=0.
|
|
41
|
-
Requires-Dist: psycopg2-binary (>=2.9.
|
|
42
|
-
Requires-Dist: pydantic (>=2.
|
|
43
|
-
Requires-Dist: pydantic-settings (>=2.
|
|
40
|
+
Requires-Dist: onetl[all] (>=0.15.0,<0.16.0) ; extra == "worker"
|
|
41
|
+
Requires-Dist: psycopg2-binary (>=2.9.11,<3.0.0) ; extra == "server" or extra == "worker"
|
|
42
|
+
Requires-Dist: pydantic (>=2.12.4,<3.0.0)
|
|
43
|
+
Requires-Dist: pydantic-settings (>=2.12.0,<3.0.0)
|
|
44
|
+
Requires-Dist: pydantic-settings-logging (>=0.1.1,<0.2.0)
|
|
44
45
|
Requires-Dist: pyjwt (>=2.10.1,<3.0.0) ; extra == "server"
|
|
45
46
|
Requires-Dist: pyspark (<4.0.0) ; extra == "worker"
|
|
46
|
-
Requires-Dist: python-json-logger
|
|
47
|
-
Requires-Dist: python-keycloak (>=5.
|
|
47
|
+
Requires-Dist: python-json-logger (>=4.0.0,<5.0.0)
|
|
48
|
+
Requires-Dist: python-keycloak (>=5.8.1,<6.0.0) ; extra == "server"
|
|
48
49
|
Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
|
|
49
|
-
Requires-Dist: pyyaml ; extra == "server"
|
|
50
|
-
Requires-Dist: sqlalchemy (>=2.0.
|
|
51
|
-
Requires-Dist: sqlalchemy-utils (>=0.
|
|
50
|
+
Requires-Dist: pyyaml (>=6.0.3,<7.0.0) ; extra == "server"
|
|
51
|
+
Requires-Dist: sqlalchemy (>=2.0.44,<3.0.0)
|
|
52
|
+
Requires-Dist: sqlalchemy-utils (>=0.42.0,<0.43.0)
|
|
52
53
|
Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
|
|
53
|
-
Requires-Dist: uuid6 (>=2025.0.
|
|
54
|
-
Requires-Dist: uvicorn (>=0.
|
|
54
|
+
Requires-Dist: uuid6 (>=2025.0.1,<2026.0.0)
|
|
55
|
+
Requires-Dist: uvicorn (>=0.38.0,<0.39.0) ; extra == "server"
|
|
55
56
|
Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
|
|
56
57
|
Project-URL: Documentation, https://syncmaster.readthedocs.io
|
|
57
58
|
Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
|
|
@@ -61,8 +62,11 @@ Description-Content-Type: text/x-rst
|
|
|
61
62
|
|
|
62
63
|
.. _readme:
|
|
63
64
|
|
|
64
|
-
|
|
65
|
-
|
|
65
|
+
|Logo|
|
|
66
|
+
|
|
67
|
+
.. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/e3afdfe835fc6ee943f1105cfe3d624c01123bd6/docs/_static/logo_wide_red_text.svg
|
|
68
|
+
:alt: Data.SyncMaster logo
|
|
69
|
+
:target: https://github.com/MobileTeleSystems/data-syncmaster
|
|
66
70
|
|
|
67
71
|
|Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
|
|
68
72
|
|Build Status| |Coverage| |pre-commit.ci|
|
|
@@ -87,31 +91,29 @@ Data.SyncMaster
|
|
|
87
91
|
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
88
92
|
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
89
93
|
|
|
90
|
-
|
|
91
94
|
What is Data.SyncMaster?
|
|
92
95
|
------------------------
|
|
93
96
|
|
|
94
|
-
Data.SyncMaster is as
|
|
97
|
+
Data.SyncMaster is as no-code ETL tool for transfering data between databases and file systems.
|
|
95
98
|
List of currently supported connections:
|
|
96
99
|
|
|
97
100
|
* Apache Hive
|
|
98
101
|
* Clickhouse
|
|
99
|
-
*
|
|
100
|
-
* Oracle
|
|
102
|
+
* Iceberg (REST Catalog + S3)
|
|
101
103
|
* MSSQL
|
|
102
104
|
* MySQL
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
+
* Oracle
|
|
106
|
+
* Postgres
|
|
105
107
|
* FTP
|
|
106
108
|
* FTPS
|
|
107
|
-
*
|
|
109
|
+
* HDFS
|
|
110
|
+
* S3
|
|
108
111
|
* Samba
|
|
112
|
+
* SFTP
|
|
109
113
|
* WebDAV
|
|
110
114
|
|
|
111
115
|
Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
|
|
112
116
|
|
|
113
|
-
**Note**: service is under active development, and is not ready to use.
|
|
114
|
-
|
|
115
117
|
Goals
|
|
116
118
|
-----
|
|
117
119
|
|
|
@@ -122,8 +124,8 @@ Goals
|
|
|
122
124
|
Non-goals
|
|
123
125
|
---------
|
|
124
126
|
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
+
* No streaming support for now, only batch operations
|
|
128
|
+
* This is not a change data capture (CDC) solution
|
|
127
129
|
|
|
128
130
|
.. documentation
|
|
129
131
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
.. _readme:
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
|Logo|
|
|
4
|
+
|
|
5
|
+
.. |Logo| image:: https://raw.githubusercontent.com/MobileTeleSystems/syncmaster/e3afdfe835fc6ee943f1105cfe3d624c01123bd6/docs/_static/logo_wide_red_text.svg
|
|
6
|
+
:alt: Data.SyncMaster logo
|
|
7
|
+
:target: https://github.com/MobileTeleSystems/data-syncmaster
|
|
5
8
|
|
|
6
9
|
|Repo Status| |Docker image| |PyPI| |PyPI License| |PyPI Python Version| |Documentation|
|
|
7
10
|
|Build Status| |Coverage| |pre-commit.ci|
|
|
@@ -26,31 +29,29 @@ Data.SyncMaster
|
|
|
26
29
|
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
27
30
|
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
28
31
|
|
|
29
|
-
|
|
30
32
|
What is Data.SyncMaster?
|
|
31
33
|
------------------------
|
|
32
34
|
|
|
33
|
-
Data.SyncMaster is as
|
|
35
|
+
Data.SyncMaster is as no-code ETL tool for transfering data between databases and file systems.
|
|
34
36
|
List of currently supported connections:
|
|
35
37
|
|
|
36
38
|
* Apache Hive
|
|
37
39
|
* Clickhouse
|
|
38
|
-
*
|
|
39
|
-
* Oracle
|
|
40
|
+
* Iceberg (REST Catalog + S3)
|
|
40
41
|
* MSSQL
|
|
41
42
|
* MySQL
|
|
42
|
-
*
|
|
43
|
-
*
|
|
43
|
+
* Oracle
|
|
44
|
+
* Postgres
|
|
44
45
|
* FTP
|
|
45
46
|
* FTPS
|
|
46
|
-
*
|
|
47
|
+
* HDFS
|
|
48
|
+
* S3
|
|
47
49
|
* Samba
|
|
50
|
+
* SFTP
|
|
48
51
|
* WebDAV
|
|
49
52
|
|
|
50
53
|
Based on `onETL <https://onetl.readthedocs.io/>`_ and `Apache Spark <https://spark.apache.org/>`_.
|
|
51
54
|
|
|
52
|
-
**Note**: service is under active development, and is not ready to use.
|
|
53
|
-
|
|
54
55
|
Goals
|
|
55
56
|
-----
|
|
56
57
|
|
|
@@ -61,8 +62,8 @@ Goals
|
|
|
61
62
|
Non-goals
|
|
62
63
|
---------
|
|
63
64
|
|
|
64
|
-
*
|
|
65
|
-
*
|
|
65
|
+
* No streaming support for now, only batch operations
|
|
66
|
+
* This is not a change data capture (CDC) solution
|
|
66
67
|
|
|
67
68
|
.. documentation
|
|
68
69
|
|
|
@@ -4,10 +4,10 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "data-syncmaster"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
license = "Apache-2.0"
|
|
9
9
|
description = "Syncmaster REST API + Worker"
|
|
10
|
-
authors = ["
|
|
10
|
+
authors = ["MWS Data Bridge <onetools@mts.ru>"]
|
|
11
11
|
readme = "README.rst"
|
|
12
12
|
classifiers = [
|
|
13
13
|
"Development Status :: 3 - Alpha",
|
|
@@ -44,118 +44,95 @@ exclude = [
|
|
|
44
44
|
|
|
45
45
|
[tool.poetry.dependencies]
|
|
46
46
|
python = "^3.11"
|
|
47
|
-
pydantic = "^2.
|
|
48
|
-
pydantic-settings =
|
|
49
|
-
|
|
50
|
-
sqlalchemy
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
47
|
+
pydantic = "^2.12.4"
|
|
48
|
+
pydantic-settings = "^2.12.0"
|
|
49
|
+
pydantic-settings-logging = "^0.1.1"
|
|
50
|
+
sqlalchemy = "^2.0.44"
|
|
51
|
+
sqlalchemy-utils = "^0.42.0"
|
|
52
|
+
pyyaml = "^6.0.3"
|
|
53
|
+
coloredlogs = "^15.0.1"
|
|
54
|
+
python-json-logger = "^4.0.0"
|
|
55
|
+
celery = "^5.5.3"
|
|
56
|
+
asgi-correlation-id = "^4.3.4"
|
|
57
|
+
uuid6 = "^2025.0.1"
|
|
58
|
+
fastapi = {version = "^0.124.0", optional = true}
|
|
59
|
+
uvicorn = { version = "^0.38.0", optional = true }
|
|
60
|
+
alembic = { version = "^1.17.1", optional = true }
|
|
55
61
|
pyjwt = { version = "^2.10.1", optional = true }
|
|
56
62
|
jinja2 = { version = "^3.1.6", optional = true }
|
|
57
63
|
python-multipart = { version = "^0.0.20", optional = true }
|
|
58
|
-
|
|
59
|
-
onetl = { version = ">=0.13.5,<0.15.0", extras = ["all"], optional = true }
|
|
64
|
+
onetl = { version = "^0.15.0", extras = ["all"], optional = true }
|
|
60
65
|
pyspark = { version = "<4.0.0", optional = true }
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
coloredlogs = {version = "*", optional = true}
|
|
65
|
-
python-json-logger = {version = "*", optional = true}
|
|
66
|
-
asyncpg = { version = "^0.30.0", optional = true }
|
|
67
|
-
apscheduler = { version = "^3.11.0", optional = true }
|
|
66
|
+
psycopg2-binary = { version = "^2.9.11", optional = true }
|
|
67
|
+
asyncpg = { version = ">=0.30,<0.32", optional = true }
|
|
68
|
+
apscheduler = { version = "^3.11.1", optional = true }
|
|
68
69
|
starlette-exporter = { version = "^0.23.0", optional = true }
|
|
69
|
-
python-keycloak = { version = "^5.
|
|
70
|
+
python-keycloak = { version = "^5.8.1", optional = true }
|
|
70
71
|
itsdangerous = { version = "^2.2.0", optional = true }
|
|
71
|
-
horizon-hwm-store = { version = "^1.1.
|
|
72
|
+
horizon-hwm-store = { version = "^1.1.4", optional = true }
|
|
72
73
|
|
|
73
74
|
[tool.poetry.extras]
|
|
74
75
|
server = [
|
|
75
|
-
"pydantic-settings",
|
|
76
|
-
"sqlalchemy",
|
|
77
|
-
"sqlalchemy-utils",
|
|
78
76
|
"fastapi",
|
|
79
77
|
"starlette-exporter",
|
|
80
78
|
"uvicorn",
|
|
81
79
|
"alembic",
|
|
82
80
|
"python-multipart",
|
|
83
81
|
"pyjwt",
|
|
84
|
-
"asgi-correlation-id",
|
|
85
82
|
"jinja2",
|
|
86
83
|
"psycopg2-binary",
|
|
87
|
-
"uuid6",
|
|
88
|
-
"coloredlogs",
|
|
89
|
-
"python-json-logger",
|
|
90
84
|
"asyncpg",
|
|
91
85
|
"python-keycloak",
|
|
92
86
|
"itsdangerous",
|
|
93
87
|
"pyyaml",
|
|
94
|
-
"celery",
|
|
95
88
|
]
|
|
96
89
|
|
|
97
90
|
worker = [
|
|
98
|
-
"pydantic-settings",
|
|
99
|
-
"sqlalchemy",
|
|
100
|
-
"sqlalchemy-utils",
|
|
101
|
-
"celery",
|
|
102
91
|
"onetl",
|
|
103
92
|
"pyspark",
|
|
104
|
-
"asgi-correlation-id",
|
|
105
93
|
"jinja2",
|
|
106
94
|
"psycopg2-binary",
|
|
107
|
-
"uuid6",
|
|
108
|
-
"coloredlogs",
|
|
109
|
-
"python-json-logger",
|
|
110
|
-
"pyyaml",
|
|
111
95
|
"horizon-hwm-store",
|
|
112
96
|
]
|
|
113
97
|
|
|
114
98
|
scheduler = [
|
|
115
99
|
"apscheduler",
|
|
116
|
-
"pydantic-settings",
|
|
117
|
-
"sqlalchemy",
|
|
118
|
-
"sqlalchemy-utils",
|
|
119
100
|
"asyncpg",
|
|
120
|
-
"pyyaml",
|
|
121
|
-
"coloredlogs",
|
|
122
|
-
"python-json-logger",
|
|
123
|
-
"celery",
|
|
124
101
|
]
|
|
125
102
|
|
|
126
103
|
[tool.poetry.group.test.dependencies]
|
|
127
|
-
pytest = "^8.
|
|
104
|
+
pytest = "^8.4.2"
|
|
128
105
|
httpx = "^0.28.1"
|
|
129
106
|
pytest-asyncio = ">=0.26.0,<1.0"
|
|
130
|
-
pytest-randomly = "
|
|
131
|
-
pytest-rerunfailures = "
|
|
132
|
-
pytest-deadfixtures = "^
|
|
133
|
-
pytest-mock = "^3.
|
|
134
|
-
pytest-lazy-fixtures = "^1.
|
|
135
|
-
faker = "^
|
|
136
|
-
coverage = "^7.
|
|
137
|
-
gevent = "
|
|
107
|
+
pytest-randomly = "^4.0.1"
|
|
108
|
+
pytest-rerunfailures = "^16.1"
|
|
109
|
+
pytest-deadfixtures = "^3.0.0"
|
|
110
|
+
pytest-mock = "^3.15.1"
|
|
111
|
+
pytest-lazy-fixtures = "^1.4.0"
|
|
112
|
+
faker = "^38.2.0"
|
|
113
|
+
coverage = "^7.12.0"
|
|
114
|
+
gevent = "^25.9.1"
|
|
138
115
|
respx = "^0.22.0"
|
|
139
|
-
dirty-equals = "^0.
|
|
116
|
+
dirty-equals = "^0.11.0"
|
|
140
117
|
|
|
141
118
|
[tool.poetry.group.dev.dependencies]
|
|
142
|
-
mypy = "^1.
|
|
143
|
-
pre-commit = "^4.
|
|
144
|
-
black = "^25.
|
|
145
|
-
flake8 = "^7.
|
|
119
|
+
mypy = "^1.18.2"
|
|
120
|
+
pre-commit = "^4.5.0"
|
|
121
|
+
black = "^25.12.0"
|
|
122
|
+
flake8 = "^7.3.0"
|
|
146
123
|
flake8-pyproject = "^1.2.3"
|
|
147
|
-
sqlalchemy = {extras = ["mypy"], version = "^2.0.
|
|
124
|
+
sqlalchemy = {extras = ["mypy"], version = "^2.0.44"}
|
|
148
125
|
types-jwcrypto = "^1.5.0"
|
|
149
126
|
|
|
150
127
|
[tool.poetry.group.docs.dependencies]
|
|
151
128
|
autodoc-pydantic = "^2.2.0"
|
|
152
|
-
numpydoc = "^1.
|
|
129
|
+
numpydoc = "^1.9.0"
|
|
153
130
|
sphinx = "^8.2.3"
|
|
154
|
-
furo = "
|
|
131
|
+
furo = "^2025.9.25"
|
|
155
132
|
sphinx-copybutton = "^0.5.2"
|
|
156
133
|
sphinxcontrib-towncrier = "^0.5.0a0"
|
|
157
|
-
towncrier = "^
|
|
158
|
-
sphinx-issues = "^5.0.
|
|
134
|
+
towncrier = "^25.8.0"
|
|
135
|
+
sphinx-issues = "^5.0.1"
|
|
159
136
|
sphinx-design = "^0.6.1"
|
|
160
137
|
sphinx-favicon = "^1.0.1"
|
|
161
138
|
sphinx-argparse = "^0.5.2"
|
|
@@ -285,7 +262,7 @@ max-annotation-complexity = 4
|
|
|
285
262
|
max-returns = 5
|
|
286
263
|
max-awaits = 5
|
|
287
264
|
max-local-variables = 20
|
|
288
|
-
max-name-length =
|
|
265
|
+
max-name-length = 65
|
|
289
266
|
# Max of expressions in a function
|
|
290
267
|
max-expressions = 15
|
|
291
268
|
# Max args in a function
|
|
@@ -459,8 +436,10 @@ ignore = [
|
|
|
459
436
|
per-file-ignores = [
|
|
460
437
|
# WPS102 Found incorrect module name pattern
|
|
461
438
|
# WPS432 Found magic number: 256
|
|
462
|
-
|
|
463
|
-
|
|
439
|
+
# WPS226 Found string literal over-use: value > 15
|
|
440
|
+
# WPS342 Found implicit raw string
|
|
441
|
+
"*migrations/*.py:WPS102,WPS432,WPS226,WPS342",
|
|
442
|
+
"*db/models/*.py:WPS102,WPS432,WPS342",
|
|
464
443
|
"*db/mixins/*.py:WPS102,WPS432",
|
|
465
444
|
# WPS432 Found magic number: 180
|
|
466
445
|
"*settings/*.py:WPS432",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: 2023-
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-present MTS PJSC
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
_raw_version = "0.
|
|
4
|
+
_raw_version = "0.3.0"
|
|
5
5
|
# version always contain only release number like 0.0.1
|
|
6
6
|
__version__ = ".".join(_raw_version.split(".")[:3]) # noqa: WPS410
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: 2023-
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-present MTS PJSC
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
import asyncio
|
|
4
4
|
import os
|
|
@@ -7,23 +7,34 @@ from logging.config import fileConfig
|
|
|
7
7
|
from alembic import context
|
|
8
8
|
from alembic.script import ScriptDirectory
|
|
9
9
|
from celery.backends.database.session import ResultModelBase
|
|
10
|
+
from pydantic import Field
|
|
10
11
|
from sqlalchemy import pool
|
|
11
12
|
from sqlalchemy.engine import Connection
|
|
12
13
|
from sqlalchemy.ext.asyncio import async_engine_from_config
|
|
13
14
|
|
|
14
15
|
from syncmaster.db.models import Base
|
|
15
|
-
from syncmaster.server.settings import
|
|
16
|
+
from syncmaster.server.settings import (
|
|
17
|
+
DEFAULT_LOGGING_SETTINGS,
|
|
18
|
+
BaseSettings,
|
|
19
|
+
DatabaseSettings,
|
|
20
|
+
LoggingSettings,
|
|
21
|
+
)
|
|
16
22
|
|
|
17
23
|
config = context.config
|
|
18
24
|
|
|
19
25
|
|
|
26
|
+
class MigrationAppSettings(BaseSettings):
|
|
27
|
+
database: DatabaseSettings = Field(default_factory=DatabaseSettings, description="Database settings")
|
|
28
|
+
logging: LoggingSettings = Field(default=DEFAULT_LOGGING_SETTINGS, description="Logging settings")
|
|
29
|
+
|
|
30
|
+
|
|
20
31
|
if config.config_file_name is not None:
|
|
21
32
|
fileConfig(config.config_file_name)
|
|
22
33
|
|
|
23
34
|
if not config.get_main_option("sqlalchemy.url"):
|
|
24
35
|
# read application settings only if sqlalchemy.url is not being passed via cli arguments
|
|
25
36
|
# TODO: remove settings object creating during import
|
|
26
|
-
config.set_main_option("sqlalchemy.url",
|
|
37
|
+
config.set_main_option("sqlalchemy.url", MigrationAppSettings().database.url)
|
|
27
38
|
|
|
28
39
|
target_metadata = (
|
|
29
40
|
Base.metadata,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: 2023-
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-present MTS PJSC
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -18,6 +18,7 @@ from syncmaster.db.models.group import Group
|
|
|
18
18
|
class ConnectionType(StrEnum):
|
|
19
19
|
POSTGRES = "postgres"
|
|
20
20
|
HIVE = "hive"
|
|
21
|
+
ICEBERG = "iceberg"
|
|
21
22
|
ORACLE = "oracle"
|
|
22
23
|
CLICKHOUSE = "clickhouse"
|
|
23
24
|
MSSQL = "mssql"
|