data-syncmaster 0.2.4__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/PKG-INFO +11 -8
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/README.rst +3 -2
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/pyproject.toml +11 -9
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/__init__.py +1 -1
- data_syncmaster-0.2.5/syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py +243 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/connection.py +19 -5
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/group.py +16 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/queue.py +16 -2
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/transfer.py +38 -12
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/base.py +11 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/connection.py +3 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/group.py +7 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/queue.py +3 -1
- data_syncmaster-0.2.5/syncmaster/db/repositories/search.py +96 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/transfer.py +3 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/auth.py +16 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/auth.py +21 -11
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/handler.py +5 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/providers/auth/base_provider.py +8 -3
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/providers/auth/dummy_provider.py +14 -12
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/providers/auth/keycloak_provider.py +32 -30
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/base.py +3 -4
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/spark.py +1 -1
- data_syncmaster-0.2.4/syncmaster/server/utils/state.py +0 -15
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/LICENSE.txt +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/factory.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/README +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/__main__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/alembic.ini +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/env.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/script.py.mako +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0002_create_group_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0003_create_queue_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0004_create_connection_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0005_create_user_group_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0006_create_auth_data_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0007_create_transfer_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0008_create_run_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/migrations/versions/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/mixins/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/mixins/resource.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/mixins/timestamp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/apscheduler_job.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/auth_data.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/run.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/models/user.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/credentials_repository.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/repository_with_owner.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/run.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/user.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/repositories/utils.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/db/utils.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/dto/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/dto/connections.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/dto/runs.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/dto/transfers.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/dto/transfers_resources.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/dto/transfers_strategy.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/registration.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/schemas/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/schemas/bad_request.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/schemas/invalid_request.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/errors/schemas/not_authorized.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/connection.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/credentials.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/group.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/queue.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/redirect.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/run.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/transfer.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/exceptions/user.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/__main__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/celery.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/settings/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/transfer_fetcher.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/transfer_job_manager.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/scheduler/utils.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/ping.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/auth/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/auth/basic.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/auth/s3.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/auth/samba.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/auth/token.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connection_types.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/clickhouse.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/connection.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/connection_base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/ftp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/ftps.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/hdfs.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/hive.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/mssql.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/mysql.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/oracle.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/postgres.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/s3.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/samba.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/sftp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/connections/webdav.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/file_formats.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/groups.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/page.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/queue.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfer_types.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/db.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/ftp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/ftps.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/s3.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/samba.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/sftp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file/webdav.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/file_format.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/resources.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/run.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/strategy.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/transformations/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/transformations/dataframe_columns_filter.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/transformations/dataframe_rows_filter.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transfers/transformations/file_metadata_filter.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/transformation_types.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/types.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/schemas/v1/users.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/__main__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/monitoring.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/router.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/connections.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/groups.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/queue.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/router.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/runs.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/transfers.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/api/v1/users.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/dependencies/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/dependencies/get_access_token.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/dependencies/stub.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/cors.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/monitoring/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/monitoring/metrics.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/openapi.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/request_id.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/session.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/middlewares/static_files.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/providers/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/providers/auth/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/scripts/export_openapi_schema.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/scripts/manage_superusers.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/services/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/services/get_user.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/services/unit_of_work.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/auth/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/auth/dummy.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/auth/jwt.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/auth/keycloak.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/cors.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/monitoring.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/openapi.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/request_id.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/session.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/settings/server/static_files.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/utils/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/utils/jwt.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/server/utils/slug.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/broker.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/credentials.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/database.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/log/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/log/colored.yml +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/log/json.yml +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/settings/log/plain.yml +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/celery.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/controller.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/base.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/clickhouse.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/hive.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/mssql.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/mysql.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/oracle.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/db/postgres.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/ftp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/ftps.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/hdfs.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/local_df.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/remote_df.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/s3.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/samba.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/sftp.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/handlers/file/webdav.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/settings/__init__.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/settings/hwm_store.py +0 -0
- {data_syncmaster-0.2.4 → data_syncmaster-0.2.5}/syncmaster/worker/transfer.py +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: data-syncmaster
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Syncmaster REST API + Worker
|
|
5
5
|
License: Apache-2.0
|
|
6
|
+
License-File: LICENSE.txt
|
|
6
7
|
Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
|
|
7
8
|
Author: DataOps.ETL
|
|
8
9
|
Author-email: onetools@mts.ru
|
|
@@ -18,6 +19,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
18
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
21
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
23
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
22
24
|
Classifier: Topic :: Software Development :: Libraries
|
|
23
25
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
@@ -31,11 +33,11 @@ Requires-Dist: asgi-correlation-id (>=4.3.4,<5.0.0) ; extra == "server" or extra
|
|
|
31
33
|
Requires-Dist: asyncpg (>=0.30.0,<0.31.0) ; extra == "server" or extra == "scheduler"
|
|
32
34
|
Requires-Dist: celery (>=5.5.0,<6.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
33
35
|
Requires-Dist: coloredlogs ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
34
|
-
Requires-Dist: fastapi (>=0.115.12,<0.
|
|
36
|
+
Requires-Dist: fastapi (>=0.115.12,<0.119.0) ; extra == "server"
|
|
35
37
|
Requires-Dist: horizon-hwm-store (>=1.1.2,<2.0.0) ; extra == "worker"
|
|
36
38
|
Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
|
|
37
39
|
Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
|
|
38
|
-
Requires-Dist: onetl[all] (>=0.13.5,<0.
|
|
40
|
+
Requires-Dist: onetl[all] (>=0.13.5,<0.15.0) ; extra == "worker"
|
|
39
41
|
Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0) ; extra == "server" or extra == "worker"
|
|
40
42
|
Requires-Dist: pydantic (>=2.11.7,<3.0.0)
|
|
41
43
|
Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
@@ -46,10 +48,10 @@ Requires-Dist: python-keycloak (>=5.3.1,<6.0.0) ; extra == "server"
|
|
|
46
48
|
Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
|
|
47
49
|
Requires-Dist: pyyaml ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
48
50
|
Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
49
|
-
Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.
|
|
51
|
+
Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.43.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
|
|
50
52
|
Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
|
|
51
53
|
Requires-Dist: uuid6 (>=2025.0.0,<2026.0.0) ; extra == "server" or extra == "worker"
|
|
52
|
-
Requires-Dist: uvicorn (>=0.34,<0.
|
|
54
|
+
Requires-Dist: uvicorn (>=0.34,<0.38) ; extra == "server"
|
|
53
55
|
Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
|
|
54
56
|
Project-URL: Documentation, https://syncmaster.readthedocs.io
|
|
55
57
|
Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster
|
|
@@ -79,8 +81,9 @@ Data.SyncMaster
|
|
|
79
81
|
:target: https://syncmaster.readthedocs.io
|
|
80
82
|
.. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Run%20All%20Tests/badge.svg
|
|
81
83
|
:target: https://github.com/MobileTeleSystems/syncmaster/actions
|
|
82
|
-
.. |Coverage| image:: https://
|
|
83
|
-
|
|
84
|
+
.. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
|
|
85
|
+
MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
|
|
86
|
+
:target: https://github.com/MobileTeleSystems/syncmaster/actions
|
|
84
87
|
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
85
88
|
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
86
89
|
|
|
@@ -20,8 +20,9 @@ Data.SyncMaster
|
|
|
20
20
|
:target: https://syncmaster.readthedocs.io
|
|
21
21
|
.. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Run%20All%20Tests/badge.svg
|
|
22
22
|
:target: https://github.com/MobileTeleSystems/syncmaster/actions
|
|
23
|
-
.. |Coverage| image:: https://
|
|
24
|
-
|
|
23
|
+
.. |Coverage| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/
|
|
24
|
+
MTSOnGithub/03e73a82ecc4709934540ce8201cc3b4/raw/syncmaster_badge.json
|
|
25
|
+
:target: https://github.com/MobileTeleSystems/syncmaster/actions
|
|
25
26
|
.. |pre-commit.ci| image:: https://results.pre-commit.ci/badge/github/MobileTeleSystems/syncmaster/develop.svg
|
|
26
27
|
:target: https://results.pre-commit.ci/latest/github/MobileTeleSystems/syncmaster/develop
|
|
27
28
|
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "data-syncmaster"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.5"
|
|
8
8
|
license = "Apache-2.0"
|
|
9
9
|
description = "Syncmaster REST API + Worker"
|
|
10
10
|
authors = ["DataOps.ETL <onetools@mts.ru>"]
|
|
@@ -47,16 +47,16 @@ python = "^3.11"
|
|
|
47
47
|
pydantic = "^2.11.7"
|
|
48
48
|
pydantic-settings = { version = "^2.8.1", optional = true }
|
|
49
49
|
sqlalchemy = { version = "^2.0.40", optional = true }
|
|
50
|
-
sqlalchemy-utils = { version = "
|
|
51
|
-
fastapi = { version = "
|
|
50
|
+
sqlalchemy-utils = { version = ">=0.41.2,<0.43.0", optional = true }
|
|
51
|
+
fastapi = { version = ">=0.115.12,<0.119.0", optional = true}
|
|
52
52
|
asgi-correlation-id = {version = "^4.3.4", optional = true}
|
|
53
|
-
uvicorn = { version = ">=0.34,<0.
|
|
53
|
+
uvicorn = { version = ">=0.34,<0.38", optional = true }
|
|
54
54
|
alembic = { version = "^1.15.2", optional = true }
|
|
55
55
|
pyjwt = { version = "^2.10.1", optional = true }
|
|
56
56
|
jinja2 = { version = "^3.1.6", optional = true }
|
|
57
57
|
python-multipart = { version = "^0.0.20", optional = true }
|
|
58
58
|
celery = { version = "^5.5.0", optional = true }
|
|
59
|
-
onetl = { version = "
|
|
59
|
+
onetl = { version = ">=0.13.5,<0.15.0", extras = ["all"], optional = true }
|
|
60
60
|
pyspark = { version = "<4.0.0", optional = true }
|
|
61
61
|
pyyaml = { version = "*", optional = true }
|
|
62
62
|
psycopg2-binary = { version = "^2.9.10", optional = true }
|
|
@@ -127,15 +127,16 @@ scheduler = [
|
|
|
127
127
|
pytest = "^8.3.5"
|
|
128
128
|
httpx = "^0.28.1"
|
|
129
129
|
pytest-asyncio = ">=0.26.0,<1.0"
|
|
130
|
-
pytest-randomly = "
|
|
131
|
-
pytest-rerunfailures = "
|
|
130
|
+
pytest-randomly = ">=3.15,<5.0"
|
|
131
|
+
pytest-rerunfailures = ">=15,<17"
|
|
132
132
|
pytest-deadfixtures = "^2.2.1"
|
|
133
133
|
pytest-mock = "^3.14.0"
|
|
134
134
|
pytest-lazy-fixtures = "^1.1.1"
|
|
135
135
|
faker = "^37.4.0"
|
|
136
136
|
coverage = "^7.9.1"
|
|
137
137
|
gevent = ">=24.11.1,<26.0.0"
|
|
138
|
-
|
|
138
|
+
respx = "^0.22.0"
|
|
139
|
+
dirty-equals = "^0.9.0"
|
|
139
140
|
|
|
140
141
|
[tool.poetry.group.dev.dependencies]
|
|
141
142
|
mypy = "^1.15.0"
|
|
@@ -144,12 +145,13 @@ black = "^25.1.0"
|
|
|
144
145
|
flake8 = "^7.2.0"
|
|
145
146
|
flake8-pyproject = "^1.2.3"
|
|
146
147
|
sqlalchemy = {extras = ["mypy"], version = "^2.0.40"}
|
|
148
|
+
types-jwcrypto = "^1.5.0"
|
|
147
149
|
|
|
148
150
|
[tool.poetry.group.docs.dependencies]
|
|
149
151
|
autodoc-pydantic = "^2.2.0"
|
|
150
152
|
numpydoc = "^1.8.0"
|
|
151
153
|
sphinx = "^8.2.3"
|
|
152
|
-
furo = "
|
|
154
|
+
furo = ">=2024.8.6,<2026.0.0"
|
|
153
155
|
sphinx-copybutton = "^0.5.2"
|
|
154
156
|
sphinxcontrib-towncrier = "^0.5.0a0"
|
|
155
157
|
towncrier = "^24.8.0"
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""Update text search
|
|
4
|
+
|
|
5
|
+
Revision ID: 0012
|
|
6
|
+
Revises: 0011
|
|
7
|
+
Create Date: 2025-08-10 20:03:02.105470
|
|
8
|
+
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import sqlalchemy as sa
|
|
12
|
+
from alembic import op
|
|
13
|
+
from sqlalchemy.dialects import postgresql
|
|
14
|
+
|
|
15
|
+
# revision identifiers, used by Alembic.
|
|
16
|
+
revision = "0012"
|
|
17
|
+
down_revision = "0011"
|
|
18
|
+
branch_labels = None
|
|
19
|
+
depends_on = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def upgrade() -> None:
|
|
23
|
+
op.drop_index(op.f("idx_connection_search_vector"), table_name="connection", postgresql_using="gin")
|
|
24
|
+
op.drop_column("connection", "search_vector")
|
|
25
|
+
op.drop_column("group", "search_vector")
|
|
26
|
+
op.drop_index(op.f("idx_transfer_search_vector"), table_name="transfer", postgresql_using="gin")
|
|
27
|
+
op.drop_column("transfer", "search_vector")
|
|
28
|
+
op.drop_column("queue", "search_vector")
|
|
29
|
+
|
|
30
|
+
op.add_column(
|
|
31
|
+
"connection",
|
|
32
|
+
sa.Column(
|
|
33
|
+
"search_vector",
|
|
34
|
+
postgresql.TSVECTOR(),
|
|
35
|
+
sa.Computed(
|
|
36
|
+
"""
|
|
37
|
+
-- === NAME FIELD ===
|
|
38
|
+
-- Russian stemming for better morphological matching of regular words
|
|
39
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
40
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
41
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
42
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
43
|
+
-- (used when 'name' contains technical fields)
|
|
44
|
+
|| to_tsvector(
|
|
45
|
+
'simple',
|
|
46
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
-- === HOST FIELD (from JSON) ===
|
|
50
|
+
-- Simple dictionary (no stemming) for exact match
|
|
51
|
+
|| to_tsvector('simple', coalesce(data->>'host', ''))
|
|
52
|
+
-- Simple dictionary with translate(): split by . / - _ : \\ for partial token matching
|
|
53
|
+
|| to_tsvector(
|
|
54
|
+
'simple',
|
|
55
|
+
translate(coalesce(data->>'host', ''), './-_:\\', ' ')
|
|
56
|
+
)
|
|
57
|
+
""",
|
|
58
|
+
persisted=True,
|
|
59
|
+
),
|
|
60
|
+
nullable=False,
|
|
61
|
+
),
|
|
62
|
+
)
|
|
63
|
+
op.create_index(
|
|
64
|
+
"idx_connection_search_vector",
|
|
65
|
+
"connection",
|
|
66
|
+
["search_vector"],
|
|
67
|
+
unique=False,
|
|
68
|
+
postgresql_using="gin",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
op.add_column(
|
|
72
|
+
"group",
|
|
73
|
+
sa.Column(
|
|
74
|
+
"search_vector",
|
|
75
|
+
postgresql.TSVECTOR(),
|
|
76
|
+
sa.Computed(
|
|
77
|
+
"""
|
|
78
|
+
-- === NAME FIELD ===
|
|
79
|
+
-- Russian stemming for better morphological matching of regular words
|
|
80
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
81
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
82
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
83
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
84
|
+
-- (used when 'name' contains technical fields)
|
|
85
|
+
|| to_tsvector(
|
|
86
|
+
'simple',
|
|
87
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
88
|
+
)
|
|
89
|
+
""",
|
|
90
|
+
persisted=True,
|
|
91
|
+
),
|
|
92
|
+
nullable=False,
|
|
93
|
+
),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
op.add_column(
|
|
97
|
+
"queue",
|
|
98
|
+
sa.Column(
|
|
99
|
+
"search_vector",
|
|
100
|
+
postgresql.TSVECTOR(),
|
|
101
|
+
sa.Computed(
|
|
102
|
+
"""
|
|
103
|
+
-- === NAME FIELD ===
|
|
104
|
+
-- Russian stemming for better morphological matching of regular words
|
|
105
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
106
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
107
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
108
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
109
|
+
-- (used when 'name' contains technical fields)
|
|
110
|
+
|| to_tsvector(
|
|
111
|
+
'simple',
|
|
112
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
113
|
+
)
|
|
114
|
+
""",
|
|
115
|
+
persisted=True,
|
|
116
|
+
),
|
|
117
|
+
nullable=False,
|
|
118
|
+
),
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
op.add_column(
|
|
122
|
+
"transfer",
|
|
123
|
+
sa.Column(
|
|
124
|
+
"search_vector",
|
|
125
|
+
postgresql.TSVECTOR(),
|
|
126
|
+
sa.Computed(
|
|
127
|
+
"""
|
|
128
|
+
-- === NAME FIELD ===
|
|
129
|
+
-- Russian stemming for better morphological matching of regular words
|
|
130
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
131
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
132
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
133
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
134
|
+
-- (used when 'name' contains technical fields)
|
|
135
|
+
|| to_tsvector(
|
|
136
|
+
'simple',
|
|
137
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
-- === TABLE NAME FIELDS ===
|
|
141
|
+
-- Simple dictionary (no stemming) for exact match
|
|
142
|
+
|| to_tsvector('simple', coalesce(source_params->>'table_name', ''))
|
|
143
|
+
|| to_tsvector('simple', coalesce(target_params->>'table_name', ''))
|
|
144
|
+
-- Simple dictionary with translate(): split by . / - _ : \\ for partial token matching
|
|
145
|
+
|| to_tsvector(
|
|
146
|
+
'simple',
|
|
147
|
+
translate(coalesce(source_params->>'table_name', ''), './-_:\\', ' ')
|
|
148
|
+
)
|
|
149
|
+
|| to_tsvector(
|
|
150
|
+
'simple',
|
|
151
|
+
translate(coalesce(target_params->>'table_name', ''), './-_:\\', ' ')
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
-- === DIRECTORY PATH FIELDS ===
|
|
155
|
+
-- Simple dictionary (no stemming) for exact match
|
|
156
|
+
|| to_tsvector('simple', coalesce(source_params->>'directory_path', ''))
|
|
157
|
+
|| to_tsvector('simple', coalesce(target_params->>'directory_path', ''))
|
|
158
|
+
-- Simple dictionary with translate(): split by . / - _ : \\ for partial token matching
|
|
159
|
+
|| to_tsvector(
|
|
160
|
+
'simple',
|
|
161
|
+
translate(coalesce(source_params->>'directory_path', ''), './-_:\\', ' ')
|
|
162
|
+
)
|
|
163
|
+
|| to_tsvector(
|
|
164
|
+
'simple',
|
|
165
|
+
translate(coalesce(target_params->>'directory_path', ''), './-_:\\', ' ')
|
|
166
|
+
)
|
|
167
|
+
""",
|
|
168
|
+
persisted=True,
|
|
169
|
+
),
|
|
170
|
+
nullable=False,
|
|
171
|
+
),
|
|
172
|
+
)
|
|
173
|
+
op.create_index("idx_transfer_search_vector", "transfer", ["search_vector"], unique=False, postgresql_using="gin")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def downgrade() -> None:
|
|
177
|
+
op.drop_index("idx_transfer_search_vector", table_name="transfer", postgresql_using="gin")
|
|
178
|
+
op.drop_column("transfer", "search_vector")
|
|
179
|
+
op.drop_column("group", "search_vector")
|
|
180
|
+
op.drop_index("idx_connection_search_vector", table_name="connection", postgresql_using="gin")
|
|
181
|
+
op.drop_column("connection", "search_vector")
|
|
182
|
+
op.drop_column("queue", "search_vector")
|
|
183
|
+
|
|
184
|
+
op.add_column(
|
|
185
|
+
"transfer",
|
|
186
|
+
sa.Column(
|
|
187
|
+
"search_vector",
|
|
188
|
+
postgresql.TSVECTOR(),
|
|
189
|
+
sa.Computed(
|
|
190
|
+
"to_tsvector('english'::regconfig, (((((((((((((((((((name)::text || ' '::text) || COALESCE(json_extract_path_text(source_params, VARIADIC ARRAY['table_name'::text]), ''::text)) || ' '::text) || COALESCE(json_extract_path_text(target_params, VARIADIC ARRAY['table_name'::text]), ''::text)) || ' '::text) || COALESCE(json_extract_path_text(source_params, VARIADIC ARRAY['directory_path'::text]), ''::text)) || ' '::text) || COALESCE(json_extract_path_text(target_params, VARIADIC ARRAY['directory_path'::text]), ''::text)) || ' '::text) || translate((name)::text, './'::text, ' '::text)) || ' '::text) || COALESCE(translate(json_extract_path_text(source_params, VARIADIC ARRAY['table_name'::text]), './'::text, ' '::text), ''::text)) || ' '::text) || COALESCE(translate(json_extract_path_text(target_params, VARIADIC ARRAY['table_name'::text]), './'::text, ' '::text), ''::text)) || ' '::text) || COALESCE(translate(json_extract_path_text(source_params, VARIADIC ARRAY['directory_path'::text]), './'::text, ' '::text), ''::text)) || ' '::text) || COALESCE(translate(json_extract_path_text(target_params, VARIADIC ARRAY['directory_path'::text]), './'::text, ' '::text), ''::text)))",
|
|
191
|
+
persisted=True,
|
|
192
|
+
),
|
|
193
|
+
autoincrement=False,
|
|
194
|
+
nullable=False,
|
|
195
|
+
),
|
|
196
|
+
)
|
|
197
|
+
op.create_index(
|
|
198
|
+
op.f("idx_transfer_search_vector"),
|
|
199
|
+
"transfer",
|
|
200
|
+
["search_vector"],
|
|
201
|
+
unique=False,
|
|
202
|
+
postgresql_using="gin",
|
|
203
|
+
)
|
|
204
|
+
op.add_column(
|
|
205
|
+
"group",
|
|
206
|
+
sa.Column(
|
|
207
|
+
"search_vector",
|
|
208
|
+
postgresql.TSVECTOR(),
|
|
209
|
+
sa.Computed("to_tsvector('english'::regconfig, (name)::text)", persisted=True),
|
|
210
|
+
autoincrement=False,
|
|
211
|
+
nullable=False,
|
|
212
|
+
),
|
|
213
|
+
)
|
|
214
|
+
op.add_column(
|
|
215
|
+
"connection",
|
|
216
|
+
sa.Column(
|
|
217
|
+
"search_vector",
|
|
218
|
+
postgresql.TSVECTOR(),
|
|
219
|
+
sa.Computed(
|
|
220
|
+
"to_tsvector('english'::regconfig, (((((name)::text || ' '::text) || COALESCE(json_extract_path_text(data, VARIADIC ARRAY['host'::text]), ''::text)) || ' '::text) || COALESCE(translate(json_extract_path_text(data, VARIADIC ARRAY['host'::text]), '.'::text, ' '::text), ''::text)))",
|
|
221
|
+
persisted=True,
|
|
222
|
+
),
|
|
223
|
+
autoincrement=False,
|
|
224
|
+
nullable=False,
|
|
225
|
+
),
|
|
226
|
+
)
|
|
227
|
+
op.create_index(
|
|
228
|
+
op.f("idx_connection_search_vector"),
|
|
229
|
+
"connection",
|
|
230
|
+
["search_vector"],
|
|
231
|
+
unique=False,
|
|
232
|
+
postgresql_using="gin",
|
|
233
|
+
)
|
|
234
|
+
op.add_column(
|
|
235
|
+
"queue",
|
|
236
|
+
sa.Column(
|
|
237
|
+
"search_vector",
|
|
238
|
+
postgresql.TSVECTOR(),
|
|
239
|
+
sa.Computed("to_tsvector('english'::regconfig, (name)::text)", persisted=True),
|
|
240
|
+
autoincrement=False,
|
|
241
|
+
nullable=False,
|
|
242
|
+
),
|
|
243
|
+
)
|
|
@@ -41,11 +41,25 @@ class Connection(Base, ResourceMixin, TimestampMixin):
|
|
|
41
41
|
TSVECTOR,
|
|
42
42
|
Computed(
|
|
43
43
|
"""
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
-- === NAME FIELD ===
|
|
45
|
+
-- Russian stemming for better morphological matching of regular words
|
|
46
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
47
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
48
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
49
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
50
|
+
-- (used when 'name' contains technical fields)
|
|
51
|
+
|| to_tsvector(
|
|
52
|
+
'simple',
|
|
53
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
-- === HOST FIELD (from JSON) ===
|
|
57
|
+
-- Simple dictionary (no stemming) for exact match
|
|
58
|
+
|| to_tsvector('simple', coalesce(data->>'host', ''))
|
|
59
|
+
-- Simple dictionary with translate(): split by . / - _ : \\ for partial token matching
|
|
60
|
+
|| to_tsvector(
|
|
61
|
+
'simple',
|
|
62
|
+
translate(coalesce(data->>'host', ''), './-_:\\', ' ')
|
|
49
63
|
)
|
|
50
64
|
""",
|
|
51
65
|
persisted=True,
|
|
@@ -77,13 +77,26 @@ class Group(Base, TimestampMixin):
|
|
|
77
77
|
|
|
78
78
|
owner: Mapped[User] = relationship(User)
|
|
79
79
|
queue: Mapped[Queue] = relationship(back_populates="group", cascade="all, delete-orphan")
|
|
80
|
-
|
|
81
80
|
search_vector: Mapped[str] = mapped_column(
|
|
82
81
|
TSVECTOR,
|
|
83
|
-
Computed(
|
|
82
|
+
Computed(
|
|
83
|
+
"""
|
|
84
|
+
-- === NAME FIELD ===
|
|
85
|
+
-- Russian stemming for better morphological matching of regular words
|
|
86
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
87
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
88
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
89
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
90
|
+
-- (used when 'name' contains technical fields)
|
|
91
|
+
|| to_tsvector(
|
|
92
|
+
'simple',
|
|
93
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
94
|
+
)
|
|
95
|
+
""",
|
|
96
|
+
persisted=True,
|
|
97
|
+
),
|
|
84
98
|
nullable=False,
|
|
85
99
|
deferred=True,
|
|
86
|
-
doc="Full-text search vector",
|
|
87
100
|
)
|
|
88
101
|
|
|
89
102
|
def __repr__(self) -> str:
|
|
@@ -25,10 +25,24 @@ class Queue(Base, ResourceMixin, TimestampMixin):
|
|
|
25
25
|
|
|
26
26
|
search_vector: Mapped[str] = mapped_column(
|
|
27
27
|
TSVECTOR,
|
|
28
|
-
Computed(
|
|
28
|
+
Computed(
|
|
29
|
+
"""
|
|
30
|
+
-- === NAME FIELD ===
|
|
31
|
+
-- Russian stemming for better morphological matching of regular words
|
|
32
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
33
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
34
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
35
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
36
|
+
-- (used when 'name' contains technical fields)
|
|
37
|
+
|| to_tsvector(
|
|
38
|
+
'simple',
|
|
39
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
40
|
+
)
|
|
41
|
+
""",
|
|
42
|
+
persisted=True,
|
|
43
|
+
),
|
|
29
44
|
nullable=False,
|
|
30
45
|
deferred=True,
|
|
31
|
-
doc="Full-text search vector",
|
|
32
46
|
)
|
|
33
47
|
|
|
34
48
|
def __repr__(self):
|
|
@@ -65,18 +65,44 @@ class Transfer(
|
|
|
65
65
|
TSVECTOR,
|
|
66
66
|
Computed(
|
|
67
67
|
"""
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
68
|
+
-- === NAME FIELD ===
|
|
69
|
+
-- Russian stemming for better morphological matching of regular words
|
|
70
|
+
to_tsvector('russian', coalesce(name, ''))
|
|
71
|
+
-- Simple dictionary (no stemming) for exact token match
|
|
72
|
+
|| to_tsvector('simple', coalesce(name, ''))
|
|
73
|
+
-- Simple dictionary with translate(): split by . / - _ : \
|
|
74
|
+
-- (used when 'name' contains technical fields)
|
|
75
|
+
|| to_tsvector(
|
|
76
|
+
'simple',
|
|
77
|
+
translate(coalesce(name, ''), './-_:\\', ' ')
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
-- === TABLE NAME FIELDS ===
|
|
81
|
+
-- Simple dictionary (no stemming) for exact match
|
|
82
|
+
|| to_tsvector('simple', coalesce(source_params->>'table_name', ''))
|
|
83
|
+
|| to_tsvector('simple', coalesce(target_params->>'table_name', ''))
|
|
84
|
+
-- Simple dictionary with translate(): split by . / - _ : \\ for partial token matching
|
|
85
|
+
|| to_tsvector(
|
|
86
|
+
'simple',
|
|
87
|
+
translate(coalesce(source_params->>'table_name', ''), './-_:\\', ' ')
|
|
88
|
+
)
|
|
89
|
+
|| to_tsvector(
|
|
90
|
+
'simple',
|
|
91
|
+
translate(coalesce(target_params->>'table_name', ''), './-_:\\', ' ')
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
-- === DIRECTORY PATH FIELDS ===
|
|
95
|
+
-- Simple dictionary (no stemming) for exact match
|
|
96
|
+
|| to_tsvector('simple', coalesce(source_params->>'directory_path', ''))
|
|
97
|
+
|| to_tsvector('simple', coalesce(target_params->>'directory_path', ''))
|
|
98
|
+
-- Simple dictionary with translate(): split by . / - _ : \\ for partial token matching
|
|
99
|
+
|| to_tsvector(
|
|
100
|
+
'simple',
|
|
101
|
+
translate(coalesce(source_params->>'directory_path', ''), './-_:\\', ' ')
|
|
102
|
+
)
|
|
103
|
+
|| to_tsvector(
|
|
104
|
+
'simple',
|
|
105
|
+
translate(coalesce(target_params->>'directory_path', ''), './-_:\\', ' ')
|
|
80
106
|
)
|
|
81
107
|
""",
|
|
82
108
|
persisted=True,
|
|
@@ -3,7 +3,16 @@
|
|
|
3
3
|
from abc import ABC
|
|
4
4
|
from typing import Any, Generic, TypeVar
|
|
5
5
|
|
|
6
|
-
from sqlalchemy import
|
|
6
|
+
from sqlalchemy import (
|
|
7
|
+
ColumnElement,
|
|
8
|
+
ScalarResult,
|
|
9
|
+
Select,
|
|
10
|
+
delete,
|
|
11
|
+
func,
|
|
12
|
+
insert,
|
|
13
|
+
select,
|
|
14
|
+
update,
|
|
15
|
+
)
|
|
7
16
|
from sqlalchemy.exc import NoResultFound
|
|
8
17
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
9
18
|
|
|
@@ -98,8 +107,7 @@ class Repository(Generic[Model], ABC):
|
|
|
98
107
|
page_size=page_size,
|
|
99
108
|
)
|
|
100
109
|
|
|
101
|
-
def _construct_vector_search(self, query: Select,
|
|
102
|
-
ts_query = func.plainto_tsquery("english", search_query)
|
|
110
|
+
def _construct_vector_search(self, query: Select, ts_query: ColumnElement) -> Select:
|
|
103
111
|
query = (
|
|
104
112
|
query.where(self._model.search_vector.op("@@")(ts_query))
|
|
105
113
|
.add_columns(func.ts_rank(self._model.search_vector, ts_query).label("rank"))
|
|
@@ -8,6 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
8
8
|
|
|
9
9
|
from syncmaster.db.models import Connection
|
|
10
10
|
from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner
|
|
11
|
+
from syncmaster.db.repositories.search import make_tsquery
|
|
11
12
|
from syncmaster.db.utils import Pagination
|
|
12
13
|
from syncmaster.exceptions import EntityNotFoundError, SyncmasterError
|
|
13
14
|
from syncmaster.exceptions.connection import (
|
|
@@ -35,9 +36,8 @@ class ConnectionRepository(RepositoryWithOwner[Connection]):
|
|
|
35
36
|
Connection.group_id == group_id,
|
|
36
37
|
)
|
|
37
38
|
if search_query:
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
stmt = self._construct_vector_search(stmt, combined_query)
|
|
39
|
+
ts_query = make_tsquery(search_query)
|
|
40
|
+
stmt = self._construct_vector_search(stmt, ts_query)
|
|
41
41
|
|
|
42
42
|
if connection_type is not None:
|
|
43
43
|
stmt = stmt.where(Connection.type.in_(connection_type))
|
|
@@ -10,6 +10,7 @@ from sqlalchemy.orm import joinedload
|
|
|
10
10
|
|
|
11
11
|
from syncmaster.db.models import Group, GroupMemberRole, User, UserGroup
|
|
12
12
|
from syncmaster.db.repositories.base import Repository
|
|
13
|
+
from syncmaster.db.repositories.search import make_tsquery
|
|
13
14
|
from syncmaster.db.utils import Pagination, Permission
|
|
14
15
|
from syncmaster.exceptions import EntityNotFoundError, SyncmasterError
|
|
15
16
|
from syncmaster.exceptions.group import (
|
|
@@ -33,7 +34,8 @@ class GroupRepository(Repository[Group]):
|
|
|
33
34
|
) -> Pagination:
|
|
34
35
|
stmt = select(Group)
|
|
35
36
|
if search_query:
|
|
36
|
-
|
|
37
|
+
ts_query = make_tsquery(search_query)
|
|
38
|
+
stmt = self._construct_vector_search(stmt, ts_query)
|
|
37
39
|
|
|
38
40
|
paginated_result = await self._paginate_scalar_result(
|
|
39
41
|
query=stmt.order_by(Group.name),
|
|
@@ -78,7 +80,8 @@ class GroupRepository(Repository[Group]):
|
|
|
78
80
|
|
|
79
81
|
# apply search filtering if a search query is provided
|
|
80
82
|
if search_query:
|
|
81
|
-
|
|
83
|
+
ts_query = make_tsquery(search_query)
|
|
84
|
+
owned_groups_stmt = self._construct_vector_search(owned_groups_stmt, ts_query)
|
|
82
85
|
|
|
83
86
|
# get total count of owned groups
|
|
84
87
|
total_owned_groups = (
|
|
@@ -114,7 +117,8 @@ class GroupRepository(Repository[Group]):
|
|
|
114
117
|
|
|
115
118
|
# apply search filtering if a search query is provided
|
|
116
119
|
if search_query:
|
|
117
|
-
|
|
120
|
+
ts_query = make_tsquery(search_query)
|
|
121
|
+
user_groups_stmt = self._construct_vector_search(user_groups_stmt, ts_query)
|
|
118
122
|
|
|
119
123
|
# get total count of user groups
|
|
120
124
|
total_user_groups = (
|
|
@@ -9,6 +9,7 @@ from sqlalchemy.orm import selectinload
|
|
|
9
9
|
|
|
10
10
|
from syncmaster.db.models import Group, GroupMemberRole, Queue, User, UserGroup
|
|
11
11
|
from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner
|
|
12
|
+
from syncmaster.db.repositories.search import make_tsquery
|
|
12
13
|
from syncmaster.db.utils import Permission
|
|
13
14
|
from syncmaster.exceptions import EntityNotFoundError, SyncmasterError
|
|
14
15
|
from syncmaster.exceptions.group import GroupNotFoundError
|
|
@@ -59,7 +60,8 @@ class QueueRepository(RepositoryWithOwner[Queue]):
|
|
|
59
60
|
Queue.group_id == group_id,
|
|
60
61
|
)
|
|
61
62
|
if search_query:
|
|
62
|
-
|
|
63
|
+
ts_query = make_tsquery(search_query)
|
|
64
|
+
stmt = self._construct_vector_search(stmt, ts_query)
|
|
63
65
|
|
|
64
66
|
return await self._paginate_scalar_result(
|
|
65
67
|
query=stmt.order_by(Queue.id),
|