data-syncmaster 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. data_syncmaster-0.1.1.dist-info/LICENSE.txt +203 -0
  2. data_syncmaster-0.1.1.dist-info/METADATA +115 -0
  3. data_syncmaster-0.1.1.dist-info/RECORD +110 -0
  4. data_syncmaster-0.1.1.dist-info/WHEEL +4 -0
  5. syncmaster/__init__.py +6 -0
  6. syncmaster/backend/__init__.py +2 -0
  7. syncmaster/backend/api/__init__.py +2 -0
  8. syncmaster/backend/api/deps.py +20 -0
  9. syncmaster/backend/api/monitoring.py +10 -0
  10. syncmaster/backend/api/router.py +10 -0
  11. syncmaster/backend/api/v1/__init__.py +2 -0
  12. syncmaster/backend/api/v1/auth/__init__.py +2 -0
  13. syncmaster/backend/api/v1/auth/router.py +32 -0
  14. syncmaster/backend/api/v1/auth/utils.py +26 -0
  15. syncmaster/backend/api/v1/connections.py +300 -0
  16. syncmaster/backend/api/v1/groups.py +225 -0
  17. syncmaster/backend/api/v1/queue.py +148 -0
  18. syncmaster/backend/api/v1/router.py +18 -0
  19. syncmaster/backend/api/v1/transfers/__init__.py +2 -0
  20. syncmaster/backend/api/v1/transfers/router.py +469 -0
  21. syncmaster/backend/api/v1/transfers/utils.py +17 -0
  22. syncmaster/backend/api/v1/users.py +75 -0
  23. syncmaster/backend/export_openapi_schema.py +26 -0
  24. syncmaster/backend/handler.py +203 -0
  25. syncmaster/backend/logger.py +2 -0
  26. syncmaster/backend/main.py +63 -0
  27. syncmaster/backend/pre_start.py +94 -0
  28. syncmaster/backend/services/__init__.py +4 -0
  29. syncmaster/backend/services/auth.py +58 -0
  30. syncmaster/backend/services/unit_of_work.py +44 -0
  31. syncmaster/config.py +110 -0
  32. syncmaster/db/__init__.py +2 -0
  33. syncmaster/db/alembic.ini +41 -0
  34. syncmaster/db/base.py +28 -0
  35. syncmaster/db/factory.py +37 -0
  36. syncmaster/db/migrations/README +1 -0
  37. syncmaster/db/migrations/__init__.py +2 -0
  38. syncmaster/db/migrations/env.py +87 -0
  39. syncmaster/db/migrations/script.py.mako +24 -0
  40. syncmaster/db/migrations/versions/2023-11-23_478240cdad4b_init.py +242 -0
  41. syncmaster/db/migrations/versions/__init__.py +2 -0
  42. syncmaster/db/mixins.py +33 -0
  43. syncmaster/db/models.py +194 -0
  44. syncmaster/db/repositories/__init__.py +22 -0
  45. syncmaster/db/repositories/base.py +109 -0
  46. syncmaster/db/repositories/connection.py +138 -0
  47. syncmaster/db/repositories/credentials_repository.py +87 -0
  48. syncmaster/db/repositories/group.py +264 -0
  49. syncmaster/db/repositories/queue.py +195 -0
  50. syncmaster/db/repositories/repository_with_owner.py +115 -0
  51. syncmaster/db/repositories/run.py +78 -0
  52. syncmaster/db/repositories/transfer.py +202 -0
  53. syncmaster/db/repositories/user.py +72 -0
  54. syncmaster/db/repositories/utils.py +25 -0
  55. syncmaster/db/utils.py +31 -0
  56. syncmaster/dto/__init__.py +2 -0
  57. syncmaster/dto/connections.py +60 -0
  58. syncmaster/dto/transfers.py +46 -0
  59. syncmaster/exceptions/__init__.py +13 -0
  60. syncmaster/exceptions/base.py +12 -0
  61. syncmaster/exceptions/connection.py +28 -0
  62. syncmaster/exceptions/credentials.py +8 -0
  63. syncmaster/exceptions/group.py +27 -0
  64. syncmaster/exceptions/queue.py +16 -0
  65. syncmaster/exceptions/run.py +19 -0
  66. syncmaster/exceptions/transfer.py +39 -0
  67. syncmaster/exceptions/user.py +11 -0
  68. syncmaster/schemas/__init__.py +2 -0
  69. syncmaster/schemas/v1/__init__.py +54 -0
  70. syncmaster/schemas/v1/auth.py +12 -0
  71. syncmaster/schemas/v1/connection_types.py +9 -0
  72. syncmaster/schemas/v1/connections/__init__.py +2 -0
  73. syncmaster/schemas/v1/connections/connection.py +146 -0
  74. syncmaster/schemas/v1/connections/hdfs.py +40 -0
  75. syncmaster/schemas/v1/connections/hive.py +40 -0
  76. syncmaster/schemas/v1/connections/oracle.py +58 -0
  77. syncmaster/schemas/v1/connections/postgres.py +48 -0
  78. syncmaster/schemas/v1/connections/s3.py +66 -0
  79. syncmaster/schemas/v1/file_formats.py +7 -0
  80. syncmaster/schemas/v1/groups.py +39 -0
  81. syncmaster/schemas/v1/page.py +40 -0
  82. syncmaster/schemas/v1/queue.py +32 -0
  83. syncmaster/schemas/v1/status.py +16 -0
  84. syncmaster/schemas/v1/transfer_types.py +6 -0
  85. syncmaster/schemas/v1/transfers/__init__.py +172 -0
  86. syncmaster/schemas/v1/transfers/db.py +23 -0
  87. syncmaster/schemas/v1/transfers/file/__init__.py +2 -0
  88. syncmaster/schemas/v1/transfers/file/base.py +47 -0
  89. syncmaster/schemas/v1/transfers/file/hdfs.py +27 -0
  90. syncmaster/schemas/v1/transfers/file/s3.py +27 -0
  91. syncmaster/schemas/v1/transfers/file_format.py +29 -0
  92. syncmaster/schemas/v1/transfers/run.py +37 -0
  93. syncmaster/schemas/v1/transfers/strategy.py +15 -0
  94. syncmaster/schemas/v1/types.py +5 -0
  95. syncmaster/schemas/v1/users.py +83 -0
  96. syncmaster/worker/__init__.py +2 -0
  97. syncmaster/worker/base.py +14 -0
  98. syncmaster/worker/config.py +18 -0
  99. syncmaster/worker/controller.py +127 -0
  100. syncmaster/worker/handlers/__init__.py +2 -0
  101. syncmaster/worker/handlers/base.py +49 -0
  102. syncmaster/worker/handlers/file/__init__.py +2 -0
  103. syncmaster/worker/handlers/file/base.py +56 -0
  104. syncmaster/worker/handlers/file/hdfs.py +14 -0
  105. syncmaster/worker/handlers/file/s3.py +20 -0
  106. syncmaster/worker/handlers/hive.py +41 -0
  107. syncmaster/worker/handlers/oracle.py +48 -0
  108. syncmaster/worker/handlers/postgres.py +47 -0
  109. syncmaster/worker/spark.py +93 -0
  110. syncmaster/worker/transfer.py +85 -0
@@ -0,0 +1,2 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,87 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ import asyncio
4
+ from logging.config import fileConfig
5
+
6
+ from alembic import context
7
+ from celery.backends.database.session import ResultModelBase
8
+ from sqlalchemy import pool
9
+ from sqlalchemy.engine import Connection
10
+ from sqlalchemy.ext.asyncio import async_engine_from_config
11
+
12
+ from syncmaster.config import Settings
13
+ from syncmaster.db.base import Base
14
+
15
+ config = context.config
16
+
17
+
18
+ if config.config_file_name is not None:
19
+ fileConfig(config.config_file_name)
20
+
21
+ config.set_main_option("sqlalchemy.url", Settings().build_db_connection_uri())
22
+
23
+ target_metadata = (
24
+ Base.metadata,
25
+ ResultModelBase.metadata,
26
+ )
27
+
28
+
29
+ def run_migrations_offline() -> None:
30
+ """Run migrations in 'offline' mode.
31
+
32
+ This configures the context with just a URL
33
+ and not an Engine, though an Engine is acceptable
34
+ here as well. By skipping the Engine creation
35
+ we don't even need a DBAPI to be available.
36
+
37
+ Calls to context.execute() here emit the given string to the
38
+ script output.
39
+
40
+ """
41
+ url = config.get_main_option("sqlalchemy.url")
42
+ context.configure(
43
+ url=url,
44
+ target_metadata=target_metadata,
45
+ literal_binds=True,
46
+ dialect_opts={"paramstyle": "named"},
47
+ )
48
+
49
+ with context.begin_transaction():
50
+ context.run_migrations()
51
+
52
+
53
+ def do_run_migrations(connection: Connection) -> None:
54
+ context.configure(connection=connection, target_metadata=target_metadata)
55
+
56
+ with context.begin_transaction():
57
+ context.run_migrations()
58
+
59
+
60
+ async def run_async_migrations() -> None:
61
+ """In this scenario we need to create an Engine
62
+ and associate a connection with the context.
63
+
64
+ """
65
+
66
+ connectable = async_engine_from_config(
67
+ config.get_section(config.config_ini_section, {}),
68
+ prefix="sqlalchemy.",
69
+ poolclass=pool.NullPool,
70
+ )
71
+
72
+ async with connectable.connect() as connection:
73
+ await connection.run_sync(do_run_migrations)
74
+
75
+ await connectable.dispose()
76
+
77
+
78
+ def run_migrations_online() -> None:
79
+ """Run migrations in 'online' mode."""
80
+
81
+ asyncio.run(run_async_migrations())
82
+
83
+
84
+ if context.is_offline_mode():
85
+ run_migrations_offline()
86
+ else:
87
+ run_migrations_online()
@@ -0,0 +1,24 @@
1
+ """${message}
2
+
3
+ Revision ID: ${up_revision}
4
+ Revises: ${down_revision | comma,n}
5
+ Create Date: ${create_date}
6
+
7
+ """
8
+ from alembic import op
9
+ import sqlalchemy as sa
10
+ ${imports if imports else ""}
11
+
12
+ # revision identifiers, used by Alembic.
13
+ revision = ${repr(up_revision)}
14
+ down_revision = ${repr(down_revision)}
15
+ branch_labels = ${repr(branch_labels)}
16
+ depends_on = ${repr(depends_on)}
17
+
18
+
19
+ def upgrade() -> None:
20
+ ${upgrades if upgrades else "pass"}
21
+
22
+
23
+ def downgrade() -> None:
24
+ ${downgrades if downgrades else "pass"}
@@ -0,0 +1,242 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ """init
4
+
5
+ Revision ID: 478240cdad4b
6
+ Revises:
7
+ Create Date: 2023-11-23 11:35:18.193060
8
+
9
+ """
10
+ import sqlalchemy as sa
11
+ from alembic import op
12
+
13
+ # revision identifiers, used by Alembic.
14
+ revision = "478240cdad4b"
15
+ down_revision = None
16
+ branch_labels = None
17
+ depends_on = None
18
+
19
+
20
+ def exists_table(table_name: str) -> bool:
21
+ conn = op.get_bind()
22
+ inspector = sa.inspect(conn)
23
+ tables = inspector.get_table_names()
24
+ return table_name in tables
25
+
26
+
27
+ def upgrade() -> None:
28
+ # ### commands auto generated by Alembic - please adjust! ###
29
+ op.create_table(
30
+ "user",
31
+ sa.Column("id", sa.BigInteger(), nullable=False),
32
+ sa.Column("username", sa.String(length=256), nullable=False),
33
+ sa.Column("is_superuser", sa.Boolean(), nullable=False),
34
+ sa.Column("is_active", sa.Boolean(), nullable=False),
35
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
36
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
37
+ sa.Column("is_deleted", sa.Boolean(), nullable=False),
38
+ sa.PrimaryKeyConstraint("id", name=op.f("pk__user")),
39
+ )
40
+ op.create_index(op.f("ix__user__username"), "user", ["username"], unique=True)
41
+ op.create_table(
42
+ "group",
43
+ sa.Column("id", sa.BigInteger(), nullable=False),
44
+ sa.Column("name", sa.String(length=256), nullable=False),
45
+ sa.Column("description", sa.String(length=512), nullable=False),
46
+ sa.Column("owner_id", sa.BigInteger(), nullable=False),
47
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
48
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
49
+ sa.Column("is_deleted", sa.Boolean(), nullable=False),
50
+ sa.ForeignKeyConstraint(["owner_id"], ["user.id"], name=op.f("fk__group__owner_id__user"), ondelete="CASCADE"),
51
+ sa.PrimaryKeyConstraint("id", name=op.f("pk__group")),
52
+ sa.UniqueConstraint("name", name=op.f("uq__group__name")),
53
+ )
54
+ op.create_index(op.f("ix__group__owner_id"), "group", ["owner_id"], unique=False)
55
+ op.create_table(
56
+ "connection",
57
+ sa.Column("id", sa.BigInteger(), nullable=False),
58
+ sa.Column("group_id", sa.BigInteger(), nullable=False),
59
+ sa.Column("name", sa.String(length=128), nullable=False),
60
+ sa.Column("description", sa.String(length=512), nullable=False),
61
+ sa.Column("data", sa.JSON(), nullable=False),
62
+ sa.Column("is_deleted", sa.Boolean(), nullable=False),
63
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
64
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
65
+ sa.ForeignKeyConstraint(
66
+ ["group_id"], ["group.id"], name=op.f("fk__connection__group_id__group"), ondelete="CASCADE"
67
+ ),
68
+ sa.PrimaryKeyConstraint("id", name=op.f("pk__connection")),
69
+ sa.UniqueConstraint("name", "group_id", name=op.f("uq__connection__name_group_id")),
70
+ )
71
+ op.create_index(op.f("ix__connection__group_id"), "connection", ["group_id"], unique=False)
72
+ op.create_table(
73
+ "queue",
74
+ sa.Column("name", sa.String(length=128), nullable=False),
75
+ sa.Column("id", sa.BigInteger(), nullable=False),
76
+ sa.Column("group_id", sa.BigInteger(), nullable=False),
77
+ sa.Column("description", sa.String(length=512), nullable=False),
78
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
79
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
80
+ sa.Column("is_deleted", sa.Boolean(), nullable=False),
81
+ sa.ForeignKeyConstraint(
82
+ ["group_id"], ["group.id"], name=op.f("fk__queue__group_id__group"), ondelete="CASCADE"
83
+ ),
84
+ sa.PrimaryKeyConstraint("id", name=op.f("pk__queue")),
85
+ sa.UniqueConstraint("name", name=op.f("uq__queue__name")),
86
+ )
87
+ op.create_index(op.f("ix__queue__group_id"), "queue", ["group_id"], unique=False)
88
+ op.create_table(
89
+ "user_group",
90
+ sa.Column("user_id", sa.BigInteger(), nullable=False),
91
+ sa.Column("group_id", sa.BigInteger(), nullable=False),
92
+ sa.Column("role", sa.String(255), nullable=False),
93
+ sa.ForeignKeyConstraint(
94
+ ["group_id"], ["group.id"], name=op.f("fk__user_group__group_id__group"), ondelete="CASCADE"
95
+ ),
96
+ sa.ForeignKeyConstraint(
97
+ ["user_id"], ["user.id"], name=op.f("fk__user_group__user_id__user"), ondelete="CASCADE"
98
+ ),
99
+ sa.PrimaryKeyConstraint("user_id", "group_id", name=op.f("pk__user_group")),
100
+ )
101
+ op.create_index(op.f("ix__user_group__group_id"), "user_group", ["group_id"], unique=False)
102
+ op.create_index(op.f("ix__user_group__user_id"), "user_group", ["user_id"], unique=False)
103
+ op.create_table(
104
+ "auth_data",
105
+ sa.Column("connection_id", sa.BigInteger(), nullable=False),
106
+ sa.Column("value", sa.String(), nullable=False),
107
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
108
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
109
+ sa.ForeignKeyConstraint(
110
+ ["connection_id"],
111
+ ["connection.id"],
112
+ name=op.f("fk__auth_data__connection_id__connection"),
113
+ ondelete="CASCADE",
114
+ ),
115
+ sa.PrimaryKeyConstraint("connection_id", name=op.f("pk__auth_data")),
116
+ )
117
+ op.create_table(
118
+ "transfer",
119
+ sa.Column("id", sa.BigInteger(), nullable=False),
120
+ sa.Column("group_id", sa.BigInteger(), nullable=False),
121
+ sa.Column("name", sa.String(length=128), nullable=False),
122
+ sa.Column("description", sa.String(length=512), nullable=False),
123
+ sa.Column("source_connection_id", sa.BigInteger(), nullable=False),
124
+ sa.Column("target_connection_id", sa.BigInteger(), nullable=False),
125
+ sa.Column("strategy_params", sa.JSON(), nullable=False),
126
+ sa.Column("source_params", sa.JSON(), nullable=False),
127
+ sa.Column("target_params", sa.JSON(), nullable=False),
128
+ sa.Column("is_scheduled", sa.Boolean(), nullable=False),
129
+ sa.Column("schedule", sa.String(length=32), nullable=False),
130
+ sa.Column("queue_id", sa.BigInteger(), nullable=False),
131
+ sa.Column("is_deleted", sa.Boolean(), nullable=False),
132
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
133
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
134
+ sa.ForeignKeyConstraint(
135
+ ["group_id"], ["group.id"], name=op.f("fk__transfer__group_id__group"), ondelete="CASCADE"
136
+ ),
137
+ sa.ForeignKeyConstraint(
138
+ ["queue_id"], ["queue.id"], name=op.f("fk__transfer__queue_id__queue"), ondelete="CASCADE"
139
+ ),
140
+ sa.ForeignKeyConstraint(
141
+ ["source_connection_id"],
142
+ ["connection.id"],
143
+ name=op.f("fk__transfer__source_connection_id__connection"),
144
+ ondelete="CASCADE",
145
+ ),
146
+ sa.ForeignKeyConstraint(
147
+ ["target_connection_id"],
148
+ ["connection.id"],
149
+ name=op.f("fk__transfer__target_connection_id__connection"),
150
+ ondelete="CASCADE",
151
+ ),
152
+ sa.PrimaryKeyConstraint("id", name=op.f("pk__transfer")),
153
+ sa.UniqueConstraint("name", "group_id", name=op.f("uq__transfer__name_group_id")),
154
+ )
155
+ op.create_index(op.f("ix__transfer__group_id"), "transfer", ["group_id"], unique=False)
156
+ op.create_index(op.f("ix__transfer__source_connection_id"), "transfer", ["source_connection_id"], unique=False)
157
+ op.create_index(op.f("ix__transfer__target_connection_id"), "transfer", ["target_connection_id"], unique=False)
158
+ op.create_table(
159
+ "run",
160
+ sa.Column("id", sa.BigInteger(), nullable=False),
161
+ sa.Column("transfer_id", sa.BigInteger(), nullable=False),
162
+ sa.Column("started_at", sa.DateTime(), nullable=True),
163
+ sa.Column("ended_at", sa.DateTime(), nullable=True),
164
+ sa.Column("status", sa.String(255), nullable=False),
165
+ sa.Column("log_url", sa.String(length=512), nullable=True),
166
+ sa.Column("transfer_dump", sa.JSON(), nullable=False),
167
+ sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
168
+ sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
169
+ sa.ForeignKeyConstraint(
170
+ ["transfer_id"], ["transfer.id"], name=op.f("fk__run__transfer_id__transfer"), ondelete="CASCADE"
171
+ ),
172
+ sa.PrimaryKeyConstraint("id", name=op.f("pk__run")),
173
+ )
174
+ op.create_index(op.f("ix__run__status"), "run", ["status"], unique=False)
175
+ op.create_index(op.f("ix__run__transfer_id"), "run", ["transfer_id"], unique=False)
176
+
177
+ if exists_table("celery_taskmeta"):
178
+ return
179
+ task_id_sequence = sa.Sequence("task_id_sequence")
180
+ op.execute(sa.schema.CreateSequence(task_id_sequence, if_not_exists=True))
181
+
182
+ op.create_table(
183
+ "celery_taskmeta",
184
+ sa.Column("id", sa.Integer(), task_id_sequence, autoincrement=True, nullable=False),
185
+ sa.Column("task_id", sa.String(length=155), nullable=True),
186
+ sa.Column("status", sa.String(length=50), nullable=True),
187
+ sa.Column("result", sa.PickleType(), nullable=True),
188
+ sa.Column("date_done", sa.DateTime(), nullable=True),
189
+ sa.Column("traceback", sa.Text(), nullable=True),
190
+ sa.Column("name", sa.String(length=155), nullable=True),
191
+ sa.Column("args", sa.LargeBinary(), nullable=True),
192
+ sa.Column("kwargs", sa.LargeBinary(), nullable=True),
193
+ sa.Column("worker", sa.String(length=155), nullable=True),
194
+ sa.Column("retries", sa.Integer(), nullable=True),
195
+ sa.Column("queue", sa.String(length=155), nullable=True),
196
+ sa.PrimaryKeyConstraint("id"),
197
+ sa.UniqueConstraint("task_id"),
198
+ sqlite_autoincrement=True,
199
+ )
200
+
201
+ taskset_id_sequence = sa.Sequence("taskset_id_sequence")
202
+ op.execute(sa.schema.CreateSequence(taskset_id_sequence, if_not_exists=True))
203
+
204
+ op.create_table(
205
+ "celery_tasksetmeta",
206
+ sa.Column("id", sa.Integer(), taskset_id_sequence, autoincrement=True, nullable=False),
207
+ sa.Column("taskset_id", sa.String(length=155), nullable=True),
208
+ sa.Column("result", sa.PickleType(), nullable=True),
209
+ sa.Column("date_done", sa.DateTime(), nullable=True),
210
+ sa.PrimaryKeyConstraint("id"),
211
+ sa.UniqueConstraint("taskset_id"),
212
+ sqlite_autoincrement=True,
213
+ )
214
+ # ### end Alembic commands ###
215
+
216
+
217
+ def downgrade() -> None:
218
+ # ### commands auto generated by Alembic - please adjust! ###
219
+ op.drop_table("celery_tasksetmeta")
220
+ op.drop_table("celery_taskmeta")
221
+ op.drop_index(op.f("ix__run__transfer_id"), table_name="run")
222
+ op.drop_index(op.f("ix__run__status"), table_name="run")
223
+ op.drop_table("run")
224
+ op.drop_index(op.f("ix__transfer__target_connection_id"), table_name="transfer")
225
+ op.drop_index(op.f("ix__transfer__source_connection_id"), table_name="transfer")
226
+ op.drop_index(op.f("ix__transfer__group_id"), table_name="transfer")
227
+ op.drop_table("transfer")
228
+ op.drop_table("auth_data")
229
+ op.drop_index(op.f("ix__user_group__user_id"), table_name="user_group")
230
+ op.drop_index(op.f("ix__user_group__group_id"), table_name="user_group")
231
+ op.drop_table("user_group")
232
+ op.drop_index(op.f("ix__queue__group_id"), table_name="queue")
233
+ op.drop_table("queue")
234
+ op.drop_index(op.f("ix__connection__group_id"), table_name="connection")
235
+ op.drop_table("connection")
236
+ op.drop_index(op.f("ix__group__owner_id"), table_name="group")
237
+ op.drop_table("group")
238
+ op.drop_index(op.f("ix__user__username"), table_name="user")
239
+ op.drop_table("user")
240
+ op.execute("drop sequence task_id_sequence")
241
+ op.execute("drop sequence taskset_id_sequence")
242
+ # ### end Alembic commands ###
@@ -0,0 +1,2 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,33 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ from datetime import datetime
4
+
5
+ from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, String, func
6
+ from sqlalchemy.orm import Mapped, mapped_column
7
+
8
+
9
+ class TimestampMixin:
10
+ created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), nullable=False)
11
+ updated_at: Mapped[datetime] = mapped_column(
12
+ DateTime,
13
+ server_default=func.now(),
14
+ server_onupdate=func.now(),
15
+ onupdate=datetime.now,
16
+ nullable=False,
17
+ )
18
+
19
+
20
+ class DeletableMixin:
21
+ is_deleted: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
22
+
23
+
24
+ class ResourceMixin:
25
+ id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
26
+ name: Mapped[str] = mapped_column(String(128), nullable=False, default="")
27
+ group_id: Mapped[int] = mapped_column(
28
+ BigInteger,
29
+ ForeignKey("group.id", ondelete="CASCADE"),
30
+ nullable=False,
31
+ index=True,
32
+ )
33
+ description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
@@ -0,0 +1,194 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ from __future__ import annotations
4
+
5
+ import enum
6
+ from datetime import datetime
7
+ from typing import Any
8
+
9
+ from sqlalchemy import (
10
+ JSON,
11
+ BigInteger,
12
+ Boolean,
13
+ DateTime,
14
+ ForeignKey,
15
+ PrimaryKeyConstraint,
16
+ String,
17
+ UniqueConstraint,
18
+ )
19
+ from sqlalchemy.orm import Mapped, declared_attr, mapped_column, relationship
20
+ from sqlalchemy_utils import ChoiceType
21
+
22
+ from syncmaster.db.base import Base
23
+ from syncmaster.db.mixins import DeletableMixin, ResourceMixin, TimestampMixin
24
+
25
+
26
+ class GroupMemberRole(enum.StrEnum):
27
+ Maintainer = "Maintainer"
28
+ Developer = "Developer"
29
+ Guest = "Guest"
30
+
31
+
32
+ class User(Base, TimestampMixin, DeletableMixin):
33
+ id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
34
+ username: Mapped[str] = mapped_column(String(256), nullable=False, unique=True, index=True)
35
+ is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
36
+ is_active: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
37
+
38
+ def __repr__(self) -> str:
39
+ return f"User(username={self.username}, is_superuser={self.is_superuser}, is_active={self.is_active})"
40
+
41
+
42
+ class Group(Base, TimestampMixin, DeletableMixin):
43
+ id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
44
+ name: Mapped[str] = mapped_column(String(256), nullable=False, unique=True)
45
+ description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
46
+ owner_id: Mapped[int] = mapped_column(ForeignKey("user.id", ondelete="CASCADE"), nullable=False, index=True)
47
+
48
+ owner: Mapped[User] = relationship("User")
49
+ members: Mapped[list[User]] = relationship("User", secondary="user_group")
50
+ queue: Mapped[Queue] = relationship(back_populates="group")
51
+
52
+ def __repr__(self) -> str:
53
+ return f"Group(name={self.name}, owner_id={self.owner_id})"
54
+
55
+
56
+ class UserGroup(Base):
57
+ __table_args__ = (PrimaryKeyConstraint("user_id", "group_id"),)
58
+ user_id: Mapped[int] = mapped_column(
59
+ BigInteger,
60
+ ForeignKey("user.id", ondelete="CASCADE"),
61
+ nullable=False,
62
+ index=True,
63
+ )
64
+ group_id: Mapped[int] = mapped_column(
65
+ BigInteger,
66
+ ForeignKey("group.id", ondelete="CASCADE"),
67
+ nullable=False,
68
+ index=True,
69
+ )
70
+ role: Mapped[GroupMemberRole] = mapped_column(
71
+ ChoiceType(GroupMemberRole),
72
+ nullable=False,
73
+ )
74
+
75
+
76
+ class Connection(Base, ResourceMixin, DeletableMixin, TimestampMixin):
77
+ data: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
78
+
79
+ group: Mapped[Group] = relationship("Group")
80
+
81
+ def __repr__(self):
82
+ return f"<Connection " f"name={self.name} " f"description={self.description} " f"group_id={self.group_id}>"
83
+
84
+ @declared_attr
85
+ def __table_args__(cls) -> tuple:
86
+ return (UniqueConstraint("name", "group_id"),)
87
+
88
+
89
+ class AuthData(Base, TimestampMixin):
90
+ connection_id: Mapped[int] = mapped_column(
91
+ BigInteger,
92
+ ForeignKey("connection.id", ondelete="CASCADE"),
93
+ primary_key=True,
94
+ )
95
+ value: Mapped[str] = mapped_column(nullable=False)
96
+
97
+
98
+ class Transfer(
99
+ Base,
100
+ ResourceMixin,
101
+ DeletableMixin,
102
+ TimestampMixin,
103
+ ):
104
+ source_connection_id: Mapped[int] = mapped_column(
105
+ BigInteger,
106
+ ForeignKey("connection.id", ondelete="CASCADE"),
107
+ nullable=False,
108
+ index=True,
109
+ )
110
+ target_connection_id: Mapped[int] = mapped_column(
111
+ BigInteger,
112
+ ForeignKey("connection.id", ondelete="CASCADE"),
113
+ nullable=False,
114
+ index=True,
115
+ )
116
+ strategy_params: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
117
+ source_params: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
118
+ target_params: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
119
+ is_scheduled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
120
+ schedule: Mapped[str] = mapped_column(String(32), nullable=False, default="")
121
+ queue_id: Mapped[int] = mapped_column(
122
+ BigInteger,
123
+ ForeignKey("queue.id", ondelete="CASCADE"),
124
+ nullable=False,
125
+ )
126
+
127
+ group: Mapped[Group] = relationship("Group")
128
+ source_connection: Mapped[Connection] = relationship(foreign_keys=source_connection_id)
129
+ target_connection: Mapped[Connection] = relationship(foreign_keys=target_connection_id)
130
+ queue: Mapped[Queue] = relationship(back_populates="transfers")
131
+
132
+ @declared_attr
133
+ def __table_args__(cls) -> tuple:
134
+ return (UniqueConstraint("name", "group_id"),)
135
+
136
+
137
+ class Status(enum.StrEnum):
138
+ CREATED = "CREATED"
139
+ STARTED = "STARTED"
140
+ FAILED = "FAILED"
141
+ SEND_STOP_SIGNAL = "SEND_STOP_SIGNAL"
142
+ STOPPED = "STOPPED"
143
+ FINISHED = "FINISHED"
144
+
145
+
146
+ class Run(Base, TimestampMixin):
147
+ id: Mapped[int] = mapped_column(
148
+ BigInteger,
149
+ primary_key=True,
150
+ )
151
+ transfer_id: Mapped[int] = mapped_column(
152
+ BigInteger,
153
+ ForeignKey("transfer.id", ondelete="CASCADE"),
154
+ nullable=False,
155
+ index=True,
156
+ )
157
+ started_at: Mapped[datetime | None] = mapped_column(
158
+ DateTime,
159
+ nullable=True,
160
+ default=None,
161
+ )
162
+ ended_at: Mapped[datetime | None] = mapped_column(
163
+ DateTime,
164
+ nullable=True,
165
+ default=None,
166
+ )
167
+ status: Mapped[Status] = mapped_column(
168
+ ChoiceType(Status),
169
+ nullable=False,
170
+ default=Status.CREATED,
171
+ index=True,
172
+ )
173
+ log_url: Mapped[str] = mapped_column(String(512), nullable=True)
174
+ transfer_dump: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
175
+
176
+ transfer: Mapped[Transfer] = relationship("Transfer")
177
+
178
+ def __repr__(self):
179
+ return (
180
+ f"<Run "
181
+ f"id={self.id} "
182
+ f"transfer_id={self.transfer_id} "
183
+ f"created_at={self.created_at:%Y-%m-%d %H:%M:%S}>"
184
+ )
185
+
186
+
187
+ class Queue(Base, ResourceMixin, TimestampMixin, DeletableMixin):
188
+ name: Mapped[str] = mapped_column(String(128), nullable=False, unique=True)
189
+
190
+ transfers: Mapped[list[Transfer]] = relationship(back_populates="queue")
191
+ group: Mapped[Group] = relationship(back_populates="queue")
192
+
193
+ def __repr__(self):
194
+ return f"<Queue name={self.name} description={self.description}>"
@@ -0,0 +1,22 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from syncmaster.db.repositories.connection import ConnectionRepository
5
+ from syncmaster.db.repositories.credentials_repository import CredentialsRepository
6
+ from syncmaster.db.repositories.group import GroupRepository
7
+ from syncmaster.db.repositories.queue import QueueRepository
8
+ from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner
9
+ from syncmaster.db.repositories.run import RunRepository
10
+ from syncmaster.db.repositories.transfer import TransferRepository
11
+ from syncmaster.db.repositories.user import UserRepository
12
+
13
+ __all__ = [
14
+ "ConnectionRepository",
15
+ "CredentialsRepository",
16
+ "GroupRepository",
17
+ "QueueRepository",
18
+ "RepositoryWithOwner",
19
+ "RunRepository",
20
+ "TransferRepository",
21
+ "UserRepository",
22
+ ]