data-syncmaster 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_syncmaster-0.1.1.dist-info/LICENSE.txt +203 -0
- data_syncmaster-0.1.1.dist-info/METADATA +115 -0
- data_syncmaster-0.1.1.dist-info/RECORD +110 -0
- data_syncmaster-0.1.1.dist-info/WHEEL +4 -0
- syncmaster/__init__.py +6 -0
- syncmaster/backend/__init__.py +2 -0
- syncmaster/backend/api/__init__.py +2 -0
- syncmaster/backend/api/deps.py +20 -0
- syncmaster/backend/api/monitoring.py +10 -0
- syncmaster/backend/api/router.py +10 -0
- syncmaster/backend/api/v1/__init__.py +2 -0
- syncmaster/backend/api/v1/auth/__init__.py +2 -0
- syncmaster/backend/api/v1/auth/router.py +32 -0
- syncmaster/backend/api/v1/auth/utils.py +26 -0
- syncmaster/backend/api/v1/connections.py +300 -0
- syncmaster/backend/api/v1/groups.py +225 -0
- syncmaster/backend/api/v1/queue.py +148 -0
- syncmaster/backend/api/v1/router.py +18 -0
- syncmaster/backend/api/v1/transfers/__init__.py +2 -0
- syncmaster/backend/api/v1/transfers/router.py +469 -0
- syncmaster/backend/api/v1/transfers/utils.py +17 -0
- syncmaster/backend/api/v1/users.py +75 -0
- syncmaster/backend/export_openapi_schema.py +26 -0
- syncmaster/backend/handler.py +203 -0
- syncmaster/backend/logger.py +2 -0
- syncmaster/backend/main.py +63 -0
- syncmaster/backend/pre_start.py +94 -0
- syncmaster/backend/services/__init__.py +4 -0
- syncmaster/backend/services/auth.py +58 -0
- syncmaster/backend/services/unit_of_work.py +44 -0
- syncmaster/config.py +110 -0
- syncmaster/db/__init__.py +2 -0
- syncmaster/db/alembic.ini +41 -0
- syncmaster/db/base.py +28 -0
- syncmaster/db/factory.py +37 -0
- syncmaster/db/migrations/README +1 -0
- syncmaster/db/migrations/__init__.py +2 -0
- syncmaster/db/migrations/env.py +87 -0
- syncmaster/db/migrations/script.py.mako +24 -0
- syncmaster/db/migrations/versions/2023-11-23_478240cdad4b_init.py +242 -0
- syncmaster/db/migrations/versions/__init__.py +2 -0
- syncmaster/db/mixins.py +33 -0
- syncmaster/db/models.py +194 -0
- syncmaster/db/repositories/__init__.py +22 -0
- syncmaster/db/repositories/base.py +109 -0
- syncmaster/db/repositories/connection.py +138 -0
- syncmaster/db/repositories/credentials_repository.py +87 -0
- syncmaster/db/repositories/group.py +264 -0
- syncmaster/db/repositories/queue.py +195 -0
- syncmaster/db/repositories/repository_with_owner.py +115 -0
- syncmaster/db/repositories/run.py +78 -0
- syncmaster/db/repositories/transfer.py +202 -0
- syncmaster/db/repositories/user.py +72 -0
- syncmaster/db/repositories/utils.py +25 -0
- syncmaster/db/utils.py +31 -0
- syncmaster/dto/__init__.py +2 -0
- syncmaster/dto/connections.py +60 -0
- syncmaster/dto/transfers.py +46 -0
- syncmaster/exceptions/__init__.py +13 -0
- syncmaster/exceptions/base.py +12 -0
- syncmaster/exceptions/connection.py +28 -0
- syncmaster/exceptions/credentials.py +8 -0
- syncmaster/exceptions/group.py +27 -0
- syncmaster/exceptions/queue.py +16 -0
- syncmaster/exceptions/run.py +19 -0
- syncmaster/exceptions/transfer.py +39 -0
- syncmaster/exceptions/user.py +11 -0
- syncmaster/schemas/__init__.py +2 -0
- syncmaster/schemas/v1/__init__.py +54 -0
- syncmaster/schemas/v1/auth.py +12 -0
- syncmaster/schemas/v1/connection_types.py +9 -0
- syncmaster/schemas/v1/connections/__init__.py +2 -0
- syncmaster/schemas/v1/connections/connection.py +146 -0
- syncmaster/schemas/v1/connections/hdfs.py +40 -0
- syncmaster/schemas/v1/connections/hive.py +40 -0
- syncmaster/schemas/v1/connections/oracle.py +58 -0
- syncmaster/schemas/v1/connections/postgres.py +48 -0
- syncmaster/schemas/v1/connections/s3.py +66 -0
- syncmaster/schemas/v1/file_formats.py +7 -0
- syncmaster/schemas/v1/groups.py +39 -0
- syncmaster/schemas/v1/page.py +40 -0
- syncmaster/schemas/v1/queue.py +32 -0
- syncmaster/schemas/v1/status.py +16 -0
- syncmaster/schemas/v1/transfer_types.py +6 -0
- syncmaster/schemas/v1/transfers/__init__.py +172 -0
- syncmaster/schemas/v1/transfers/db.py +23 -0
- syncmaster/schemas/v1/transfers/file/__init__.py +2 -0
- syncmaster/schemas/v1/transfers/file/base.py +47 -0
- syncmaster/schemas/v1/transfers/file/hdfs.py +27 -0
- syncmaster/schemas/v1/transfers/file/s3.py +27 -0
- syncmaster/schemas/v1/transfers/file_format.py +29 -0
- syncmaster/schemas/v1/transfers/run.py +37 -0
- syncmaster/schemas/v1/transfers/strategy.py +15 -0
- syncmaster/schemas/v1/types.py +5 -0
- syncmaster/schemas/v1/users.py +83 -0
- syncmaster/worker/__init__.py +2 -0
- syncmaster/worker/base.py +14 -0
- syncmaster/worker/config.py +18 -0
- syncmaster/worker/controller.py +127 -0
- syncmaster/worker/handlers/__init__.py +2 -0
- syncmaster/worker/handlers/base.py +49 -0
- syncmaster/worker/handlers/file/__init__.py +2 -0
- syncmaster/worker/handlers/file/base.py +56 -0
- syncmaster/worker/handlers/file/hdfs.py +14 -0
- syncmaster/worker/handlers/file/s3.py +20 -0
- syncmaster/worker/handlers/hive.py +41 -0
- syncmaster/worker/handlers/oracle.py +48 -0
- syncmaster/worker/handlers/postgres.py +47 -0
- syncmaster/worker/spark.py +93 -0
- syncmaster/worker/transfer.py +85 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import asyncio
|
|
4
|
+
from logging.config import fileConfig
|
|
5
|
+
|
|
6
|
+
from alembic import context
|
|
7
|
+
from celery.backends.database.session import ResultModelBase
|
|
8
|
+
from sqlalchemy import pool
|
|
9
|
+
from sqlalchemy.engine import Connection
|
|
10
|
+
from sqlalchemy.ext.asyncio import async_engine_from_config
|
|
11
|
+
|
|
12
|
+
from syncmaster.config import Settings
|
|
13
|
+
from syncmaster.db.base import Base
|
|
14
|
+
|
|
15
|
+
config = context.config
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
if config.config_file_name is not None:
|
|
19
|
+
fileConfig(config.config_file_name)
|
|
20
|
+
|
|
21
|
+
config.set_main_option("sqlalchemy.url", Settings().build_db_connection_uri())
|
|
22
|
+
|
|
23
|
+
target_metadata = (
|
|
24
|
+
Base.metadata,
|
|
25
|
+
ResultModelBase.metadata,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def run_migrations_offline() -> None:
|
|
30
|
+
"""Run migrations in 'offline' mode.
|
|
31
|
+
|
|
32
|
+
This configures the context with just a URL
|
|
33
|
+
and not an Engine, though an Engine is acceptable
|
|
34
|
+
here as well. By skipping the Engine creation
|
|
35
|
+
we don't even need a DBAPI to be available.
|
|
36
|
+
|
|
37
|
+
Calls to context.execute() here emit the given string to the
|
|
38
|
+
script output.
|
|
39
|
+
|
|
40
|
+
"""
|
|
41
|
+
url = config.get_main_option("sqlalchemy.url")
|
|
42
|
+
context.configure(
|
|
43
|
+
url=url,
|
|
44
|
+
target_metadata=target_metadata,
|
|
45
|
+
literal_binds=True,
|
|
46
|
+
dialect_opts={"paramstyle": "named"},
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
with context.begin_transaction():
|
|
50
|
+
context.run_migrations()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def do_run_migrations(connection: Connection) -> None:
|
|
54
|
+
context.configure(connection=connection, target_metadata=target_metadata)
|
|
55
|
+
|
|
56
|
+
with context.begin_transaction():
|
|
57
|
+
context.run_migrations()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
async def run_async_migrations() -> None:
|
|
61
|
+
"""In this scenario we need to create an Engine
|
|
62
|
+
and associate a connection with the context.
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
connectable = async_engine_from_config(
|
|
67
|
+
config.get_section(config.config_ini_section, {}),
|
|
68
|
+
prefix="sqlalchemy.",
|
|
69
|
+
poolclass=pool.NullPool,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
async with connectable.connect() as connection:
|
|
73
|
+
await connection.run_sync(do_run_migrations)
|
|
74
|
+
|
|
75
|
+
await connectable.dispose()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def run_migrations_online() -> None:
|
|
79
|
+
"""Run migrations in 'online' mode."""
|
|
80
|
+
|
|
81
|
+
asyncio.run(run_async_migrations())
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
if context.is_offline_mode():
|
|
85
|
+
run_migrations_offline()
|
|
86
|
+
else:
|
|
87
|
+
run_migrations_online()
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""${message}
|
|
2
|
+
|
|
3
|
+
Revision ID: ${up_revision}
|
|
4
|
+
Revises: ${down_revision | comma,n}
|
|
5
|
+
Create Date: ${create_date}
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from alembic import op
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
${imports if imports else ""}
|
|
11
|
+
|
|
12
|
+
# revision identifiers, used by Alembic.
|
|
13
|
+
revision = ${repr(up_revision)}
|
|
14
|
+
down_revision = ${repr(down_revision)}
|
|
15
|
+
branch_labels = ${repr(branch_labels)}
|
|
16
|
+
depends_on = ${repr(depends_on)}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def upgrade() -> None:
|
|
20
|
+
${upgrades if upgrades else "pass"}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def downgrade() -> None:
|
|
24
|
+
${downgrades if downgrades else "pass"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""init
|
|
4
|
+
|
|
5
|
+
Revision ID: 478240cdad4b
|
|
6
|
+
Revises:
|
|
7
|
+
Create Date: 2023-11-23 11:35:18.193060
|
|
8
|
+
|
|
9
|
+
"""
|
|
10
|
+
import sqlalchemy as sa
|
|
11
|
+
from alembic import op
|
|
12
|
+
|
|
13
|
+
# revision identifiers, used by Alembic.
|
|
14
|
+
revision = "478240cdad4b"
|
|
15
|
+
down_revision = None
|
|
16
|
+
branch_labels = None
|
|
17
|
+
depends_on = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def exists_table(table_name: str) -> bool:
|
|
21
|
+
conn = op.get_bind()
|
|
22
|
+
inspector = sa.inspect(conn)
|
|
23
|
+
tables = inspector.get_table_names()
|
|
24
|
+
return table_name in tables
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def upgrade() -> None:
|
|
28
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
29
|
+
op.create_table(
|
|
30
|
+
"user",
|
|
31
|
+
sa.Column("id", sa.BigInteger(), nullable=False),
|
|
32
|
+
sa.Column("username", sa.String(length=256), nullable=False),
|
|
33
|
+
sa.Column("is_superuser", sa.Boolean(), nullable=False),
|
|
34
|
+
sa.Column("is_active", sa.Boolean(), nullable=False),
|
|
35
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
36
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
37
|
+
sa.Column("is_deleted", sa.Boolean(), nullable=False),
|
|
38
|
+
sa.PrimaryKeyConstraint("id", name=op.f("pk__user")),
|
|
39
|
+
)
|
|
40
|
+
op.create_index(op.f("ix__user__username"), "user", ["username"], unique=True)
|
|
41
|
+
op.create_table(
|
|
42
|
+
"group",
|
|
43
|
+
sa.Column("id", sa.BigInteger(), nullable=False),
|
|
44
|
+
sa.Column("name", sa.String(length=256), nullable=False),
|
|
45
|
+
sa.Column("description", sa.String(length=512), nullable=False),
|
|
46
|
+
sa.Column("owner_id", sa.BigInteger(), nullable=False),
|
|
47
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
48
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
49
|
+
sa.Column("is_deleted", sa.Boolean(), nullable=False),
|
|
50
|
+
sa.ForeignKeyConstraint(["owner_id"], ["user.id"], name=op.f("fk__group__owner_id__user"), ondelete="CASCADE"),
|
|
51
|
+
sa.PrimaryKeyConstraint("id", name=op.f("pk__group")),
|
|
52
|
+
sa.UniqueConstraint("name", name=op.f("uq__group__name")),
|
|
53
|
+
)
|
|
54
|
+
op.create_index(op.f("ix__group__owner_id"), "group", ["owner_id"], unique=False)
|
|
55
|
+
op.create_table(
|
|
56
|
+
"connection",
|
|
57
|
+
sa.Column("id", sa.BigInteger(), nullable=False),
|
|
58
|
+
sa.Column("group_id", sa.BigInteger(), nullable=False),
|
|
59
|
+
sa.Column("name", sa.String(length=128), nullable=False),
|
|
60
|
+
sa.Column("description", sa.String(length=512), nullable=False),
|
|
61
|
+
sa.Column("data", sa.JSON(), nullable=False),
|
|
62
|
+
sa.Column("is_deleted", sa.Boolean(), nullable=False),
|
|
63
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
64
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
65
|
+
sa.ForeignKeyConstraint(
|
|
66
|
+
["group_id"], ["group.id"], name=op.f("fk__connection__group_id__group"), ondelete="CASCADE"
|
|
67
|
+
),
|
|
68
|
+
sa.PrimaryKeyConstraint("id", name=op.f("pk__connection")),
|
|
69
|
+
sa.UniqueConstraint("name", "group_id", name=op.f("uq__connection__name_group_id")),
|
|
70
|
+
)
|
|
71
|
+
op.create_index(op.f("ix__connection__group_id"), "connection", ["group_id"], unique=False)
|
|
72
|
+
op.create_table(
|
|
73
|
+
"queue",
|
|
74
|
+
sa.Column("name", sa.String(length=128), nullable=False),
|
|
75
|
+
sa.Column("id", sa.BigInteger(), nullable=False),
|
|
76
|
+
sa.Column("group_id", sa.BigInteger(), nullable=False),
|
|
77
|
+
sa.Column("description", sa.String(length=512), nullable=False),
|
|
78
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
79
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
80
|
+
sa.Column("is_deleted", sa.Boolean(), nullable=False),
|
|
81
|
+
sa.ForeignKeyConstraint(
|
|
82
|
+
["group_id"], ["group.id"], name=op.f("fk__queue__group_id__group"), ondelete="CASCADE"
|
|
83
|
+
),
|
|
84
|
+
sa.PrimaryKeyConstraint("id", name=op.f("pk__queue")),
|
|
85
|
+
sa.UniqueConstraint("name", name=op.f("uq__queue__name")),
|
|
86
|
+
)
|
|
87
|
+
op.create_index(op.f("ix__queue__group_id"), "queue", ["group_id"], unique=False)
|
|
88
|
+
op.create_table(
|
|
89
|
+
"user_group",
|
|
90
|
+
sa.Column("user_id", sa.BigInteger(), nullable=False),
|
|
91
|
+
sa.Column("group_id", sa.BigInteger(), nullable=False),
|
|
92
|
+
sa.Column("role", sa.String(255), nullable=False),
|
|
93
|
+
sa.ForeignKeyConstraint(
|
|
94
|
+
["group_id"], ["group.id"], name=op.f("fk__user_group__group_id__group"), ondelete="CASCADE"
|
|
95
|
+
),
|
|
96
|
+
sa.ForeignKeyConstraint(
|
|
97
|
+
["user_id"], ["user.id"], name=op.f("fk__user_group__user_id__user"), ondelete="CASCADE"
|
|
98
|
+
),
|
|
99
|
+
sa.PrimaryKeyConstraint("user_id", "group_id", name=op.f("pk__user_group")),
|
|
100
|
+
)
|
|
101
|
+
op.create_index(op.f("ix__user_group__group_id"), "user_group", ["group_id"], unique=False)
|
|
102
|
+
op.create_index(op.f("ix__user_group__user_id"), "user_group", ["user_id"], unique=False)
|
|
103
|
+
op.create_table(
|
|
104
|
+
"auth_data",
|
|
105
|
+
sa.Column("connection_id", sa.BigInteger(), nullable=False),
|
|
106
|
+
sa.Column("value", sa.String(), nullable=False),
|
|
107
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
108
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
109
|
+
sa.ForeignKeyConstraint(
|
|
110
|
+
["connection_id"],
|
|
111
|
+
["connection.id"],
|
|
112
|
+
name=op.f("fk__auth_data__connection_id__connection"),
|
|
113
|
+
ondelete="CASCADE",
|
|
114
|
+
),
|
|
115
|
+
sa.PrimaryKeyConstraint("connection_id", name=op.f("pk__auth_data")),
|
|
116
|
+
)
|
|
117
|
+
op.create_table(
|
|
118
|
+
"transfer",
|
|
119
|
+
sa.Column("id", sa.BigInteger(), nullable=False),
|
|
120
|
+
sa.Column("group_id", sa.BigInteger(), nullable=False),
|
|
121
|
+
sa.Column("name", sa.String(length=128), nullable=False),
|
|
122
|
+
sa.Column("description", sa.String(length=512), nullable=False),
|
|
123
|
+
sa.Column("source_connection_id", sa.BigInteger(), nullable=False),
|
|
124
|
+
sa.Column("target_connection_id", sa.BigInteger(), nullable=False),
|
|
125
|
+
sa.Column("strategy_params", sa.JSON(), nullable=False),
|
|
126
|
+
sa.Column("source_params", sa.JSON(), nullable=False),
|
|
127
|
+
sa.Column("target_params", sa.JSON(), nullable=False),
|
|
128
|
+
sa.Column("is_scheduled", sa.Boolean(), nullable=False),
|
|
129
|
+
sa.Column("schedule", sa.String(length=32), nullable=False),
|
|
130
|
+
sa.Column("queue_id", sa.BigInteger(), nullable=False),
|
|
131
|
+
sa.Column("is_deleted", sa.Boolean(), nullable=False),
|
|
132
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
133
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
134
|
+
sa.ForeignKeyConstraint(
|
|
135
|
+
["group_id"], ["group.id"], name=op.f("fk__transfer__group_id__group"), ondelete="CASCADE"
|
|
136
|
+
),
|
|
137
|
+
sa.ForeignKeyConstraint(
|
|
138
|
+
["queue_id"], ["queue.id"], name=op.f("fk__transfer__queue_id__queue"), ondelete="CASCADE"
|
|
139
|
+
),
|
|
140
|
+
sa.ForeignKeyConstraint(
|
|
141
|
+
["source_connection_id"],
|
|
142
|
+
["connection.id"],
|
|
143
|
+
name=op.f("fk__transfer__source_connection_id__connection"),
|
|
144
|
+
ondelete="CASCADE",
|
|
145
|
+
),
|
|
146
|
+
sa.ForeignKeyConstraint(
|
|
147
|
+
["target_connection_id"],
|
|
148
|
+
["connection.id"],
|
|
149
|
+
name=op.f("fk__transfer__target_connection_id__connection"),
|
|
150
|
+
ondelete="CASCADE",
|
|
151
|
+
),
|
|
152
|
+
sa.PrimaryKeyConstraint("id", name=op.f("pk__transfer")),
|
|
153
|
+
sa.UniqueConstraint("name", "group_id", name=op.f("uq__transfer__name_group_id")),
|
|
154
|
+
)
|
|
155
|
+
op.create_index(op.f("ix__transfer__group_id"), "transfer", ["group_id"], unique=False)
|
|
156
|
+
op.create_index(op.f("ix__transfer__source_connection_id"), "transfer", ["source_connection_id"], unique=False)
|
|
157
|
+
op.create_index(op.f("ix__transfer__target_connection_id"), "transfer", ["target_connection_id"], unique=False)
|
|
158
|
+
op.create_table(
|
|
159
|
+
"run",
|
|
160
|
+
sa.Column("id", sa.BigInteger(), nullable=False),
|
|
161
|
+
sa.Column("transfer_id", sa.BigInteger(), nullable=False),
|
|
162
|
+
sa.Column("started_at", sa.DateTime(), nullable=True),
|
|
163
|
+
sa.Column("ended_at", sa.DateTime(), nullable=True),
|
|
164
|
+
sa.Column("status", sa.String(255), nullable=False),
|
|
165
|
+
sa.Column("log_url", sa.String(length=512), nullable=True),
|
|
166
|
+
sa.Column("transfer_dump", sa.JSON(), nullable=False),
|
|
167
|
+
sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
168
|
+
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
|
|
169
|
+
sa.ForeignKeyConstraint(
|
|
170
|
+
["transfer_id"], ["transfer.id"], name=op.f("fk__run__transfer_id__transfer"), ondelete="CASCADE"
|
|
171
|
+
),
|
|
172
|
+
sa.PrimaryKeyConstraint("id", name=op.f("pk__run")),
|
|
173
|
+
)
|
|
174
|
+
op.create_index(op.f("ix__run__status"), "run", ["status"], unique=False)
|
|
175
|
+
op.create_index(op.f("ix__run__transfer_id"), "run", ["transfer_id"], unique=False)
|
|
176
|
+
|
|
177
|
+
if exists_table("celery_taskmeta"):
|
|
178
|
+
return
|
|
179
|
+
task_id_sequence = sa.Sequence("task_id_sequence")
|
|
180
|
+
op.execute(sa.schema.CreateSequence(task_id_sequence, if_not_exists=True))
|
|
181
|
+
|
|
182
|
+
op.create_table(
|
|
183
|
+
"celery_taskmeta",
|
|
184
|
+
sa.Column("id", sa.Integer(), task_id_sequence, autoincrement=True, nullable=False),
|
|
185
|
+
sa.Column("task_id", sa.String(length=155), nullable=True),
|
|
186
|
+
sa.Column("status", sa.String(length=50), nullable=True),
|
|
187
|
+
sa.Column("result", sa.PickleType(), nullable=True),
|
|
188
|
+
sa.Column("date_done", sa.DateTime(), nullable=True),
|
|
189
|
+
sa.Column("traceback", sa.Text(), nullable=True),
|
|
190
|
+
sa.Column("name", sa.String(length=155), nullable=True),
|
|
191
|
+
sa.Column("args", sa.LargeBinary(), nullable=True),
|
|
192
|
+
sa.Column("kwargs", sa.LargeBinary(), nullable=True),
|
|
193
|
+
sa.Column("worker", sa.String(length=155), nullable=True),
|
|
194
|
+
sa.Column("retries", sa.Integer(), nullable=True),
|
|
195
|
+
sa.Column("queue", sa.String(length=155), nullable=True),
|
|
196
|
+
sa.PrimaryKeyConstraint("id"),
|
|
197
|
+
sa.UniqueConstraint("task_id"),
|
|
198
|
+
sqlite_autoincrement=True,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
taskset_id_sequence = sa.Sequence("taskset_id_sequence")
|
|
202
|
+
op.execute(sa.schema.CreateSequence(taskset_id_sequence, if_not_exists=True))
|
|
203
|
+
|
|
204
|
+
op.create_table(
|
|
205
|
+
"celery_tasksetmeta",
|
|
206
|
+
sa.Column("id", sa.Integer(), taskset_id_sequence, autoincrement=True, nullable=False),
|
|
207
|
+
sa.Column("taskset_id", sa.String(length=155), nullable=True),
|
|
208
|
+
sa.Column("result", sa.PickleType(), nullable=True),
|
|
209
|
+
sa.Column("date_done", sa.DateTime(), nullable=True),
|
|
210
|
+
sa.PrimaryKeyConstraint("id"),
|
|
211
|
+
sa.UniqueConstraint("taskset_id"),
|
|
212
|
+
sqlite_autoincrement=True,
|
|
213
|
+
)
|
|
214
|
+
# ### end Alembic commands ###
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def downgrade() -> None:
|
|
218
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
219
|
+
op.drop_table("celery_tasksetmeta")
|
|
220
|
+
op.drop_table("celery_taskmeta")
|
|
221
|
+
op.drop_index(op.f("ix__run__transfer_id"), table_name="run")
|
|
222
|
+
op.drop_index(op.f("ix__run__status"), table_name="run")
|
|
223
|
+
op.drop_table("run")
|
|
224
|
+
op.drop_index(op.f("ix__transfer__target_connection_id"), table_name="transfer")
|
|
225
|
+
op.drop_index(op.f("ix__transfer__source_connection_id"), table_name="transfer")
|
|
226
|
+
op.drop_index(op.f("ix__transfer__group_id"), table_name="transfer")
|
|
227
|
+
op.drop_table("transfer")
|
|
228
|
+
op.drop_table("auth_data")
|
|
229
|
+
op.drop_index(op.f("ix__user_group__user_id"), table_name="user_group")
|
|
230
|
+
op.drop_index(op.f("ix__user_group__group_id"), table_name="user_group")
|
|
231
|
+
op.drop_table("user_group")
|
|
232
|
+
op.drop_index(op.f("ix__queue__group_id"), table_name="queue")
|
|
233
|
+
op.drop_table("queue")
|
|
234
|
+
op.drop_index(op.f("ix__connection__group_id"), table_name="connection")
|
|
235
|
+
op.drop_table("connection")
|
|
236
|
+
op.drop_index(op.f("ix__group__owner_id"), table_name="group")
|
|
237
|
+
op.drop_table("group")
|
|
238
|
+
op.drop_index(op.f("ix__user__username"), table_name="user")
|
|
239
|
+
op.drop_table("user")
|
|
240
|
+
op.execute("drop sequence task_id_sequence")
|
|
241
|
+
op.execute("drop sequence taskset_id_sequence")
|
|
242
|
+
# ### end Alembic commands ###
|
syncmaster/db/mixins.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, String, func
|
|
6
|
+
from sqlalchemy.orm import Mapped, mapped_column
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TimestampMixin:
|
|
10
|
+
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), nullable=False)
|
|
11
|
+
updated_at: Mapped[datetime] = mapped_column(
|
|
12
|
+
DateTime,
|
|
13
|
+
server_default=func.now(),
|
|
14
|
+
server_onupdate=func.now(),
|
|
15
|
+
onupdate=datetime.now,
|
|
16
|
+
nullable=False,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DeletableMixin:
|
|
21
|
+
is_deleted: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ResourceMixin:
|
|
25
|
+
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
26
|
+
name: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
|
27
|
+
group_id: Mapped[int] = mapped_column(
|
|
28
|
+
BigInteger,
|
|
29
|
+
ForeignKey("group.id", ondelete="CASCADE"),
|
|
30
|
+
nullable=False,
|
|
31
|
+
index=True,
|
|
32
|
+
)
|
|
33
|
+
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
syncmaster/db/models.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import enum
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from sqlalchemy import (
|
|
10
|
+
JSON,
|
|
11
|
+
BigInteger,
|
|
12
|
+
Boolean,
|
|
13
|
+
DateTime,
|
|
14
|
+
ForeignKey,
|
|
15
|
+
PrimaryKeyConstraint,
|
|
16
|
+
String,
|
|
17
|
+
UniqueConstraint,
|
|
18
|
+
)
|
|
19
|
+
from sqlalchemy.orm import Mapped, declared_attr, mapped_column, relationship
|
|
20
|
+
from sqlalchemy_utils import ChoiceType
|
|
21
|
+
|
|
22
|
+
from syncmaster.db.base import Base
|
|
23
|
+
from syncmaster.db.mixins import DeletableMixin, ResourceMixin, TimestampMixin
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class GroupMemberRole(enum.StrEnum):
|
|
27
|
+
Maintainer = "Maintainer"
|
|
28
|
+
Developer = "Developer"
|
|
29
|
+
Guest = "Guest"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class User(Base, TimestampMixin, DeletableMixin):
|
|
33
|
+
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
34
|
+
username: Mapped[str] = mapped_column(String(256), nullable=False, unique=True, index=True)
|
|
35
|
+
is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
36
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
37
|
+
|
|
38
|
+
def __repr__(self) -> str:
|
|
39
|
+
return f"User(username={self.username}, is_superuser={self.is_superuser}, is_active={self.is_active})"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Group(Base, TimestampMixin, DeletableMixin):
|
|
43
|
+
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
44
|
+
name: Mapped[str] = mapped_column(String(256), nullable=False, unique=True)
|
|
45
|
+
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
|
46
|
+
owner_id: Mapped[int] = mapped_column(ForeignKey("user.id", ondelete="CASCADE"), nullable=False, index=True)
|
|
47
|
+
|
|
48
|
+
owner: Mapped[User] = relationship("User")
|
|
49
|
+
members: Mapped[list[User]] = relationship("User", secondary="user_group")
|
|
50
|
+
queue: Mapped[Queue] = relationship(back_populates="group")
|
|
51
|
+
|
|
52
|
+
def __repr__(self) -> str:
|
|
53
|
+
return f"Group(name={self.name}, owner_id={self.owner_id})"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class UserGroup(Base):
|
|
57
|
+
__table_args__ = (PrimaryKeyConstraint("user_id", "group_id"),)
|
|
58
|
+
user_id: Mapped[int] = mapped_column(
|
|
59
|
+
BigInteger,
|
|
60
|
+
ForeignKey("user.id", ondelete="CASCADE"),
|
|
61
|
+
nullable=False,
|
|
62
|
+
index=True,
|
|
63
|
+
)
|
|
64
|
+
group_id: Mapped[int] = mapped_column(
|
|
65
|
+
BigInteger,
|
|
66
|
+
ForeignKey("group.id", ondelete="CASCADE"),
|
|
67
|
+
nullable=False,
|
|
68
|
+
index=True,
|
|
69
|
+
)
|
|
70
|
+
role: Mapped[GroupMemberRole] = mapped_column(
|
|
71
|
+
ChoiceType(GroupMemberRole),
|
|
72
|
+
nullable=False,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class Connection(Base, ResourceMixin, DeletableMixin, TimestampMixin):
|
|
77
|
+
data: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
|
|
78
|
+
|
|
79
|
+
group: Mapped[Group] = relationship("Group")
|
|
80
|
+
|
|
81
|
+
def __repr__(self):
|
|
82
|
+
return f"<Connection " f"name={self.name} " f"description={self.description} " f"group_id={self.group_id}>"
|
|
83
|
+
|
|
84
|
+
@declared_attr
|
|
85
|
+
def __table_args__(cls) -> tuple:
|
|
86
|
+
return (UniqueConstraint("name", "group_id"),)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class AuthData(Base, TimestampMixin):
|
|
90
|
+
connection_id: Mapped[int] = mapped_column(
|
|
91
|
+
BigInteger,
|
|
92
|
+
ForeignKey("connection.id", ondelete="CASCADE"),
|
|
93
|
+
primary_key=True,
|
|
94
|
+
)
|
|
95
|
+
value: Mapped[str] = mapped_column(nullable=False)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Transfer(
|
|
99
|
+
Base,
|
|
100
|
+
ResourceMixin,
|
|
101
|
+
DeletableMixin,
|
|
102
|
+
TimestampMixin,
|
|
103
|
+
):
|
|
104
|
+
source_connection_id: Mapped[int] = mapped_column(
|
|
105
|
+
BigInteger,
|
|
106
|
+
ForeignKey("connection.id", ondelete="CASCADE"),
|
|
107
|
+
nullable=False,
|
|
108
|
+
index=True,
|
|
109
|
+
)
|
|
110
|
+
target_connection_id: Mapped[int] = mapped_column(
|
|
111
|
+
BigInteger,
|
|
112
|
+
ForeignKey("connection.id", ondelete="CASCADE"),
|
|
113
|
+
nullable=False,
|
|
114
|
+
index=True,
|
|
115
|
+
)
|
|
116
|
+
strategy_params: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
|
|
117
|
+
source_params: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
|
|
118
|
+
target_params: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
|
|
119
|
+
is_scheduled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
120
|
+
schedule: Mapped[str] = mapped_column(String(32), nullable=False, default="")
|
|
121
|
+
queue_id: Mapped[int] = mapped_column(
|
|
122
|
+
BigInteger,
|
|
123
|
+
ForeignKey("queue.id", ondelete="CASCADE"),
|
|
124
|
+
nullable=False,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
group: Mapped[Group] = relationship("Group")
|
|
128
|
+
source_connection: Mapped[Connection] = relationship(foreign_keys=source_connection_id)
|
|
129
|
+
target_connection: Mapped[Connection] = relationship(foreign_keys=target_connection_id)
|
|
130
|
+
queue: Mapped[Queue] = relationship(back_populates="transfers")
|
|
131
|
+
|
|
132
|
+
@declared_attr
|
|
133
|
+
def __table_args__(cls) -> tuple:
|
|
134
|
+
return (UniqueConstraint("name", "group_id"),)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class Status(enum.StrEnum):
|
|
138
|
+
CREATED = "CREATED"
|
|
139
|
+
STARTED = "STARTED"
|
|
140
|
+
FAILED = "FAILED"
|
|
141
|
+
SEND_STOP_SIGNAL = "SEND_STOP_SIGNAL"
|
|
142
|
+
STOPPED = "STOPPED"
|
|
143
|
+
FINISHED = "FINISHED"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class Run(Base, TimestampMixin):
|
|
147
|
+
id: Mapped[int] = mapped_column(
|
|
148
|
+
BigInteger,
|
|
149
|
+
primary_key=True,
|
|
150
|
+
)
|
|
151
|
+
transfer_id: Mapped[int] = mapped_column(
|
|
152
|
+
BigInteger,
|
|
153
|
+
ForeignKey("transfer.id", ondelete="CASCADE"),
|
|
154
|
+
nullable=False,
|
|
155
|
+
index=True,
|
|
156
|
+
)
|
|
157
|
+
started_at: Mapped[datetime | None] = mapped_column(
|
|
158
|
+
DateTime,
|
|
159
|
+
nullable=True,
|
|
160
|
+
default=None,
|
|
161
|
+
)
|
|
162
|
+
ended_at: Mapped[datetime | None] = mapped_column(
|
|
163
|
+
DateTime,
|
|
164
|
+
nullable=True,
|
|
165
|
+
default=None,
|
|
166
|
+
)
|
|
167
|
+
status: Mapped[Status] = mapped_column(
|
|
168
|
+
ChoiceType(Status),
|
|
169
|
+
nullable=False,
|
|
170
|
+
default=Status.CREATED,
|
|
171
|
+
index=True,
|
|
172
|
+
)
|
|
173
|
+
log_url: Mapped[str] = mapped_column(String(512), nullable=True)
|
|
174
|
+
transfer_dump: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default={})
|
|
175
|
+
|
|
176
|
+
transfer: Mapped[Transfer] = relationship("Transfer")
|
|
177
|
+
|
|
178
|
+
def __repr__(self):
|
|
179
|
+
return (
|
|
180
|
+
f"<Run "
|
|
181
|
+
f"id={self.id} "
|
|
182
|
+
f"transfer_id={self.transfer_id} "
|
|
183
|
+
f"created_at={self.created_at:%Y-%m-%d %H:%M:%S}>"
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class Queue(Base, ResourceMixin, TimestampMixin, DeletableMixin):
|
|
188
|
+
name: Mapped[str] = mapped_column(String(128), nullable=False, unique=True)
|
|
189
|
+
|
|
190
|
+
transfers: Mapped[list[Transfer]] = relationship(back_populates="queue")
|
|
191
|
+
group: Mapped[Group] = relationship(back_populates="queue")
|
|
192
|
+
|
|
193
|
+
def __repr__(self):
|
|
194
|
+
return f"<Queue name={self.name} description={self.description}>"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from syncmaster.db.repositories.connection import ConnectionRepository
|
|
5
|
+
from syncmaster.db.repositories.credentials_repository import CredentialsRepository
|
|
6
|
+
from syncmaster.db.repositories.group import GroupRepository
|
|
7
|
+
from syncmaster.db.repositories.queue import QueueRepository
|
|
8
|
+
from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner
|
|
9
|
+
from syncmaster.db.repositories.run import RunRepository
|
|
10
|
+
from syncmaster.db.repositories.transfer import TransferRepository
|
|
11
|
+
from syncmaster.db.repositories.user import UserRepository
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"ConnectionRepository",
|
|
15
|
+
"CredentialsRepository",
|
|
16
|
+
"GroupRepository",
|
|
17
|
+
"QueueRepository",
|
|
18
|
+
"RepositoryWithOwner",
|
|
19
|
+
"RunRepository",
|
|
20
|
+
"TransferRepository",
|
|
21
|
+
"UserRepository",
|
|
22
|
+
]
|