data-syncmaster 0.1.1__py3-none-any.whl

syncmaster/db/utils.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+import math
+from collections.abc import Sequence
+from enum import IntEnum
+from typing import Any
+
+
+class Pagination:
+    def __init__(self, items: Sequence[Any], page: int, page_size: int, total: int) -> None:
+        self.items = items
+        self.total = total
+        self.page_size = page_size
+        self.page = page
+        self.previous_page = None
+        self.next_page = None
+        self.has_previous = page > 1
+        if self.has_previous:
+            self.previous_page = page - 1
+        previous_items = (page - 1) * page_size
+        self.has_next = previous_items + len(items) < total
+        if self.has_next:
+            self.next_page = page + 1
+        self.pages = int(math.ceil(total / float(page_size))) or 1
+
+
+class Permission(IntEnum):
+    NONE = 0
+    READ = 1
+    WRITE = 2  # same as CREATE, UPDATE, START, STOP
+    DELETE = 3

syncmaster/dto/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0

syncmaster/dto/connections.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from dataclasses import dataclass
+
+
+@dataclass
+class ConnectionDTO:
+    pass
+
+
+@dataclass
+class PostgresConnectionDTO(ConnectionDTO):
+    host: str
+    port: int
+    user: str
+    password: str
+    additional_params: dict
+    database_name: str
+    type: str = "postgres"
+
+
+@dataclass
+class OracleConnectionDTO(ConnectionDTO):
+    host: str
+    port: int
+    user: str
+    password: str
+    additional_params: dict
+    sid: str | None = None
+    service_name: str | None = None
+    type: str = "oracle"
+
+
+@dataclass
+class HiveConnectionDTO(ConnectionDTO):
+    user: str
+    password: str
+    cluster: str
+    type: str = "hive"
+
+
+@dataclass
+class HDFSConnectionDTO(ConnectionDTO):
+    user: str
+    password: str
+    cluster: str
+    type: str = "hdfs"
+
+
+@dataclass
+class S3ConnectionDTO(ConnectionDTO):
+    host: str
+    port: int
+    access_key: str
+    secret_key: str
+    bucket: str
+    additional_params: dict
+    region: str | None = None
+    protocol: str = "https"
+    type: str = "s3"

syncmaster/dto/transfers.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from dataclasses import dataclass
+
+from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine
+
+
+@dataclass
+class TransferDTO:
+    pass
+
+
+@dataclass
+class PostgresTransferDTO(TransferDTO):
+    table_name: str
+    type: str = "postgres"
+
+
+@dataclass
+class OracleTransferDTO(TransferDTO):
+    table_name: str
+    type: str = "oracle"
+
+
+@dataclass
+class HiveTransferDTO(TransferDTO):
+    table_name: str
+    type: str = "hive"
+
+
+@dataclass
+class S3TransferDTO(TransferDTO):
+    directory_path: str
+    file_format: CSV | JSONLine | JSON
+    options: dict
+    df_schema: dict | None = None
+    type: str = "s3"
+
+
+@dataclass
+class HDFSTransferDTO(TransferDTO):
+    directory_path: str
+    file_format: CSV | JSONLine | JSON
+    options: dict
+    df_schema: dict | None = None
+    type: str = "hdfs"

syncmaster/exceptions/__init__.py

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.exceptions.base import (
+    ActionNotAllowedError,
+    EntityNotFoundError,
+    SyncmasterError,
+)
+
+__all__ = [
+    "ActionNotAllowedError",
+    "EntityNotFoundError",
+    "SyncmasterError",
+]

syncmaster/exceptions/base.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+class SyncmasterError(Exception):
+    pass
+
+
+class EntityNotFoundError(SyncmasterError):
+    pass
+
+
+class ActionNotAllowedError(SyncmasterError):
+    pass

syncmaster/exceptions/connection.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.exceptions.base import EntityNotFoundError, SyncmasterError
+
+
+class ConnectionNotFoundError(EntityNotFoundError):
+    pass
+
+
+class ConnectionOwnerError(SyncmasterError):
+    pass
+
+
+class ConnectionTypeNotRecognizedError(SyncmasterError):
+    pass
+
+
+class UserDoNotHaveRightsInTheTargetGroupError(SyncmasterError):
+    pass
+
+
+class DuplicatedConnectionNameError(SyncmasterError):
+    pass
+
+
+class ConnectionDeleteError(SyncmasterError):
+    def __init__(self, message: str) -> None:
+        self.message = message

syncmaster/exceptions/credentials.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.exceptions import SyncmasterError
+
+
+class AuthDataNotFoundError(SyncmasterError):
+    def __init__(self, message: str) -> None:
+        self.message = message

syncmaster/exceptions/group.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.exceptions.base import SyncmasterError
+
+
+class GroupNameAlreadyExistsError(SyncmasterError):
+    pass
+
+
+class GroupAdminNotFoundError(SyncmasterError):
+    pass
+
+
+class GroupAlreadyExistsError(SyncmasterError):
+    pass
+
+
+class AlreadyIsGroupMemberError(SyncmasterError):
+    pass
+
+
+class AlreadyIsNotGroupMemberError(SyncmasterError):
+    pass
+
+
+class GroupNotFoundError(SyncmasterError):
+    pass

syncmaster/exceptions/queue.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.exceptions.base import EntityNotFoundError, SyncmasterError
+
+
+class QueueNotFoundError(EntityNotFoundError):
+    pass
+
+
+class QueueDeleteError(SyncmasterError):
+    def __init__(self, message: str) -> None:
+        self.message = message
+
+
+class DifferentTransferAndQueueGroupError(SyncmasterError):
+    pass

syncmaster/exceptions/run.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.db.models import Status
+from syncmaster.exceptions.base import EntityNotFoundError, SyncmasterError
+
+
+class RunNotFoundError(EntityNotFoundError):
+    pass
+
+
+class CannotStopRunError(SyncmasterError):
+    def __init__(self, run_id: int, current_status: Status):
+        self.run_id = run_id
+        self.current_status = current_status
+
+
+class CannotConnectToTaskQueueError(SyncmasterError):
+    def __init__(self, run_id: int):
+        self.run_id = run_id

syncmaster/exceptions/transfer.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+from syncmaster.exceptions.base import SyncmasterError
+
+
+class TransferOwnerError(SyncmasterError):
+    pass
+
+
+class TransferNotFoundError(SyncmasterError):
+    pass
+
+
+class DuplicatedTransferNameError(SyncmasterError):
+    pass
+
+
+class DifferentTransferAndConnectionsGroupsError(SyncmasterError):
+    pass
+
+
+class DifferentTypeConnectionsAndParamsError(SyncmasterError):
+    def __init__(
+        self,
+        connection_type: str,
+        params_type: str,
+        conn: Literal["source", "target"],
+    ):
+        self.connection_type = connection_type
+        self.params_type = params_type
+        self.conn = conn.capitalize()
+
+    @property
+    def message(self) -> str:
+        return (
+            f"{self.conn} connection has type `{self.connection_type}` " f"but its params has `{self.params_type}` type"
+        )

syncmaster/exceptions/user.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.exceptions.base import SyncmasterError
+
+
+class UsernameAlreadyExistsError(SyncmasterError):
+    pass
+
+
+class UserNotFoundError(SyncmasterError):
+    pass

syncmaster/schemas/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0

syncmaster/schemas/v1/__init__.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from syncmaster.schemas.v1.auth import AuthTokenSchema, TokenPayloadSchema
+from syncmaster.schemas.v1.connections.connection import (
+    ConnectionCopySchema,
+    ConnectionPageSchema,
+    CreateConnectionSchema,
+    ReadConnectionSchema,
+    UpdateConnectionSchema,
+)
+from syncmaster.schemas.v1.groups import (
+    AddUserSchema,
+    CreateGroupSchema,
+    GroupPageSchema,
+    ReadGroupSchema,
+    UpdateGroupSchema,
+)
+from syncmaster.schemas.v1.queue import (
+    CreateQueueSchema,
+    QueuePageSchema,
+    ReadQueueSchema,
+    UpdateQueueSchema,
+)
+from syncmaster.schemas.v1.transfers import (
+    CopyTransferSchema,
+    CreateTransferSchema,
+    ReadFullTransferSchema,
+    ReadTransferSchema,
+    TransferPageSchema,
+    UpdateTransferSchema,
+)
+from syncmaster.schemas.v1.transfers.db import (
+    HiveReadTransferSourceAndTarget,
+    OracleReadTransferSourceAndTarget,
+    PostgresReadTransferSourceAndTarget,
+    ReadDBTransfer,
+)
+from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine
+from syncmaster.schemas.v1.transfers.run import (
+    CreateRunSchema,
+    ReadRunSchema,
+    RunPageSchema,
+    ShortRunSchema,
+)
+from syncmaster.schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy
+from syncmaster.schemas.v1.users import (
+    FullUserSchema,
+    ReadGroupMember,
+    ReadUserSchema,
+    UpdateUserSchema,
+    UserPageSchema,
+    UserPageSchemaAsGroupMember,
+)

syncmaster/schemas/v1/auth.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from pydantic import BaseModel
+
+
+class TokenPayloadSchema(BaseModel):
+    user_id: int
+
+
+class AuthTokenSchema(BaseModel):
+    access_token: str
+    refresh_token: str

syncmaster/schemas/v1/connection_types.py

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+HIVE_TYPE = Literal["hive"]
+ORACLE_TYPE = Literal["oracle"]
+POSTGRES_TYPE = Literal["postgres"]
+S3_TYPE = Literal["s3"]
+HDFS_TYPE = Literal["hdfs"]

syncmaster/schemas/v1/connections/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0

syncmaster/schemas/v1/connections/connection.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, Field, model_validator
+
+from syncmaster.schemas.v1.connections.hdfs import (
+    HDFSCreateAuthSchema,
+    HDFSCreateConnectionSchema,
+    HDFSReadAuthSchema,
+    HDFSReadConnectionSchema,
+    HDFSUpdateAuthSchema,
+    HDFSUpdateConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.hive import (
+    CreateHiveAuthSchema,
+    CreateHiveConnectionSchema,
+    ReadHiveAuthSchema,
+    ReadHiveConnectionSchema,
+    UpdateHiveAuthSchema,
+    UpdateHiveConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.oracle import (
+    CreateOracleAuthSchema,
+    CreateOracleConnectionSchema,
+    ReadOracleAuthSchema,
+    ReadOracleConnectionSchema,
+    UpdateOracleAuthSchema,
+    UpdateOracleConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.postgres import (
+    CreatePostgresAuthSchema,
+    CreatePostgresConnectionSchema,
+    ReadPostgresAuthSchema,
+    ReadPostgresConnectionSchema,
+    UpdatePostgresAuthSchema,
+    UpdatePostgresConnectionSchema,
+)
+from syncmaster.schemas.v1.connections.s3 import (
+    S3CreateAuthSchema,
+    S3CreateConnectionSchema,
+    S3ReadAuthSchema,
+    S3ReadConnectionSchema,
+    S3UpdateAuthSchema,
+    S3UpdateConnectionSchema,
+)
+from syncmaster.schemas.v1.page import PageSchema
+from syncmaster.schemas.v1.types import NameConstr
+
+ReadConnectionDataSchema = (
+    ReadHiveConnectionSchema
+    | HDFSReadConnectionSchema
+    | ReadOracleConnectionSchema
+    | ReadPostgresConnectionSchema
+    | S3ReadConnectionSchema
+)
+CreateConnectionDataSchema = (
+    CreateHiveConnectionSchema
+    | CreateOracleConnectionSchema
+    | CreatePostgresConnectionSchema
+    | HDFSCreateConnectionSchema
+    | S3CreateConnectionSchema
+)
+UpdateConnectionDataSchema = (
+    UpdateHiveConnectionSchema
+    | HDFSUpdateConnectionSchema
+    | S3UpdateConnectionSchema
+    | UpdateOracleConnectionSchema
+    | UpdatePostgresConnectionSchema
+)
+ReadConnectionAuthDataSchema = (
+    ReadHiveAuthSchema | ReadOracleAuthSchema | ReadPostgresAuthSchema | S3ReadAuthSchema | HDFSReadAuthSchema
+)
+CreateConnectionAuthDataSchema = (
+    CreateHiveAuthSchema | CreateOracleAuthSchema | CreatePostgresAuthSchema | S3CreateAuthSchema | HDFSCreateAuthSchema
+)
+UpdateConnectionAuthDataSchema = (
+    UpdateHiveAuthSchema | UpdateOracleAuthSchema | UpdatePostgresAuthSchema | S3UpdateAuthSchema | HDFSUpdateAuthSchema
+)
+
+
+class ReadConnectionSchema(BaseModel):
+    id: int
+    group_id: int
+    name: str
+    description: str
+    auth_data: ReadConnectionAuthDataSchema | None = None
+    data: ReadConnectionDataSchema = Field(
+        ...,
+        discriminator="type",
+        alias="connection_data",
+    )
+
+    class Config:
+        from_attributes = True
+        populate_by_name = True
+
+
+class UpdateConnectionSchema(BaseModel):
+    name: NameConstr | None = None  # noqa: F722
+    description: str | None = None
+    auth_data: UpdateConnectionAuthDataSchema | None = Field(discriminator="type", default=None)
+    data: UpdateConnectionDataSchema | None = Field(discriminator="type", alias="connection_data", default=None)
+
+    @model_validator(mode="before")
+    def check_types(cls, values):
+        data, auth_data = values.get("connection_data"), values.get("auth_data")
+        if data and auth_data and data.get("type") != auth_data.get("type"):
+            raise ValueError("Connection data and auth data must have same types")
+        return values
+
+
+class CreateConnectionSchema(BaseModel):
+    group_id: int = Field(..., description="Connection owner group id")
+    name: NameConstr = Field(..., description="Connection name")  # noqa: F722
+    description: str = Field(..., description="Additional description")
+    data: CreateConnectionDataSchema = Field(
+        ...,
+        discriminator="type",
+        alias="connection_data",
+        description=(
+            "Data required to connect to the database. These are the parameters that are specified in the "
+            "URL request."
+        ),
+    )
+    auth_data: CreateConnectionAuthDataSchema = Field(
+        ...,
+        discriminator="type",
+        description="Credentials for authorization",
+    )
+
+    @model_validator(mode="before")
+    def check_types(cls, values):
+        data, auth_data = values.get("data"), values.get("auth_data")
+        if data and auth_data and data.type != auth_data.type:
+            raise ValueError("Connection data and auth data must have same types")
+        return values
+
+
+class ConnectionCopySchema(BaseModel):
+    new_group_id: int
+    new_name: NameConstr | None = None  # noqa: F722
+    remove_source: bool = False
+
+
+class ConnectionPageSchema(PageSchema):
+    items: list[ReadConnectionSchema]

syncmaster/schemas/v1/connections/hdfs.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, SecretStr
+
+from syncmaster.schemas.v1.connection_types import HDFS_TYPE
+
+
+class HDFSConnectionBaseSchema(BaseModel):
+    type: HDFS_TYPE
+
+    class Config:
+        from_attributes = True
+
+
+class HDFSReadConnectionSchema(HDFSConnectionBaseSchema):
+    cluster: str
+
+
+class HDFSCreateConnectionSchema(HDFSConnectionBaseSchema):
+    cluster: str
+
+
+class HDFSUpdateConnectionSchema(HDFSConnectionBaseSchema):
+    cluster: str
+
+
+class HDFSReadAuthSchema(HDFSConnectionBaseSchema):
+    user: str
+
+
+class HDFSCreateAuthSchema(HDFSConnectionBaseSchema):
+    # Needed to create a spark session. For authorization in Kerberos
+    user: str
+    password: SecretStr
+
+
+class HDFSUpdateAuthSchema(HDFSConnectionBaseSchema):
+    user: str | None = None
+    password: SecretStr | None = None

syncmaster/schemas/v1/connections/hive.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, SecretStr
+
+from syncmaster.schemas.v1.connection_types import HIVE_TYPE
+
+
+class HiveBaseSchema(BaseModel):
+    type: HIVE_TYPE
+
+    class Config:
+        from_attributes = True
+
+
+class ReadHiveConnectionSchema(HiveBaseSchema):
+    cluster: str
+
+
+class ReadHiveAuthSchema(HiveBaseSchema):
+    user: str
+
+
+class UpdateHiveConnectionSchema(HiveBaseSchema):
+    cluster: str | None = None
+
+
+class UpdateHiveAuthSchema(HiveBaseSchema):
+    user: str | None = None
+    password: SecretStr | None = None
+
+
+class CreateHiveConnectionSchema(HiveBaseSchema):
+    cluster: str
+
+
+class CreateHiveAuthSchema(HiveBaseSchema):
+    # Needed to create a spark session. For authorization in Kerberos
+    user: str
+    password: SecretStr